diff --git a/RELEASES.json b/RELEASES.json index 956bab597ecb3039dae161f294e2eb3fa55c12d2..c9c412b5aa4c406e8e72a6536236671a0178e22b 100644 --- a/RELEASES.json +++ b/RELEASES.json @@ -1,5 +1,5 @@ { - "current": "cb05235", + "current": "2b8e675", "releases": [ { "version": "v25", diff --git a/VERSION b/VERSION index 45eddd3c0d71c4e73220c88d9d8b0a0a9e99ebec..b6e3a198380940e31f3d61c68311a5543557ee81 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -cb05235 +2b8e675 diff --git a/api/ai_enrich.py b/api/ai_enrich.py index 4c9536bf8f4d88562527ad99e16300e3dc47c0f8..5b1166fe9506bf6bdd2337ca50691536165dcad8 100644 --- a/api/ai_enrich.py +++ b/api/ai_enrich.py @@ -205,19 +205,16 @@ def _usage_tokens(body): ⚠ RETURNS None WHEN THE PROVIDER DID NOT SAY, and the caller treats that as a real unknown rather than as zero. A ledger that silently books an unmeasured call at 0 reports a cheaper run than happened, which is exactly the cost-surprise complaint R13 cites. + + ⭐⭐ W35-T41 / C7 — THE READER MOVED AND THIS IS NOW ONE LINE OVER IT. `usage_ledger.total_from` + is the same logic, in the module that owns the meter; the alternative was a second reader in a + second file answering the same question about the same key, which is the shape this repo pays + for most often ([[one-question-two-normalizers]]). ⚠ The NAME and the None-semantics stay + exactly as wave 34 wrote them, because `run_field`'s ceiling arithmetic reads this and a gate + injects against its shape — lifting the body, not the signature. """ - usage = (body or {}).get('usage') - if not isinstance(usage, dict): - return None - for key in ('total_tokens', 'totalTokens'): - got = usage.get(key) - if isinstance(got, int) and not isinstance(got, bool): - return got - ins = usage.get('prompt_tokens', usage.get('input_tokens')) - outs = usage.get('completion_tokens', usage.get('output_tokens')) - if isinstance(ins, int) and isinstance(outs, int): - return ins + outs - return None + import usage_ledger + return usage_ledger.total_from(body) #: R6's subject: the em dash (U+2014) and the en dash (U+2013), as a regex character class. @@ -372,7 +369,7 @@ def scheduled_fields(defn): def run_field(table_key, col_id, *, st, rows=None, manual=False, policy=None, budget=None, - timeout=None, ask=None): + timeout=None, ask=None, user=''): """Fill one `ai_enrich` column over the rows an automatic run is allowed to touch. Returns a REPORT and never raises for a vendor problem: @@ -391,9 +388,17 @@ def run_field(table_key, col_id, *, st, rows=None, manual=False, policy=None, bu ⚠ `ask` is injectable so a gate can exercise the ledger, the ceiling and every failure path with no network and no vendor bill. It defaults to the real `_ask`. + + ⭐⭐ W35-T41 / C7 — ONE USAGE-LEDGER LINE PER RUN, NOT PER ROW, and the arithmetic is the reason: + this loop makes one LLM call per row, so a ledger write per call would be hundreds of store + writes for one click. The run already accumulates `report['tokens']`; `calls` and `unmeasured` + join it so the meter can say "40 calls, 12,000 tokens, 3 of them unmeasured" instead of a total + with an unknown shortfall. `user` is the ledger's attribution and is passed by this file's own + callers in `routes_tables`. """ import ai_review import core.user_tables as ut + import usage_ledger defn = ut.get(table_key, st=st) or {} field = next((f for f in ut.ai_enrich_fields(defn) if f.get('key') == str(col_id)), None) @@ -440,7 +445,11 @@ def run_field(table_key, col_id, *, st, rows=None, manual=False, policy=None, bu keys = [f.get('key') for f in (defn.get('fields') or []) if f.get('key')] report = {'planned': len(plan['run']), 'filled': 0, 'failed': 0, 'skipped': dict(plan['skipped']), 'tokens': 0, 'errors': [], 'limit': None, - 'provider': '', 'model': '', 'problem': ''} + 'provider': '', 'model': '', 'problem': '', + # ⭐ C7's two counters. `calls` is what the meter bills; `unmeasured` is how many of + # them the provider declined to report tokens for, so a total is never presented as + # complete when it is a floor. + 'calls': 0, 'unmeasured': 0} if not plan['run']: return report @@ -485,8 +494,14 @@ def run_field(table_key, col_id, *, st, rows=None, manual=False, policy=None, bu continue answer, tokens, problem = caller(provider, model, text[:MAX_CELL_CHARS * 8], int(cfg.get('maxTokens') or 300), tmo) + # ⚠ COUNTED HERE, ON EVERY OUTCOME — before the `problem` branch below, because a call that + # errored or answered UNKNOWN still reached the vendor and is still billed. A meter that + # counts only filled cells reports a cheap run for one that spent its whole budget failing. + report['calls'] += 1 if isinstance(tokens, int) and not isinstance(tokens, bool): report['tokens'] += tokens + else: + report['unmeasured'] += 1 if problem or not answer or answer.strip().upper() == 'UNKNOWN': # ⛔ THE CELL IS LEFT ALONE. An errored row keeps whatever it held; only the MARK # changes, so a failed run never destroys a value it could not replace. @@ -517,4 +532,19 @@ def run_field(table_key, col_id, *, st, rows=None, manual=False, policy=None, bu if report['limit'] is None and ceiling and report['tokens'] >= ceiling: report['limit'] = ceiling_report(report['tokens'], ceiling, report['filled'], report['planned']) + # ⭐⭐ C7's LEDGER LINE — one per run, after both store writes, and only when the run actually + # called something. `total=` rather than a split: `_ask`'s contract returns a total and a gate + # injects against that shape, so splitting it here would mean inventing two numbers from one. + # + # ⚠ TWO LINES WHEN A RUN HAS BOTH KINDS, and it is not tidiness. `record` marks a line + # `unmeasured` only when its total is None, so ONE line carrying 40 calls and a partial total + # would book all forty as measured and lose the shortfall — a total presented as complete when + # it is a floor, which is the exact failure `_usage_tokens`' None-semantics exists to prevent. + _measured = report['calls'] - report['unmeasured'] + if _measured > 0: + usage_ledger.record('field_agent', report['provider'], report['model'], + total=report['tokens'], calls=_measured, st=st, user=user) + if report['unmeasured'] > 0: + usage_ledger.record('field_agent', report['provider'], report['model'], + total=None, calls=report['unmeasured'], st=st, user=user) return report diff --git a/api/ai_review.py b/api/ai_review.py index 488eb3f83d4c3a76d4abc4105ddfda1e3a2a6dac..29f16c1b0d27c10df0e9d80a9c154d45652a9e16 100644 --- a/api/ai_review.py +++ b/api/ai_review.py @@ -130,6 +130,17 @@ def _parse(text, options): return "", reason +# ⭐⭐ WAVE 35 · W35-T41 / C7 — THE TWO TRANSPORTS RETURN THE RESPONSE BODY'S `usage`. +# +# ⛔ THIS FILE'S OWN HEADER USED TO BE THE PRODUCT'S CONFESSION THAT NOTHING COUNTED: `ai_enrich`'s +# docstring says *"`ai_review.decide` -- the product's only other LLM entry -- has no token +# accounting of ANY kind"*. R9 asks for ONE meter for every AI surface, so the counting has to reach +# this transport rather than be bolted onto its caller — the caller never sees the body. +# +# ⚠ A THIRD RETURN VALUE, NOT A MUTATED ARGUMENT, and both call sites are in `decide` below. The +# tuple grew from `(text, err)` to `(text, err, usage)`; `usage` is the raw `usage` OBJECT (or None), +# because reading it is `usage_ledger`'s job and a second reader here would be the two-normalizers +# shape this repo keeps paying for. def _call_openai(p, model, system, user, timeout): r = requests.post(p["url"], timeout=timeout, headers={"Authorization": f"Bearer {os.environ[p['env']].strip()}", @@ -138,12 +149,14 @@ def _call_openai(p, model, system, user, timeout): "messages": [{"role": "system", "content": system}, {"role": "user", "content": user}]}) if r.status_code >= 400: - return "", f"{p['name']} answered {r.status_code}" + # ⚠ NO BODY ON A 4xx/5xx: an error envelope carries no usage, and a provider that refused + # before running the model has nothing to bill. The call is still COUNTED by `decide`. + return "", f"{p['name']} answered {r.status_code}", None body = r.json() choices = body.get("choices") or [] if not choices: - return "", f"{p['name']} returned no choices" - return str(((choices[0] or {}).get("message") or {}).get("content") or ""), "" + return "", f"{p['name']} returned no choices", body + return str(((choices[0] or {}).get("message") or {}).get("content") or ""), "", body def _call_anthropic(p, model, system, user, timeout): @@ -154,25 +167,34 @@ def _call_anthropic(p, model, system, user, timeout): json={"model": model, "max_tokens": 300, "system": system, "messages": [{"role": "user", "content": user}]}) if r.status_code >= 400: - return "", f"anthropic answered {r.status_code}" + return "", f"anthropic answered {r.status_code}", None body = r.json() # ⛔ stop_reason FIRST. A safety refusal is a successful 200 with an EMPTY content list, so # reading content[0] before this check turns a refusal into an IndexError inside a run. + # ⚠ A refusal IS billed and its body carries `usage`, so the body rides back on this branch too. if body.get("stop_reason") == "refusal": - return "", "anthropic declined to answer this record" + return "", "anthropic declined to answer this record", body parts = [b.get("text") or "" for b in (body.get("content") or []) if isinstance(b, dict) and b.get("type") == "text"] if not parts: - return "", "anthropic returned no text" - return "".join(parts), "" + return "", "anthropic returned no text", body + return "".join(parts), "", body -def decide(*, prompt, options, row, fields=(), label="Review", timeout=None): +def decide(*, prompt, options, row, fields=(), label="Review", timeout=None, st=None, user=""): """Pick this record's next stage. Returns `(choice, meta)`. `choice` is "" whenever a person should decide — which is every failure mode there is. `meta` carries `provider`, `model`, `reason` on success, and `problem` on refusal to answer. + + ⭐ W35-T41 / C7 — `st` and `user` are the USAGE LEDGER's target. Both default to absent because + this function's caller is `automation_engine.ai_decide(rt, ...)`, in another lane's fence: it + HAS the runtime and does not pass it yet, so until it does, a review's tokens are counted as + unattributed and REPORTED by `GET /usage` rather than dropped. See `usage_ledger`'s header for + why they cannot be resolved implicitly (measured: a contextvar does not survive a FastAPI + dependency). """ + import usage_ledger # noqa: PLC0415 opts = [str(o) for o in (options or []) if str(o).strip()] if not opts: return "", {"problem": "the review offers no next stages"} @@ -188,11 +210,22 @@ def decide(*, prompt, options, row, fields=(), label="Review", timeout=None): problems = [] for p in live: model = override or p["model"] + body = None try: - text, err = (_call_anthropic if p["shape"] == "anthropic" else _call_openai)( + text, err, body = (_call_anthropic if p["shape"] == "anthropic" else _call_openai)( p, model, system, user, tmo) except Exception as e: # noqa: BLE001 text, err = "", f"{p['name']} failed: {type(e).__name__}" + # ⭐⭐ C7 — THE LEDGER LINE, BEFORE ANY BRANCH BELOW READS THE ANSWER. + # ⛔ IT IS WRITTEN ON EVERY OUTCOME THAT REACHED A PROVIDER, including a refusal and an + # unusable answer. A meter that books only successes reports a cheap week for a run that + # spent its budget being declined — and a declined call is billed. The only path that does + # NOT record is a transport that never reached the vendor (`body is None`), which spent + # nothing. + if body is not None: + ins, outs = usage_ledger.tokens_from(body) + usage_ledger.record("ai_review", p["name"], model, ins, outs, + total=usage_ledger.total_from(body), st=st, user=user) if err: problems.append(err) continue # ladder: a dead provider degrades to the next one @@ -387,12 +420,19 @@ def _flow_from_tool_call(obj): "actions": out}, "" -def draft_flow(*, prompt, catalog, required, triggers, tables, chat=None, timeout=None): +def draft_flow(*, prompt, catalog, required, triggers, tables, chat=None, timeout=None, + st=None, user=""): """A sentence -> `(draft, refusal_sentence, provider)`. ⛔ NOTHING IS SAVED HERE. Exactly one of `draft` and `refusal_sentence` is truthy — the same contract `web_agent.run_step` keeps, so a caller has no third case to get wrong. + + ⭐ W35-T41 / C7 — `st`/`user` are the usage ledger's target, exactly as on `decide` above and for + the same reason: both of this function's callers (`automation_engine._ai_agent_plan` and + `routes_automation`'s draft door) are in lane D's fence. C7 says E adds the ledger line here and + D asserts it; the two keywords are what D has to pass for the line to be attributable. """ + import usage_ledger # noqa: PLC0415 text = str(prompt or "").strip()[:MAX_PROMPT_CHARS] if not text: return None, "type what you want the automation to do", None @@ -444,12 +484,22 @@ def draft_flow(*, prompt, catalog, required, triggers, tables, chat=None, timeou problems.append(f"{p['name']}: HTTP {r.status_code}") continue try: - calls = (((r.json().get("choices") or [{}])[0].get("message") or {}) + body = r.json() + calls = (((body.get("choices") or [{}])[0].get("message") or {}) .get("tool_calls") or []) args = json.loads(calls[0]["function"]["arguments"]) if calls else None except Exception as e: # noqa: BLE001 problems.append(f"{p['name']}: unreadable answer ({type(e).__name__})") continue + # ⭐⭐ C7 — THE LEDGER LINE. Placed after the parse rather than before it so an UNREADABLE + # answer is not double-counted by the `continue` above... ⚠ which means an unreadable answer + # is NOT counted at all, and that is a deliberate, disclosed loss: a body this code cannot + # parse is a body whose `usage` it also cannot trust, and `body` is out of scope in that + # branch by construction. The 200-with-junk case is rare and named here rather than + # silently rounded to zero. + ins, outs = usage_ledger.tokens_from(body) + usage_ledger.record("automation_draft", p["name"], p["model"], ins, outs, + total=usage_ledger.total_from(body), st=st, user=user) draft, refusal = _flow_from_tool_call(args) if draft is None and isinstance(args, dict) and str(args.get("kind")) == "refused": # ⛔ A REFUSAL IS AN ANSWER, NOT A FAULT, so it does NOT fall through to a more diff --git a/api/automation_engine.py b/api/automation_engine.py index 70e7bdd612b5789d407c897277b7d5017311ac9d..5d60218280cb2c33520bb31163c4e64ea5f6adec 100644 --- a/api/automation_engine.py +++ b/api/automation_engine.py @@ -1782,12 +1782,15 @@ def clean_schedule(raw, previous=None): # ⛔ `terminal` IS THE DEFAULT and every existing automation gets it, because a stored definition # that predates this field must not start moving records on its own the day the code ships. A # loop is a thing somebody turns on. -def clean_flow(raw, previous=None, notes=None): - """Validate the builder's ordered action list. `(flow, error)`.""" +def clean_flow(raw, previous=None, notes=None, rt=None): + """Validate the builder's ordered action list. `(flow, error)`. + + ⚠ `rt` is the tenant wall for gated action kinds (W35-T35 / C8) and is simply forwarded. + """ raw = raw if isinstance(raw, dict) else {} prev = previous if isinstance(previous, dict) else {} actions, err = clean_actions( - raw.get("actions") if "actions" in raw else prev.get("actions"), notes=notes) + raw.get("actions") if "actions" in raw else prev.get("actions"), notes=notes, rt=rt) if err: return None, err return {"actions": actions}, None @@ -2062,7 +2065,7 @@ def ig_action_pinned(defn, index): return (defn or {}).get("kind") in DISCOVERY_KINDS and index == 0 -def clean_definition(raw, previous=None, username="", notes=None): +def clean_definition(raw, previous=None, username="", notes=None, rt=None): """Whole-definition validation. Returns `(defn, error)`.""" raw = raw if isinstance(raw, dict) else {} prev = previous or {} @@ -2239,7 +2242,7 @@ def clean_definition(raw, previous=None, username="", notes=None): _prev_table, _prev_enrich = discovery_seed_spec(prev.get("kind")) flow_raw = _drop_ig_seeds( flow_raw, (prev.get("config") or {}).get("targetTable") or _prev_table, _prev_enrich) - flow, ferr = clean_flow(flow_raw, prev.get("flow"), notes=notes) + flow, ferr = clean_flow(flow_raw, prev.get("flow"), notes=notes, rt=rt) if ferr: return None, ferr # ⭐ WAVE 30 — widened with the seeder above: TikTok now HAS a pinned step 1, so the rule that @@ -2315,6 +2318,22 @@ def clean_definition(raw, previous=None, username="", notes=None): "state": dict(prev.get("state") or {}), "created": prev.get("created") or _iso(), "createdBy": prev.get("createdBy") or username, + # ⭐⭐ WAVE 35 · T37 — `system` IS STICKY ACROSS A PATCH, AND IT IS **NEVER READ FROM `raw`**. + # + # ⛔ THE BUG THIS CLOSES WAS ALREADY WRITTEN AND WOULD HAVE SHIPPED. `routes_automation. + # delete_automation` refuses any stored row carrying `system`, which is what makes a seeded + # system agent undeletable — and this function is a WHITELIST that did not carry the key, so + # the FIRST EDIT of such an agent silently stripped its marker and made it deletable. The + # seed would then mint it again on the next list call: a delete that appears to work and + # undoes itself. Found by asserting the round trip rather than by reading the code. + # + # ⛔ FROM `prev` ONLY. Reading it from `raw` would let ANY caller POST + # `{"system": "anything"}` and mint themselves an automation nobody can delete — a + # privilege escalation through a field nobody validates. The flag is granted by the seeder + # and inherited from the stored row, never asserted by a payload. + # ⚠ Same shape and same reason as the pre-set FIELD flag, which DESIGN.md §4 already + # describes as "sticky across a PATCH". One idea, one behaviour, two objects. + **({"system": _s(prev.get("system"), 40)} if prev.get("system") else {}), }, None @@ -2616,7 +2635,9 @@ def create(rt, raw, username="", notes=None): # has, instead of letting the pure validator mint a second empty one. See # `discover_default_table`; a target the caller named is never rewritten. raw = _apply_discover_default(rt, raw) - defn, err = clean_definition(raw, None, username, notes=notes) + # W35-T35 (C8): `rt=` is the tenant wall for gated action kinds. Both doors pass it; a door + # that forgot would REFUSE the kind, not admit it (`TENANT_GATED_ACTIONS` is fail-closed). + defn, err = clean_definition(raw, None, username, notes=notes, rt=rt) if err: _unmint(rt, minted) return None, err @@ -2671,7 +2692,7 @@ def patch(rt, auto_id, raw, username="", notes=None): # config that has never carried a target through the pure validator, which mints the empty # `ut_ig_candidates`. `create` alone would have left the commonest path untouched. merged = _apply_discover_default(rt, merged) - defn, err = clean_definition(merged, prev, username, notes=notes) + defn, err = clean_definition(merged, prev, username, notes=notes, rt=rt) if err: return None, err defn["id"] = str(auto_id) @@ -2987,6 +3008,21 @@ def remove(rt, auto_id): # that belongs to the WRITE and cannot be removed from here. What is gone is the expensive one. # ⚠ AND THE FALLBACK IS DELIBERATE: `all_defs` degrades to the whole read on any failure, so a # store that cannot project still deletes correctly, only slower. + # + # ⭐⭐ WAVE 35 · T34 — THE RESIDUAL IS NOW MEASURED RATHER THAN ASSERTED, and the count is + # better than the paragraph above claims. Whole `user_tables` reads per delete, against a + # runtime that has production's `get_projection`: + # TAGGED (a discovery flow that spawned columns) → 1 whole + 1 projected + # UNTAGGED (a `plain` flow) → 0 whole + 1 projected + # The untagged case pays NONE because `if not plan: return` fires before the write. The + # tagged case's ONE is `rt.update`'s own strict read, and it is a FLOOR, not a choice: + # `core/store.py::get_projection` states that a write must always read whole, because a + # read-modify-write handed a rows-less document would upload the tenant with its rows deleted. + # ⛔ Reaching ZERO needs D-179's `{flowId: [(table_key, field_key)]}` index, written where the + # presets are spawned — a feature, not an edit to this function. Nothing here can do better. + # ⚠ THOSE NUMBERS WERE UNOBSERVABLE UNTIL W35-T34: `verify_automation`'s delete-speed double + # had no `get_projection`, so `all_defs` fell back and the gate measured 2 and 1 — the + # PRE-WAVE-33 path — for two waves. The double is producer-faithful now and asserts the counts. try: import core.user_tables as _ut_defs # noqa: PLC0415 _scan = dict(_ut_defs.all_defs(rt) or {}) @@ -7864,6 +7900,288 @@ def _flow_cap_reason(table_key, rt=None): return f". {cause}." + (f" To walk them all: {fix}." if fix else "") +# --------------------------------------------------------------------------------------------- +# WAVE 35 · T36 / CONTRACT C8 / OWNER RULING R10 — THE STATEMENTS BATCH: ASSEMBLE, PARK, NEVER SEND +# +# ⛔⛔ THE TICKET'S `how:` SAID TO USE "the board's strict review posture (wave 22)". THAT POSTURE +# DOES NOT EXIST, and rebuilding it would REVERSE AN OWNER RULING rather than merely miss a symbol. +# Wave 27 R3 deleted `board()`, `move_card()`, `stages_for()`, `ensure_stage_field()` and the review +# branch of the action walk (the tombstone is above `LANE_OPS`), and its reason is a DATA reason, +# verbatim: *"the board wrote MACHINE COLUMNS INTO A TENANT'S OWN TABLE — a stage select, an `_at` +# stamp and a `_cycles` counter per automation — to render a view of state the RUN LOG already +# holds."* `retire_automation_stage_fields` and `migrate_ig_tables` still DROP those columns, so a +# batch parked in a stage field would be deleted by our own migration. +# ⇒ The batch parks on the AUTOMATION DEFINITION, the shape `runs` and `reviews` already use. Zero +# machine columns on a customer's database, and the migration has nothing to take away. +# +# ⛔ A STATEMENTS FLOW IS **BATCH**-SCOPED, NOT RECORD-SCOPED, and that is why it does not go +# through `apply_actions` at all. `run_plain` walks every row of the flow's bound table, so a +# per-record arm would assemble the whole worklist once per record; and a statements agent binds NO +# table (its customers come from the Odoo AR worklist, not a `ut_*` grid), so `run_plain` would +# have answered `partial: "no database is bound yet"` and the step would NEVER HAVE RUN. The +# precedent for a kind that defines its own scope is already here: the enrich-only branch below, +# whose saved View "ARE the set of records the automation was asked to process". +# ⚠ The `_walk` arm for this kind therefore stays a REFUSAL, not a duplicate: a `send_statement` +# dropped into an ordinary record-walking flow must say it did nothing, never half-send. +# +# ⛔⛔ AND NOTHING HERE SENDS. Not one function in this section imports the mail path. The send door +# is `routes_statements`, behind `admin_gate` + `_royal_only`, calling `collections_send. +# queue_statement`, whose SAFE_MODE guardrail lives in the DATA LAYER where no route, payload or UI +# can bypass it. This module ASSEMBLES and PARKS. A person clicks Send. + +#: How many statements one parked batch holds. Bounded for the reason `MAX_RUNS` and `MAX_REVIEWS` +#: are: this rides inside the automation DEFINITION, and an unbounded list is a serialisation cost +#: on every read of the automations bucket. +#: ⚠ WHEN IT BITES IT IS DISCLOSED, never a silent `[:N]` — R6's second sentence, and +#: [[no-unverifiable-aggregates]]. `assemble_statements` appends a note naming the number left out. +MAX_STATEMENT_BATCH = 200 + +#: The collection worklist loader, injectable so a gate can drive this without Odoo credentials. +#: Production leaves it None and the real data layer answers. Same shape as `_DRAFT_CHAT`. +_STATEMENT_ROWS = [None] + + +def _collections(): + """`modules.collections_send`, imported lazily — `platform/` is not on the path at import time + for every consumer of this module, and this is the only section that needs it.""" + import modules.collections_send as cs # noqa: PLC0415 + return cs + + +def statement_steps(defn): + """The ENABLED `send_statement` actions of this flow, top level only. + + ⚠ Top level only, deliberately, and `is_statement_flow` is what makes that safe: a batch flow + is exactly one enabled action. A `send_statement` nested inside an If is NOT a batch flow, so + it falls to the ordinary record walk and is refused there with a sentence. + """ + return [a for a in (((defn or {}).get("flow") or {}).get("actions") or []) + if isinstance(a, dict) and a.get("kind") == "send_statement" + and a.get("enabled", True)] + + +def is_statement_flow(defn): + """Is this automation a statements BATCH rather than a record walk? + + ⛔ EXACTLY ONE ENABLED ACTION, AND IT IS THIS KIND. The narrowness is the safety: a flow that + also updates records has record semantics that the batch path would silently drop, so it keeps + the ordinary walk (where the arm refuses and says so). This mirrors the enrich-only branch in + `run_plain`, which is narrow for the same stated reason. + """ + enabled = [a for a in (((defn or {}).get("flow") or {}).get("actions") or []) + if isinstance(a, dict) and a.get("enabled", True)] + return len(enabled) == 1 and enabled[0].get("kind") == "send_statement" + + +def assemble_statements(defn, log=print): + """`(batch, notes)` — the statements this configuration WOULD send, rendered, plus why anybody + was left out. **Reads Odoo read-only. Sends nothing. Writes nothing.** + + Each entry is `{customer, to, subject, html, tier, overdue}`. `html` is rendered by the SAME + `render_statement_html` the send door uses, so what a person approves is what goes out — a + review screen rendering its own approximation of the mail is a review of the wrong thing. + + ⚠ A CUSTOMER WITH NO EMAIL IS *SKIPPED AND NAMED*, never dropped. `routes_statements.send` + already separates "we could not" from "there was nowhere to send"; this keeps that distinction + at the assembly end, because a batch that silently shrinks is one nobody can reconcile. + """ + cfg = (statement_steps(defn) or [{}])[0].get("config") or {} + cs = _collections() + notes = [] + loader = _STATEMENT_ROWS[0] + rows = list(loader() if loader else cs.load_collection_list(cs.Odoo())) + tier = str(cfg.get("tier") or "").strip() + if tier: + before = len(rows) + rows = [r for r in rows if str(r.get("Tier") or "") == tier] + log(f"[aios-auto] statements: {len(rows)} of {before} customers are in tier {tier}") + subject_tpl = str(cfg.get("subject") or "") or cs.DEFAULT_SUBJECT + intro_tpl = str(cfg.get("intro") or "") or cs.DEFAULT_INTRO + footer_tpl = str(cfg.get("footer") or "") or cs.DEFAULT_FOOTER + month = _dt.date.today().strftime("%B %Y") + batch, no_email = [], [] + for row in rows: + if len(batch) >= MAX_STATEMENT_BATCH: + break + name = str(row.get("Customer") or "") + to = str(row.get("Email") or "").strip() + if not to: + no_email.append(name) + continue + try: + subject = subject_tpl.format(customer=name, company=cs.COMPANY, month=month) + except (KeyError, IndexError): + # An unknown placeholder is the person's typo, not a crash. Show what they typed — + # the same choice `routes_statements.preview` already makes. + subject = subject_tpl + batch.append({"customer": name, "to": to, "subject": _s(subject, 200), + "html": _s(cs.render_statement_html(row, intro_tpl, footer_tpl), 40000), + "tier": str(row.get("Tier") or ""), "overdue": row.get("Overdue")}) + if no_email: + shown = ", ".join(no_email[:5]) + one = len(no_email) == 1 + notes.append(f"{len(no_email)} customer{'' if one else 's'} " + f"{'has' if one else 'have'} no email address on their record and " + f"{'is' if one else 'are'} not in this batch " + f"({shown}{', and others' if len(no_email) > 5 else ''}). " + f"Add an address in Odoo and run it again") + left = len(rows) - len(batch) - len(no_email) + if left > 0: + # R6's second sentence: a limit that bites is REPORTED, with its cause and what to do. + notes.append(f"{left} more customers matched but one batch holds " + f"{MAX_STATEMENT_BATCH}. Send this batch, then run it again for the rest, or " + f"narrow the tier filter") + return batch, notes + + +def park_statements(rt, auto_id, batch, notes): + """Write the assembled batch onto the DEFINITION as `pendingStatements`. Replaces, never + appends: a batch is THIS run's worklist, and two runs' statements merged into one list is a + customer invoiced twice. + + ⚠ `flush="async"` for the reason every writer in this module gives — the read-your-writes + contract means the next `GET /automations` already sees it, and only the upload is deferred. + """ + entry = {"ts": _iso(), "count": len(batch), "sent": False, + "items": list(batch)[:MAX_STATEMENT_BATCH], + "notes": [_s(n, 300) for n in (notes or [])][:25]} + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + d = cur.get(str(auto_id)) + if d is not None: + d["pendingStatements"] = entry + return cur + + _store_update(rt, _up, flush="async") + return entry + + +def clear_statements(rt, auto_id): + """Drop a parked batch — after it is sent, or when somebody discards it.""" + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + d = cur.get(str(auto_id)) + if d is not None: + d.pop("pendingStatements", None) + return cur + + _store_update(rt, _up, flush="async") + + +def run_statements(rt, defn, username="automation", log=print, step=_no_step): + """The batch runner: assemble, park, tell somebody. Returns `run_plain`'s 5-tuple. + + ⛔ THE STATE IS `partial`, NEVER `ok`, AND THAT IS DELIBERATE. `ok` would put a green dot over + a job that is only half done — the statements exist and NOBODY HAS SENT THEM. `partial` is this + module's honest-progress state and the summary says exactly what is waiting, which is what makes + the run notification (`_notify_run`) read as "come and look" rather than "done". + """ + if not is_statement_tenant(rt): + # Belt and braces behind `clean_actions`' wall: a definition stored before the wall, or + # copied between tenants, must not assemble another company's customer list. + return ("error", "statements are not configured for this workspace", {}, [], {}) + step("reading the collection list") + try: + batch, notes = assemble_statements(defn, log=log) + except Exception as exc: # noqa: BLE001 + return ("error", f"the collection list could not be read: {str(exc)[:200]}", {}, [], {}) + step("rendering statements") + park_statements(rt, defn.get("id"), batch, notes) + counts = {"statements": len(batch)} + if not batch: + return ("partial", "no customers matched, so there is nothing to review", counts, [], {}) + one = len(batch) == 1 + return ("partial", + f"{len(batch)} statement{'' if one else 's'} " + f"{'is' if one else 'are'} ready for review. Nothing has been sent. Open this agent " + f"and click Send to release them", + counts, [], {}) + + +#: ⭐⭐ WAVE 35 · T37 — THE SEEDED ROYAL IMPORTS STATEMENTS AGENT. +#: A FIXED id, because the row's own EXISTENCE is the idempotency key (see `seed_statements_agent`). +STATEMENTS_AGENT_ID = "system:statements" +SYSTEM_STATEMENTS = "statements" +#: First of the month, 06:00. A monthly statement run is what the collections worklist is for, and +#: the hour matches the Odoo sync default so the numbers it reads are the night's. +STATEMENTS_DEFAULT_CRON = "0 6 1 * *" + + +def seed_statements_agent(rt, username="system"): + """Mint Royal Imports' "Monthly statements" agent ONCE, **switched OFF**. Returns the + definition if it wrote one, else None. + + ⛔⛔ IT ARRIVES `enabled: False` AND THAT IS THE TICKET'S OWN TRAP, not a preference. An agent + that ships enabled has scheduled itself against real customers before a single person has read + its configuration — and this one's action queues mail to the tenant's actual debtors. + + ⭐ WHY STORED, WHEN WAVE 34's SYSTEM AGENT IS DERIVED. `_odoo_sync_row` gives three reasons for + deriving and only one survives contact with THIS agent (raised as `ASK D-11`): + · "two copies of one cadence" — does not apply: Odoo's cadence already lives in the connector + config, so a stored copy could drift; a statements schedule has no other home to drift from. + · "stored means the failure-pause can disable it" — INVERTS: auto-pausing statements after K + consecutive failures is correct (a dead credential should stop it), where pausing the Odoo + cadence would silently break infrastructure. + · "nothing can seed it" — real, and answered here: ONE write per tenant ever, by the + CONTAINER (which is what D-195 asks for; what it forbids is a CLI writing while the Space + is up), behind the existence check below. + And deriving cannot meet T37 anyway: a derived row is not in the automations bucket, so edit, + toggle, schedule and run would each need a bespoke door — four new mechanisms to avoid one + guarded write. The Odoo agent escapes that only because a separate resync loop already does its + work. + + ⛔ THE EXISTENCE CHECK IS THE WHOLE IDEMPOTENCY STORY, AND IT IS SAFE ONLY BECAUSE THE ROW + CANNOT BE DELETED. `routes_automation.delete_automation` refuses any stored row carrying + `system` (409, with a sentence), so "the row is present" can never become false behind our + back. If that refusal is ever relaxed, THIS FUNCTION NEEDS A SEPARATE DURABLE FLAG — otherwise + deleting the agent resurrects it on the next list call, which is a delete that undoes itself. + ⚠ And `system` is STICKY through `clean_definition` (see its return) for the same reason: an + edit that stripped the marker would make the row deletable and re-open exactly that hole. + """ + if not is_statement_tenant(rt): + return None + existing = all_definitions(rt) or {} + if STATEMENTS_AGENT_ID in existing: + return None + raw = { + "name": "Monthly statements", + "kind": "plain", + # ⚠ A SCHEDULE THAT IS OFF, not an absent schedule: the cron is the CONFIGURATION T37 asks + # a person to open and read, and a blank one would make them invent it before they could + # judge it. + "schedule": {"cron": STATEMENTS_DEFAULT_CRON, "enabled": False}, + "trigger": {"key": "schedule"}, + "flow": {"actions": [{ + "id": "act_1", "kind": "send_statement", + # ⚠ TIER BLANK = every tier, which is the honest default: narrowing to "A-Urgent" would + # be us deciding who gets chased, and the blank is the value the config panel shows as + # "all of them" rather than an empty box. + "config": {"tier": "", "subject": "", "intro": "", "footer": ""}, + }]}, + } + defn, err = clean_definition(raw, None, username, rt=rt) + if err: + return None + defn["id"] = STATEMENTS_AGENT_ID + # Stamped AFTER the clean, because `clean_definition` reads this key from `prev` only — a + # payload may never assert it (see that function's note). + defn["system"] = SYSTEM_STATEMENTS + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + # ⚠ RE-CHECKED INSIDE THE MUTATION. Two concurrent list calls can both pass the read above; + # the store applies mutations under its own lock, so this is where "only once" is actually + # decided. Without it the second writer would overwrite a row a person may already have + # edited, silently resetting their schedule and template. + if STATEMENTS_AGENT_ID not in cur: + cur[STATEMENTS_AGENT_ID] = defn + return cur + + _store_update(rt, _up, flush="async") + return defn + + def run_plain(rt, defn, username="automation", log=print, step=_no_step, rows=None): """⭐ WAVE 24 / R6 — the runner for an automation with NO machine step: its flow IS the whole automation. Returns the same 5-tuple every other runner does, so `run_now` needs no special @@ -7878,6 +8196,14 @@ def run_plain(rt, defn, username="automation", log=print, step=_no_step, rows=No exact defect this module names in three other places. It reports the honest state and says which fact is missing. """ + # ⭐⭐ WAVE 35 · T36 — THE BATCH FLOW BRANCHES BEFORE THE TABLE CHECK, and the ORDER is the + # whole of it. A statements agent binds no `ut_*` table (its customers come from the Odoo AR + # worklist), so leaving this below the `if not table` return would answer "no database is bound + # yet" and the step would never run — a feature that is whole, gated and unreachable, which is + # this repo's most-repeated failure. See the section above `run_plain` for why it is not an + # `apply_actions` arm. + if is_statement_flow(defn): + return run_statements(rt, defn, username=username, log=log, step=step) table = _flow_table(defn) if not table: return ("partial", @@ -8617,7 +8943,7 @@ AI_AGENT_MAX_STEPS = 12 _AI_AGENT_CHAT = [None] -def _ai_agent_plan(cfg, row, log=print): +def _ai_agent_plan(cfg, row, log=print, st=None, user=""): """A description + one record -> `(steps, "")` for `run_plan`, or `(None, sentence)`. ⭐⭐ W33-T56. This is the whole of the fuzzy step: the instruction and the record's own values @@ -8646,6 +8972,10 @@ def _ai_agent_plan(cfg, row, log=print): catalog=web_only, required={k: v for k, v in ACTION_REQUIRED.items() if k in WEB_KINDS}, triggers=[], tables=[], + # ⭐ W35 · C7 (`NOTE E-16`) — threaded IN from `_walk` rather than resolved here: this + # function is pure over `(cfg, row)` by design and giving it a store handle of its own + # would be a second way to reach the tenant. + st=st, user=user, chat=_AI_AGENT_CHAT[0]) if why or not draft: return None, (why or "the assistant produced no steps for that instruction") @@ -8865,6 +9195,30 @@ ACTION_CATALOG = [ "history"}, {"kind": "send_email", "label": "Send email", "group": "Connected", "ready": False, "detail": "Needs a send scope on the Gmail connection"}, + # ⭐⭐ WAVE 35 · T35 / CONTRACT C8 / OWNER RULING R10 — STATEMENTS BECOME AN AGENT STEP. + # + # Owner item 14: move Statements out of Settings and into the agent automation, "templatic and + # easily toggleable". R10 is the half that decides the shape: the step ASSEMBLES and PARKS a + # batch in a review stage, and **nothing sends without a human click**. So this row's promise + # is deliberately "prepares", not "sends" — the label a person reads must not describe an act + # the step does not perform. + # + # ⛔ ADDED BESIDE `send_email`, NEVER BY REPURPOSING IT (the ticket's own trap, D-295): a + # stored automation naming a kind that no longer exists is refused FOREVER rather than dropped + # with a reason, so adding a kind is cheap and re-pointing one is not. + # + # ⛔ `ready: True` IS WHAT MAKES IT STORABLE — `clean_actions` refuses any row whose `ready` is + # false — and the note at `enrich_tiktok` above is the reason the RUNNER ARM lands in the same + # wave rather than after it: `_walk` has no terminal `else`, so a catalog row ahead of its arm + # is addable, storable and SILENTLY INERT, which is worse than not existing. T35 lands an arm + # that reports it is not configured; T36 makes it park a real batch. + # + # ⚠ TENANT-GATED, not `ready: False`: `TENANT_GATED_ACTIONS` withholds this row from every + # tenant but #0, because the send client behind it is env-credentialed and is tenant #0's. + {"kind": "send_statement", "label": "Prepare customer statements", "group": "Connected", + "ready": True, + "detail": "Assemble this month's statements and park them for review. Nothing is sent until " + "somebody opens the batch and clicks Send"}, {"kind": "slack", "label": "Send Slack message", "group": "Connected", "ready": False, "detail": "Needs the Slack connector"}, # ── 4. ADVANCED LOGIC ──────────────────────────────────────────────────────────────────── @@ -8905,10 +9259,63 @@ def _connector_meta(key): return next((dict(v) for v in TRIGGER_CONNECTOR.values() if v.get("key") == key), None) -def action_catalog(): +#: ⭐⭐ WAVE 35 · T35 / CONTRACT C8 — THE TENANT PREDICATE `routes_statements._royal_only` USES, +#: LIFTED SO THERE IS EXACTLY ONE COPY OF IT. C8's words are "the predicate is imported, never +#: re-expressed", and this is the direction that import can run: `automation_engine` imports no +#: route module and no FastAPI (checked), and breaking that to reach `_royal_only` would drag +#: `deps` + `routes_admin` into the engine. So the ENGINE holds the test and the ROUTE calls it. +#: +#: ⛔⛔ THIS IS NOT `odoo_relational.is_royal`, AND THE DIFFERENCE IS A SEND. That one asks "is this +#: tenant ENTITLED to Odoo databases" over `RI_SLUGS = ("", "royal-imports")` — it answers TRUE for +#: the EMPTY slug and lower-cases its input. This one is `_royal_only`'s exact test: the runtime's +#: own `key`, matched exactly. A tenant whose key never got set would pass `is_royal("")` and then +#: queue statements THROUGH TENANT #0'S ENV-CREDENTIALED SEND CLIENT, i.e. email another company's +#: customers over Royal Imports' name. Entitlement to READ is not authority to SEND, and the two +#: questions keep their two predicates on purpose. Do not "unify" them. +ROYAL_TENANT_KEY = "royal-imports" + + +def is_statement_tenant(rt): + """Exactly `routes_statements._royal_only`'s test, as a boolean over the RUNTIME. + + Keyed on the runtime, never on a request field: a tenant is a property of the SESSION, so a + payload cannot argue its way into another company's sender. + """ + return getattr(rt, "key", None) == ROYAL_TENANT_KEY + + +#: Kinds only SOME tenants may see or store, as `{kind: predicate(rt) -> bool}`. +#: +#: ⛔ FAIL-CLOSED ON `rt=None`, AND THAT IS THE WHOLE SAFETY ARGUMENT. Every reader below treats an +#: absent runtime as "not allowed", so a call site that forgets to pass one makes the action VANISH +#: rather than become universal. The opposite default would mean any future caller of +#: `action_catalog()` or `clean_actions()` silently offers tenant #0's send door to every tenant — +#: an omission that is invisible in review and loud only in production [[default-must-pass-its-own-guard]]. +TENANT_GATED_ACTIONS = {"send_statement": is_statement_tenant} + +#: The dunning buckets a statements step may filter on. ⚠ ONE LITERAL, TWO READERS: this and +#: `routes_statements.statements()`'s `"tiers"` field are the same four strings, and a step +#: configured against a tier the sender's worklist does not produce would filter to nothing and +#: report success. `routes_statements` imports this rather than repeating it. +STATEMENT_TIERS = ("A-Urgent", "B-Active", "C-Light", "Monitor") + + +def _tenant_may_use(kind, rt): + """May this runtime see/store this action kind? Ungated kinds are always yes.""" + gate = TENANT_GATED_ACTIONS.get(str(kind or "")) + return True if gate is None else bool(rt is not None and gate(rt)) + + +def action_catalog(rt=None): """The catalog as the wire carries it — a copy, because a caller that mutated the module constant would change every later reader's answer. + ⭐⭐ WAVE 35 · T35 (C8): `rt` filters TENANT-GATED rows out entirely — not `ready: False`, not + `menu: False`, but ABSENT. A tenant that may not send statements should not learn that the + capability exists, and `ready: False` renders as "coming soon", which is a promise we are not + making to them. ⚠ Called with no `rt` the gated rows are withheld (fail-closed): see + `TENANT_GATED_ACTIONS`. + ⭐ WAVE 24 (C-ACT): each row is stamped with its `groupOrder`, DERIVED from `ACTION_GROUP_ORDER` rather than hand-written per row, so a group cannot be given two different orders by two rows that claim to be in it. A group nobody has ordered sorts LAST @@ -8933,6 +9340,8 @@ def action_catalog(): last = max(ACTION_GROUP_ORDER.values()) + 1 out = [] for a in ACTION_CATALOG: + if not _tenant_may_use(a.get("kind"), rt): + continue row = {**a, "groupOrder": ACTION_GROUP_ORDER.get(a.get("group"), last), "menu": bool(a.get("menu", True))} conn = _connector_meta(a.get("connector")) @@ -9024,9 +9433,17 @@ def _action_label(act): return str(act.get("kind") or "Action") -def clean_actions(raw, depth=0, _seen=None, _count=None, notes=None): +def clean_actions(raw, depth=0, _seen=None, _count=None, notes=None, rt=None): """Validate `flow.actions`. Returns `(actions, error)` — refuses, never coerces. + ⭐⭐ WAVE 35 · T35 (C8) — `rt` IS THE TENANT WALL ON THE STORE SIDE, and it is a separate wall + from `action_catalog(rt)`'s. Withholding a row from the MENU stops it being offered; it does + not stop a hand-written body naming the kind, and the picker is not a security boundary + [[opening-a-route-widens-every-field]]. ⚠ Keyword-only in effect and defaulting to None, so + every existing caller is untouched by construction — the same shape `notes` used, for the + reason this docstring already gives about a signature change failing at RUN, not at import. + ⛔ `rt=None` REFUSES a gated kind rather than allowing it (`TENANT_GATED_ACTIONS`). + Ids are STABLE: a caller's well-formed `id` is kept, so selecting an action in the builder survives a Save. A missing or colliding one is minted `act_`; minting on every clean would move the selection under the person editing it. @@ -9064,6 +9481,15 @@ def clean_actions(raw, depth=0, _seen=None, _count=None, notes=None): if not row["ready"]: return None, (f"{row['label']!r} is on the menu but not built yet. " f"{row['detail'][0].lower()}{row['detail'][1:]}") + # ⭐⭐ W35-T35 (C8) — THE TENANT WALL. Refused, not dropped, and this is the one place in + # this function where a refusal is right: D-65's "drop, never refuse" protects a stored + # automation from becoming permanently unsavable, and NO tenant this can refuse has ever + # been able to store one — the kind is withheld from their catalog, so there is no legacy + # body to strand. Dropping it silently would instead let a flow save, look saved, and never + # do the step the person configured. + if not _tenant_may_use(kind, rt): + return None, (f"{row['label']!r} is not available in this workspace. It sends as " + f"Royal Imports, using Royal Imports' own mail credentials") count[0] += 1 if count[0] > MAX_ACTIONS: return None, f"an automation runs at most {MAX_ACTIONS} actions" @@ -9105,7 +9531,7 @@ def clean_actions(raw, depth=0, _seen=None, _count=None, notes=None): if cerr: return None, cerr cfg_raw = entry.get("config") if isinstance(entry.get("config"), dict) else {} - cfg, cerr = _clean_action_config(kind, cfg_raw, depth, seen, count, notes=notes) + cfg, cerr = _clean_action_config(kind, cfg_raw, depth, seen, count, notes=notes, rt=rt) if cerr: return None, cerr # ⛔ D-75 — THE ALLOWLIST'S OWN DROPS, DIFFED HERE RATHER THAN REPORTED BY EACH ARM. Every @@ -9125,8 +9551,14 @@ def clean_actions(raw, depth=0, _seen=None, _count=None, notes=None): return out, None -def _clean_action_config(kind, cfg, depth, seen, count, notes=None): - """One action's `config`, per kind. Returns `(config, error)`.""" +def _clean_action_config(kind, cfg, depth, seen, count, notes=None, rt=None): + """One action's `config`, per kind. Returns `(config, error)`. + + ⚠ `rt` is threaded ONLY so the `group` arm can hand it back to `clean_actions` for the branch + recursion (W35-T35 / C8). Without it a tenant-gated kind is refused at the top level and + ACCEPTED inside an If, which is the half of a flow `_walk_actions`' own docstring says people + cannot see. + """ if kind == "group": # ⭐ WAVE 24 · C-FORK (owner ruling R8) — a group is a FORK now: `{branches: [...]}`, # each `{id, label, cond, actions}`. It was one condition with one action list, i.e. an @@ -9164,7 +9596,7 @@ def _clean_action_config(kind, cfg, depth, seen, count, notes=None): # D-75: threaded into the branch too — an unconfigured step inside an If is # exactly the one a person cannot see, which is `_walk_actions`' own argument. kids, kerr = clean_actions(br.get("actions"), depth + 1, seen, count, - notes=notes) + notes=notes, rt=rt) if kerr: return None, kerr if not kids: @@ -9229,6 +9661,34 @@ def _clean_action_config(kind, cfg, depth, seen, count, notes=None): f"unique on it. It writes: " + ", ".join(sorted(clean_vals))) out["uniqueOn"] = unique return out, None + if kind == "send_statement": + # ⭐⭐ WAVE 35 · T35 / R10 — WHAT A STATEMENTS STEP IS CONFIGURED WITH: a customer filter + # and a template. Both are the SENDER's own vocabulary (`routes_statements.send` passes + # `templates: {subject, intro, footer}` straight to `collections_send.queue_statement`), + # so this arm names no field the send door does not already take. + # + # ⚠ EVERY TEMPLATE FIELD IS OPTIONAL AND EMPTY MEANS "THE DEFAULT", which is exactly how + # the send door already reads them (`t.get("subject") or cs.DEFAULT_SUBJECT`). Storing our + # own copy of the default instead would freeze today's wording into every stored agent and + # silently stop tracking `collections_send`'s. + out = {} + tier = _s(cfg.get("tier"), 40).strip() + # ⛔ REFUSED, NOT COERCED, AND THE ANSWER LISTS THE REAL ONES — the same shape the connector + # cadence uses. A tier nobody sends to would filter the batch to zero customers and report + # a successful run: a silent no-op is the worst outcome available here, because the person + # believes their customers were invoiced. + # ⚠ `""` IS LEGAL and means EVERY tier. It is the stored default, so an agent saved before + # anybody picks a filter does not change meaning the day this key arrives (D-65's rule). + if tier and tier not in STATEMENT_TIERS: + return None, (f"{tier!r} is not a collection tier. Pick one of: " + + ", ".join(STATEMENT_TIERS) + ", or leave it blank for all of them") + out["tier"] = tier + for key, cap in (("subject", 200), ("intro", 4000), ("footer", 4000)): + out[key] = _s(cfg.get(key), cap) + lbl = " ".join(_s(cfg.get("label"), 60).split()) + if lbl: + out["label"] = lbl + return out, None if kind in ENRICH_KINDS: # ⭐ WAVE 30 · T08 — BOTH networks share this branch. See `ENRICH_KINDS`: the selection, the # cooldown, the limit clamps and their reasons are network-agnostic, and a second copy for @@ -10348,6 +10808,29 @@ def apply_actions(rt, defn, table_key, row_ids, username="automation", log=print # the table would silently apply one action's uniqueness rule to the other's rows. creates.setdefault((target, str(cfg.get("uniqueOn") or "")), []).append(vals) counts["created"] += 1 + elif kind == "send_statement": + # ⭐⭐ WAVE 35 · T35 / R10 — THE ARM LANDS WITH THE CATALOG ROW, ON PURPOSE. + # + # `_walk` has no terminal `else` (see the note on `enrich_tiktok` in the catalog): + # an unknown kind is walked, COUNTED, reports the run `ok` and writes nothing. So a + # catalog row whose arm arrives in a later ticket is addable, storable and silently + # inert — a step a person configured, that reports success and does nothing. This + # arm exists so that window never opens. + # + # ⛔ T35 DOES NOT SEND AND DOES NOT PARK. R10's review stage is W35-T36; until it + # lands this says so out loud and counts the record as blocked, which is the same + # shape `ai_agent` uses for a step it cannot perform. It must never fall through to + # "ok". + # ⛔ AND IT NEVER SENDS FROM HERE, in this wave or any later one. The send door is + # `routes_statements`, behind SAFE_MODE + `admin_gate` + the tenant gate, reached by + # a human click on the review batch. This arm's whole job is to PREPARE. + if "send_statement_pending" not in web_notes: + web_notes.append("send_statement_pending") + log("[aios-auto] send_statement: statements are assembled for review, not " + "sent. The review batch is not configured yet, so nothing was prepared " + "and nothing was sent.") + counts["webBlocked"] += 1 + continue elif kind == "ai_agent": # ⭐⭐ W33-T56 (owner item 7, ruling R3) — THE FUZZY STEP, AT RUN TIME. # @@ -10378,7 +10861,9 @@ def apply_actions(rt, defn, table_key, row_ids, username="automation", log=print f"browser jobs of about 10-30 seconds each.") counts["webBlocked"] += 1 continue - _plan, _why = _ai_agent_plan(cfg, row, log) + # W35 · C7: `st` + `user` so the model spend is attributed (`NOTE E-16`). + _plan, _why = _ai_agent_plan(cfg, row, log, st=rt, + user=str(defn.get("createdBy") or "")) if _why: # ⛔ NAMED, NEVER OPAQUE — the second half of the `done-when`. "The assistant # could not work out how to do that" with the reason attached is actionable; @@ -11770,9 +12255,16 @@ def ai_decide(rt, defn, act, row, row_id="", log=print): (ut_get(rt, ((defn.get("config") or {}).get("targetTable") or (defn.get("trigger") or {}).get("table") or "")) or {} ).get("fields") or []] + # ⭐ W35 · CONTRACT C7 (`NOTE E-16`) — ATTRIBUTE THE SPEND. `st=` is what lets the ledger write + # to the right tenant's store and `user=` is who it is billed to; without them the call is + # counted as UNATTRIBUTED, which is a meter that reports a total nobody can act on. + # ⚠ `createdBy` is the honest actor here: an AI review decision is made ON BEHALF of the + # automation, by a scheduler, with no person at the keyboard. Naming whoever last edited it + # would attribute a nightly run to an editor who was asleep. choice, meta = ai_review.decide(prompt=cfg.get("prompt") or "", options=options, row=row, fields=[f for f in fields if f], - label=cfg.get("label") or "Review") + label=cfg.get("label") or "Review", + st=rt, user=str(defn.get("createdBy") or "")) if not choice: if meta.get("problem"): log(f"[aios-auto] ai review declined to answer: {meta['problem']}") diff --git a/api/main.py b/api/main.py index 7a4d6b0d75cff01cda1ebb67299da4d88a4ae9f4..baadba701fda925c52c9ddc3cc1158e6357286b3 100644 --- a/api/main.py +++ b/api/main.py @@ -82,6 +82,9 @@ import routes_query # noqa: E402 (wave 32 R1/C5 — the Query module; E's rout import routes_publish # noqa: E402 (wave 33 R5/C2 — the publish door; C's router, A's line) import routes_brand # noqa: E402 (wave 33 R4/C2/C6 — connector brand marks; G's router, A's line) import routes_slack # noqa: E402 (wave 33 R4/C2 — Manage agent + the Slack door; D's router, A's line) +import routes_starred # noqa: E402 (wave 35 R4/C2/C3/C5 — the star; E's router, E's line) +import routes_usage # noqa: E402 (wave 35 R9/C7 — the AI usage meter; E's router, E's line) +import routes_feedback # noqa: E402 (wave 35 R8/C6 — feedback to the operator plane; E's router) from core import grid_events # noqa: E402 from deps import Session, module_gate # noqa: E402 @@ -310,6 +313,22 @@ app.include_router(routes_brand.router) # R4 / C2 / C6 — G's router, A's # file to register them in: "public" here IS the absence of `Depends(require_session)`, which is why # `verify_api` asserts the absence rather than an entry in a list that does not exist. app.include_router(routes_slack.router) # R4 / C2 — D's router, A's line (Manage agent + Slack) +# ⭐⭐ WAVE 35 (R4/R5/R7, contracts C2/C3/C5) — THE STAR. Mounted in the SAME change that created +# `routes_starred.py`, which is the seventh consecutive wave in which this block is the artefact the +# protocol nearly loses — and the first in which the router's own lane also owns `main.py`, so there +# is no ask/mount pair to drop. `W35-T46` asserts every one of this wave's paths in +# `app.openapi()["paths"]` and CALLS one route from each. +# ⚠ Same placement rule as every line above: ABOVE `app.mount("/", _AppStatic(...), html=True)`, or +# the router answers 404 on GET and 405 on POST while every one of its own tests passes. +app.include_router(routes_starred.router) # R4 / C2 — the star, counts, and record stars +# ⛔ R9 SAYS THIS ROUTE REPORTS AND NEVER ENFORCES, so mounting it cannot cut anybody off — there is +# no ceiling anywhere behind it (`usage_ledger` has no refusal in it). The one AI limit the product +# enforces is per COLUMN and is unchanged (`ai_enrich.ceiling_report`). +app.include_router(routes_usage.router) # R9 / C7 — GET /usage, the one AI meter +# ⛔ ITS TWO DOORS CARRY DIFFERENT WALLS ON PURPOSE (R8, and D-221 is the booked precedent): the +# POST is any authenticated session's own act, the GET is `is_platform_admin` only. Mounting it does +# not widen anything a tenant admin can reach — `verify_api` proves that by having one try. +app.include_router(routes_feedback.router) # R8 / C6 — feedback to the operator plane # --- DEPRECATED ALIASES (removed when S2's shell flips; kept so the current bundle keeps working) @@ -472,6 +491,12 @@ def _seed_and_sync_store(): # is D-107's own hypothesis 3: a partial population trips `MAX_SHRINK` and the rebuild # refuses — correctly, but for a reason that reads like a data loss scare. # ⚠ Same thread on purpose: it is already a daemon and nothing serves requests behind it. + # ⭐⭐ W35-T45 / R11 — TENANT #0'S ODOO CREDENTIAL MOVES ONTO ITS KEYCHAIN, IN THE CONTAINER. + # ⚠ BEFORE the relational rebuild, deliberately: the rebuild resolves its connector through + # `rt.odoo_source()`, so running the migration first means the very next read already goes + # through the keychain branch and a broken migration is visible in THIS boot's log rather + # than in tomorrow's. It is idempotent, so every later boot is one `list_entries` read. + _migrate_env_odoo("boot") _pull_meta("boot") _rebuild_odoo_relational("boot") # ⭐⭐ W32-T07 — owner items 13 and 15, DELIVERED. See `_sweep_automation_schemas`. @@ -483,6 +508,35 @@ def _seed_and_sync_store(): print(f"[aios-api] datastore seed/sync skipped: {e}") +def _migrate_env_odoo(why): + """⭐⭐ W35-T45 / R11 — move tenant #0's environment Odoo credential onto its keychain. + + ⛔⛔ IN THE CONTAINER, WHICH IS THE WHOLE REASON THIS IS A BOOT LINE AND NOT A SCRIPT. D-195, + measured three times: a developer's CLI write to the tenant store is reverted by the running + Space within a minute (download-modify-upload, last-write-wins) — and the write REPORTS SUCCESS + every time, then a fresh read confirms it, and it is gone by the next poll. A CLI migration would + be a dry run that lies, and what it would lie about here is a credential. + + ⚠ IDEMPOTENT AND SCOPED TO TENANT #0 by `routes_keychain.env_odoo_available`, so on every other + tenant and on every later boot this is one `list_entries` read and a line. + ⚠ FAIL-QUIET: this must never take a boot down. But it is never SILENT — a skip prints its reason, + because "already migrated", "no keychain key on this deployment" and "the env is incomplete" are + three different operator actions and a blank Keychain page cannot tell them apart. + """ + try: + import routes_keychain as _kc_routes + from harness import runtime as _runtime + rep = _kc_routes.migrate_env_odoo(_runtime.get_runtime("royal-imports")) + if rep.get("done"): + print(f"[aios-api] odoo credential migrated onto the keychain ({why}): " + f"entry={rep['entry']} carried_pause={rep['carried_pause']}" + + (f" PROBLEM: {rep['why']}" if rep.get("why") else "")) + else: + print(f"[aios-api] odoo keychain migration skipped ({why}): {rep.get('why')}") + except Exception as e: # noqa: BLE001 + print(f"[aios-api] odoo keychain migration FAILED ({why}): {type(e).__name__}: {e}") + + def _pull_meta(why): """Pull Meta Ads into THIS container's mirror, before the relational rebuild reads it. diff --git a/api/routes_automation.py b/api/routes_automation.py index 50c1df8d22f4db1ebec63ffa0f123ca4db35ff29..0b8aad0a997592d356b825513ee8877b046e066b 100644 --- a/api/routes_automation.py +++ b/api/routes_automation.py @@ -91,6 +91,27 @@ def _wire(defn, tenant): # C3 (wave 22): the trigger config rides whole — the webhook token included, because # the person configuring the external caller has to be shown the URL somewhere, and # this payload is session-gated behind the same wall as everything else here. + # ⭐⭐ WAVE 35 · T36 / R10 — THE PARKED BATCH, SUMMARISED. Without this line the whole + # ticket would be unreachable: `_wire` is a WHITELIST, so a key the engine parks on the + # definition simply never arrives, and the detail pane could not know a batch was waiting + # ([[reachable-is-not-the-same-as-built]] — the exact shape `flow` was caught in above). + # ⚠ COUNT AND NOTES ONLY, NEVER `items`. A 200-statement batch carries 200 rendered HTML + # mails; this payload rides on the automations LIST, which every rail paint reads. + # `GET /admin/statements/agent/{id}` serves the detail when somebody opens the batch. + "pendingStatements": ({ + "count": int((defn.get("pendingStatements") or {}).get("count") or 0), + "ts": (defn.get("pendingStatements") or {}).get("ts") or "", + "notes": list((defn.get("pendingStatements") or {}).get("notes") or [])[:25], + } if isinstance(defn.get("pendingStatements"), dict) else None), + # ⭐⭐ WAVE 35 · T37 — `system` FOR A **STORED** ROW. Until now only SYNTHETIC rows carried + # it, stamped by their builders AFTER this function (`_odoo_sync_row`, `_field_agent_rows`), + # so a stored row with the marker had it enforced on the server (`delete_automation` 409s) + # and INVISIBLE to the client — which paints a live-looking Delete that answers with a + # refusal. `AutomationDetail` already reads `automation.system` to disable that button and + # explain why; this is the line that lets it. + # ⚠ Absent stays absent rather than becoming `""`: every ordinary automation is not-a-system + # -agent, and an empty string is a value a client could accidentally treat as one. + **({"system": str(defn.get("system") or "")} if defn.get("system") else {}), "trigger": defn.get("trigger") or None, "statusNote": defn.get("statusNote") or "", # The one-sentence summary (airtable-brief rec 7), composed from the definition so it @@ -550,6 +571,13 @@ def _tick_state(): #: or delete — a new database cannot contain the legacy stage cells this retires. _BOARD_RETIRED = set() +#: ⭐ WAVE 35 · T37 — tenants whose statements agent has been considered THIS PROCESS. Same shape as +#: `_BOARD_RETIRED` and, like it, purely a cost saver: the DURABLE idempotency is the row's own +#: existence, which `engine.seed_statements_agent` re-checks inside its own store mutation. +#: ⚠ SO A RESTART RE-CHECKING IS HARMLESS BY CONSTRUCTION, which is the property to preserve. If +#: this set were ever the only guard, a process restart would mint a second agent. +_STATEMENTS_SEEDED = set() + #: ⭐⭐ WAVE 30 · T12 — THE PICKER DEFAULT, MEMOISED. `{tenant: (stamp, table_key)}`, the shape #: `scope_cache` stores. #: @@ -702,6 +730,20 @@ def list_automations(session: Session = Depends(_GATE)): # Idempotent Board retirement removes only engine-marked stage fields and legacy Board # state. User-created Status/Stage columns remain intact. engine.retire_automation_board_state(session.runtime, tables=tables) + # ⭐⭐ WAVE 35 · T37 / OWNER RULING R10 — ROYAL IMPORTS' STATEMENTS AGENT EXISTS BEFORE ANYBODY + # ASKS FOR IT. It is minted ONCE, by the container, SWITCHED OFF; every subsequent call finds + # it present and returns immediately. See `engine.seed_statements_agent` for why this one is + # STORED where wave 34's Odoo agent is derived, and why the row's existence is a sufficient + # idempotency key (it cannot be deleted). + # ⚠ GUARDED SO IT CAN NEVER TAKE THE RAIL DOWN. A tenant with no Odoo, a store mid-outage or a + # validator change must degrade to "no statements agent", never to a 500 on the one route the + # whole Agents surface polls. The same posture `_odoo_sync_row` takes for the same reason. + if session.tenant not in _STATEMENTS_SEEDED: + try: + engine.seed_statements_agent(session.runtime, username=session.uname or "system") + except Exception: # noqa: BLE001 + pass + _STATEMENTS_SEEDED.add(session.tenant) defs = engine.all_definitions(session.runtime) items = [_wire(d, session.tenant) for _, d in sorted(defs.items(), key=lambda kv: (kv[1].get("name") or "").lower())] @@ -801,7 +843,11 @@ def list_automations(session: Session = Depends(_GATE)): # own reason instead of omitting them — the owner asked for Airtable's full menu, and # a shorter list would imply those actions do not exist. `clean_actions` REFUSES an # unready kind, so the faded state is a wall rather than a styling choice. - "actionsCatalog": engine.action_catalog(), + # ⭐ W35-T35 (C8): `session.runtime` is what withholds a TENANT-GATED row. Passing it + # here is the whole of "the menu does not offer Send statements to another tenant"; + # the STORE-side wall is `clean_actions(rt=)`, deliberately separate, because a picker + # is not a security boundary [[opening-a-route-widens-every-field]]. + "actionsCatalog": engine.action_catalog(session.runtime), # The builder's own vocabulary: how deep a condition tree may nest, how deep groups # may nest, and the ceilings. B reads these instead of hard-coding the same numbers # into its "+ Add condition" affordance. @@ -1109,7 +1155,10 @@ def draft_automation(body: dict = Body(default=None), session: Session = Depends # filtering one and not the other describes settings for kinds the model cannot choose. That is # not merely untidy: it spends prompt on unreachable options and invites the model to reach for # one. `_ai_agent_plan` already narrows both in exactly this way; this follows it. - menu_catalog = [r for r in engine.action_catalog() if r.get("menu") is not False] + # W35-T35 (C8): tenant-gated rows are withheld here too, or the DRAFTER would offer a kind + # the same tenant's own save door then refuses — a flow the assistant proposes and the product + # will not accept. + menu_catalog = [r for r in engine.action_catalog(session.runtime) if r.get("menu") is not False] _offered = {r["kind"] for r in menu_catalog} draft, refusal, provider = ai_review.draft_flow( prompt=prompt, @@ -1117,14 +1166,21 @@ def draft_automation(body: dict = Body(default=None), session: Session = Depends required={k: v for k, v in engine.ACTION_REQUIRED.items() if k in _offered}, triggers=_triggers_vocab(session), tables=tables, + # ⭐ W35 · CONTRACT C7 (`NOTE E-16`) — the spend is ATTRIBUTED. Unlike the two engine call + # sites, this one has a real person behind it: somebody typed the sentence, so `user` is + # the caller rather than the automation's owner. + st=session.runtime, user=getattr(session, "uname", "") or "", chat=_DRAFT_CHAT[0]) if refusal or not draft: raise err(400, "draft_refused", refusal or "no automation could be drafted from that") # ── C7: run the real save-door validator and DIFF it ────────────────────────────────── wrote = draft.get("actions") or [] + # W35-T35 (C8): the SAME `rt` the menu was built with. Without it this validator would refuse + # a tenant-gated kind it had just offered the model — the draft door disagreeing with the save + # door about one tenant, which reads as "the assistant produced an invalid flow". cleaned, why = engine.clean_actions([{"kind": a.get("kind"), "config": a.get("config") or {}} - for a in wrote]) + for a in wrote], rt=session.runtime) if why or cleaned is None: raise err(400, "draft_invalid", f"the assistant produced a flow this deployment will not accept: {why}") diff --git a/api/routes_feedback.py b/api/routes_feedback.py new file mode 100644 index 0000000000000000000000000000000000000000..edf822ad1bd0ca3268a7e1bbb756be76d3c18bde --- /dev/null +++ b/api/routes_feedback.py @@ -0,0 +1,168 @@ +"""routes_feedback.py — WAVE 35 (ruling R8, contract C6): FEEDBACK GOES TO THE OPERATOR PLANE. + +R8: *"Feedback goes to the platform-wide `loopable` operator plane. No tenant sees another's, and +the submitting tenant cannot edit or delete what it sent."* + +⛔⛔ TWO DOORS, TWO DIFFERENT WALLS, AND THAT ASYMMETRY IS THE WHOLE TICKET. D-221 is already on the +register for a cross-tenant WRITE gated on `admin_gate` alone, so getting this backwards is a repeat +rather than a novelty: + + POST /feedback ANY authenticated session, any tenant. It is the TENANT'S OWN ACT. + GET /feedback `core.platform_admin.is_platform_admin` ONLY. It is the OPERATOR'S read. + +A single wall cannot express that. Gating the write on the operator predicate would mean nobody can +send feedback; gating the read on `admin_gate` would hand every tenant admin every other tenant's +words. So the walls are declared per route, and `verify_api` drives BOTH from a tenant admin, a +plain user and a platform admin rather than asserting the source. + +⛔ THE PROVENANCE IS STAMPED SERVER-SIDE AND NEVER READ FROM THE BODY. `tenant`, `user` and `ts` +come from the verified session and the clock. A body-supplied tenant would let any account file +words against another company's name, in a store only the operator reads and therefore only the +operator could ever be misled by. + +⛔ NO EDIT AND NO DELETE EXIST AT ALL — not for the tenant (R8 says so) and not for the operator +either. R8 grants the operator a READ; a delete door is a capability nobody asked for on a store +whose whole value is that it is append-only. There is nothing here to forget to wall. + +⚠ ONE STORE, AND WHERE IT PHYSICALLY LIVES IS STATED RATHER THAN IMPLIED. This uses `core.store` +with a raw key, which is the same thing `routes_platform_admin` does for `runtime.TENANTS_KEY` and +for the same reason: a platform fact belongs in the default store, not inside any tenant's +namespace. ⚠ That default is currently tenant #0's dataset repo (`OS_DATA_REPO`), so "the platform +store" and "tenant #0's repo" are the same bucket today. That is pre-existing (the control plane's +tenant records are already there) and it is a hosting fact, not a permission one — no tenant route +can read this key, because none of them names it. +""" +import time + +from fastapi import APIRouter, Body, Depends + +import core.platform_admin as platform_admin +import core.store as store +from deps import Session, err, require_session + +router = APIRouter(prefix="/api/v1") + +#: The platform-wide bucket. `{"rows": [ {id, tenant, user, category, text, ts} ]}` — newest LAST +#: in the store, newest FIRST on the wire. +STORE_KEY = "platform_feedback" + +#: ⭐ THE CATEGORY VOCABULARY IS SERVER-OWNED AND SERVED (B's `ASK B-1 (3)`, agreed). A client +#: constant beside a server enum drifts, and the drift shows up as a dropdown offering a value the +#: door refuses. B renders exactly this list; the door accepts exactly these keys. +CATEGORIES = ( + {"key": "bug", "label": "Something is broken"}, + {"key": "idea", "label": "An idea or a request"}, + {"key": "data", "label": "The numbers look wrong"}, + {"key": "speed", "label": "Something is slow"}, + {"key": "other", "label": "Something else"}, +) + +MAX_CHARS = 4000 + +#: A ceiling on the whole store. ⛔ REPORTED AT BOTH ENDS, never a silent drop: the submitter is +#: refused with the cause, and the operator's own payload says the store is full so they can act. +#: Dropping the OLDEST rows instead would delete the operator's earliest feedback to make room for +#: the newest, which is the one direction this store must never move in. +MAX_ROWS = 5000 + + +def _rows(): + try: + raw = store.get(STORE_KEY) or {} + except Exception: + return [] + rows = raw.get("rows") if isinstance(raw, dict) else None + return [r for r in rows if isinstance(r, dict)] if isinstance(rows, list) else [] + + +@router.get("/feedback/form") +def feedback_form(session: Session = Depends(require_session)): + """What the composer renders. Any session — every account may send feedback. + + ⚠ It deliberately returns NO submissions. R8 gives the tenant no read of what it sent, so a + "your feedback" list would be a second door to a thing the ruling closed. The 201 is the receipt. + """ + return {"categories": [dict(c) for c in CATEGORIES], "maxChars": MAX_CHARS} + + +@router.post("/feedback", status_code=201) +def submit_feedback(body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Send one piece of feedback to the operator plane. `{"category": key, "text": str}`. + + ⚠ ANY authenticated session, deliberately — see the header. The provenance is stamped from the + VERIFIED session, never from the body. + """ + body = body if isinstance(body, dict) else {} + category = str(body.get("category") or "").strip() + text = str(body.get("text") or "").strip() + if category not in {c["key"] for c in CATEGORIES}: + # Named, and it names the fix: a client whose dropdown has drifted from this list is a + # client whose user is about to lose what they typed. + raise err(400, "unknown_category", + "that is not one of the feedback categories. Reload the page and try again") + if not text: + raise err(400, "bad_request", "type what you want to tell us") + if len(text) > MAX_CHARS: + raise err(400, "too_long", + f"feedback is limited to {MAX_CHARS:,} characters. Yours is {len(text):,}") + if not store.available(): + raise err(503, "store_unavailable", + "feedback could not be sent right now. Nothing was saved.") + if len(_rows()) >= MAX_ROWS: + raise err(503, "feedback_full", + "the feedback store is full and we have been told. Nothing was saved, so please " + "send this again later") + row = { + # ⛔ EVERY ONE OF THESE FOUR IS SERVER-SIDE. A body-supplied tenant or user would let any + # account file words against somebody else's name in a store only the operator reads. + "tenant": str(session.tenant), + "user": str(session.uname), + "ts": int(time.time()), + "category": category, + "text": text, + } + row["id"] = f"fb_{row['ts']}_{abs(hash((row['tenant'], row['user'], text))) % 10**8:08d}" + + def _up(data): + data = data if isinstance(data, dict) else {} + rows = data.get("rows") + data["rows"] = (rows if isinstance(rows, list) else []) + [row] + return data + + try: + # `flush='sync'`: a person pressed Send and is being told it landed. The coalescing mode is + # for autosaves nobody is waiting on. + store.update(STORE_KEY, _up) + except Exception: + raise err(503, "store_unavailable", + "feedback could not be sent right now. Nothing was saved.") + return {"ok": True, "id": row["id"]} + + +@router.get("/feedback") +def list_feedback(tenant: str = "", session: Session = Depends(require_session)): + """R8's operator read: every tenant's feedback, newest first. PLATFORM ADMIN ONLY. + + ⛔ THE WALL IS `core.platform_admin.is_platform_admin`, IMPORTED AND CALLED — the double lock + (the record flag AND the `loopable` tenant), never re-expressed here. A tenant admin is not + admitted: `role: 'admin'` is one company's authority over its own workspace, and this is every + company's words in one list. + ⚠ `?tenant=` NARROWS an already-admitted operator's view. It is a convenience, not a wall, and + it is applied after the gate so it can never be the thing that admits anybody. + """ + if not platform_admin.is_platform_admin(session.user): + raise err(403, "forbidden", "this is a Loopable operator surface") + rows = _rows() + if tenant: + want = str(tenant).strip().lower() + rows = [r for r in rows if str(r.get("tenant") or "").lower() == want] + rows = sorted(rows, key=lambda r: int(r.get("ts") or 0), reverse=True) + out = {"rows": [dict(r) for r in rows], "total": len(rows), "capacity": MAX_ROWS, + "tenants": sorted({str(r.get("tenant") or "") for r in _rows() if r.get("tenant")})} + if len(_rows()) >= MAX_ROWS: + # R6's second sentence, aimed at the person who can act: submissions are being REFUSED right + # now, and the operator is the only one who can see that from here. + out["note"] = ("the feedback store is at capacity, so new submissions are being refused. " + "Archive what you have read") + return out diff --git a/api/routes_keychain.py b/api/routes_keychain.py index a5df0eb8e6dd266b6cf191c4662285aaf453fcd5..0d8534c02e4acf8e5318379694ff5a39602887f5 100644 --- a/api/routes_keychain.py +++ b/api/routes_keychain.py @@ -296,6 +296,114 @@ def _rel_reconnect(rt): return True +# ══════════════════════════════════════════════════ W35-T45 / R11: THE ENV -> KEYCHAIN MIGRATION +# +# R11: *"Tenant #0's Odoo credential MIGRATES onto the keychain, with the environment kept as +# fallback."* The owner chose this over a read-only display row WITH THE MIGRATION RISK STATED, so +# the risk is what this block is mostly about. +# +# ⭐ WHAT WAS ALREADY TRUE, CHECKED BEFORE ANY OF IT WAS WRITTEN: the keychain-first-then-env +# RESOLVER has existed since the 2026-08-04 cutover. `harness/runtime.py::odoo_source` is already +# (1) a keychain `odoo` entry, (2) tenant #0's compiled env connector, (3) None for anybody else, +# and `visible_entries` already serves a non-admin the row WITHOUT a preview. So R11 is not "build +# a resolver" — it is "give tenant #0 the ROW", which is the only reason its Keychain page looks +# empty while its Odoo grids work. +# +# ⛔⛔ AND THE ONE REAL HAZARD IS NOT THE CREDENTIAL, IT IS THE PAUSE FLAG. `odoo_flag_key()` returns +# the first keychain `odoo` entry id when one exists and `ENV_ODOO_FLAG_KEY` ("odoo-env") otherwise — +# so CREATING THE ENTRY MOVES THE ADDRESS THE PAUSE FLAG LIVES AT. A tenant #0 that was paused under +# `odoo-env` would come back UNPAUSED, silently, at the first boot after this ships: the connector +# resumes pulling live Odoo because a migration changed which key the freeze was stored under. That +# is D-259's exact shape (a pause written under one key and read under another) and +# [[a-guard-bound-to-a-role-stops-guarding-when-the-role-moves]]. The flag is carried across in the +# SAME pass, and the carry is asserted. + + +def _env_odoo_fields(): + """The four env values `odoo_client` reads, or `(None, why)` when they are not all present. + + ⛔ ALL FOUR OR NOTHING, and this completeness check is load-bearing rather than defensive. + Creating the entry makes `odoo_source` resolve through branch 1 INSTEAD of the env — so a + PARTIAL migration would hand the connector `{url, db}` and no key and take tenant #0's Odoo + offline, on a deployment where it had been working. The env fallback cannot save it, because the + entry's existence is what turns the fallback off. + ⚠ The names are `odoo_client.py`'s own (`ODOO_URL`/`ODOO_DB`/`ODOO_USER`/`ODOO_API_KEY`) and the + field names are `harness/connectors/odoo.py`'s stored shape (`{url, db, user, api_key}`). Two + vocabularies meet here; nowhere else. + """ + want = (("url", "ODOO_URL"), ("db", "ODOO_DB"), + ("user", "ODOO_USER"), ("api_key", "ODOO_API_KEY")) + got = {field: (os.environ.get(env) or "").strip() for field, env in want} + missing = sorted(env for field, env in want if not got[field]) + if missing: + return None, (f"the environment is missing {', '.join(missing)}, and a partial credential " + f"would take this tenant's Odoo offline rather than migrate it") + return got, "" + + +def migrate_env_odoo(rt): + """R11 — put tenant #0's environment Odoo credential on its keychain, once. Returns a report. + + `{"done": bool, "entry": id|"", "carried_pause": bool, "why": str}` — `why` is filled on every + path including the skips, because "already migrated", "no keychain key on this deployment" and + "the env is incomplete" are three different operator actions. + + ⛔⛔ IT MUST RUN IN THE CONTAINER, WHICH IS WHY `main.py` CALLS IT AND NO SCRIPT DOES. D-195, + measured three times: a developer's CLI write to the tenant store is reverted by the running + Space within a minute (download-modify-upload, last-write-wins) — and **the write reports success + every time**, then a fresh read confirms it, and it is gone by the next poll. A CLI migration here + would be a dry run that lies, and the thing it would lie about is a credential. + + ⚠ FAIL-QUIET AND IDEMPOTENT. It runs on EVERY boot; the second one must be a no-op and a + third-party failure must not take the boot down. + """ + out = {"done": False, "entry": "", "carried_pause": False, "why": ""} + if not env_odoo_available(rt): + # Not tenant #0, or this deployment has no env Odoo at all. Both are normal states. + out["why"] = "this tenant has no environment Odoo credential to migrate" + return out + fields, why = _env_odoo_fields() + if not fields: + out["why"] = why + return out + # ⛔ READ THE PAUSE FLAG BEFORE THE WRITE. After the entry exists, `odoo_flag_key()` answers the + # NEW key and the old one is unreachable through the resolver — so the only moment this fact can + # be observed is now. [[undo-capture-before-the-write]] applied to a guard rather than to data. + try: + was_paused = bool(((rt.get(_CONNECTOR_FLAGS_KEY) or {}) + .get(_ENV_ODOO_FLAG_KEY) or {}).get("paused")) + except Exception: # noqa: BLE001 + was_paused = False + row, why = _kc().ensure_entry_of_type( + rt, "odoo", "Odoo (migrated from this deployment)", fields, "system") + if row is None: + out["why"] = why + return out + out["done"], out["entry"] = True, row["id"] + if was_paused: + # The freeze followed the credential. Without this the connector silently RESUMES pulling + # live Odoo at the first boot after the migration. + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + entry = dict(cur.get(row["id"]) or {}) + entry["paused"] = True + entry["pausedBy"] = "system" + entry["pausedNote"] = ("carried over from the environment source when the credential " + "was migrated onto the keychain") + cur[row["id"]] = entry + return cur + + try: + rt.update(_CONNECTOR_FLAGS_KEY, _up, flush="sync") + out["carried_pause"] = True + except Exception as exc: # noqa: BLE001 + # ⛔ SAID OUT LOUD. A migration that moved the credential and lost the freeze is worse + # than one that did not run, so this is the one failure that must never be silent. + out["why"] = (f"the credential migrated but the PAUSE could not be carried over " + f"({type(exc).__name__}), so this tenant's Odoo is no longer frozen") + return out + + def _own_row(session, entry_id): """The visible row for `entry_id`, or a 404. ⛔ A 404 rather than a 403 for an entry the caller cannot see: telling a member that somebody else's personal credential EXISTS is the diff --git a/api/routes_nav.py b/api/routes_nav.py index b376a5f2e3f95e934da77c1d8589a9dd02ae3f9c..8ac00d2f5aa6453e9d49d448a09672bd0bde8a23 100644 --- a/api/routes_nav.py +++ b/api/routes_nav.py @@ -1,915 +1,890 @@ -"""routes_nav.py — X2's `GET /api/v1/nav`: the registry, filtered by what this session may open. - -SERVER-FILTERED, not client-filtered. The client renders what it is given and never decides who -may see what — a nav that hides a link the API would still serve is a UI courtesy, not a -permission. `core.perms.nav_pages` is the single predicate (shared with `may_open`'s page gate), -so the nav and the 403 can never disagree about a grant. - -The full rule set — archived is invisible to everyone, `group_only` rows are excluded so no -`parent` reference can dangle, `nav: False` surfaces ship as `chrome: 'utility'`, and the -`prefs.json` Library preference is deliberately NOT applied — is documented on -`core.perms.nav_pages`, which is where it belongs: one place, both callers. -""" -import time - -from fastapi import APIRouter, Body, Depends - -from deps import Session, err, perms, require_session - -router = APIRouter(prefix="/api/v1") - -#: Wave 2026-08-02 (C-SCHEMA): per-user folders over the database list, the view-folder -#: pattern applied to the nav. Cosmetic per-user state — placement never grants or hides a -#: surface (the server-filtered nav still decides what exists). -_NAV_PREFS_KEY = "nav_prefs" -_MAX_NAV_FOLDERS = 16 - -#: WAVE 19 (R8 / contract C1): the database's NAME and ICON overrides. -#: -#: ⚠ TENANT-WIDE, which is the whole difference from `nav_prefs` above and the reason it is a -#: separate bucket rather than another field in that one. Folders and placement are one -#: person's arrangement of their own rail — per-user by definition. What a database is CALLED -#: and what it looks like are facts about the database: a workspace where two people call the -#: same table different things has no shared vocabulary left to discuss it in. Same store, two -#: buckets, because they answer to two different owners. -#: -#: Shape: {"": {"icon": {"shape": , "tone": }, -#: "name": ""}} -_NAV_META_KEY = "nav_meta" - -#: The grid's folder-icon vocabulary, mirrored — 12 shapes x 5 tones. The CLIENT imports these -#: from `customer-grid/types` rather than redefining them; this end cannot import TypeScript, -#: so it is the one place the list is written twice. -#: -#: ⛔ WHAT AN UNKNOWN VALUE MUST NOT DO IS BE STORED. `FolderMark` indexes its path table by -#: shape and maps the result, so a shape this whitelist let through and the renderer does not -#: know is `undefined.map()` — a blank rail, from a stored preference, for every user in the -#: tenant until somebody edits the store by hand. Refusing at the door is the cheap end of -#: that. If the two lists ever drift the symptom is an icon that silently reverts to the -#: default, which is the loudest SAFE failure available here. -_ICON_SHAPES = frozenset({"folder", "star", "flag", "tag", "bookmark", "grid", - "chart", "map", "users", "clock", "heart", "bolt"}) -_ICON_TONES = frozenset({"neutral", "blue", "green", "yellow", "red"}) -_MAX_NAV_NAME = 60 - -#: WAVE 23 (contract C10 / ruling R7) — the Home landing's RECENTS. -#: -#: PER-USER, like `nav_prefs` two buckets up and unlike `nav_meta`: what I opened last is -#: nobody else's business, and a tenant-wide "recently opened" would be a surveillance feature -#: rather than a convenience one. -#: -#: ⛔ A MAP KEYED BY PAGE, NOT AN APPEND LOG, and the difference is the whole feature. -#: `{username: {pageKey: }}` — re-opening a database OVERWRITES its stamp. An -#: append-only list capped at 50 fills with fifty copies of the same ten databases inside one -#: working session, and the cap then evicts OLDEST-FIRST: the tenth database you touched falls -#: off the list while forty slots hold repeat visits to the first. Keying by page makes -#: "recent" mean what the word means, and makes the cap bound the number of DATABASES -#: remembered rather than the number of clicks. -_NAV_RECENTS_KEY = "nav_recents" -_MAX_RECENTS = 50 - -#: ⚠ EPOCH SECONDS (UTC by definition), never a formatted stamp — and this is a correction of a -#: precedent, not a preference. `user_tables.create` writes -#: `datetime.now().strftime('%Y-%m-%dT%H:%M:%S')`: naive LOCAL time, no offset. A browser parses -#: that string as its OWN local time, so on a UTC host read by a non-UTC reader "opened 30 -#: minutes ago" renders as "opened 7 hours ago" and the Today / Past-7-days buckets misfile — -#: with nothing to go red, because both ends are internally consistent. An integer instant has -#: no such reading. (D-18 made the same correction for notification stamps AFTER the defect -#: shipped; this is that lesson applied before it.) -def _now() -> int: - return int(time.time()) - - -def _clean_nav_prefs(raw, page_keys, *, keep_unknown_ut=False): - """Validated wholesale replacement, the `clean_folders` posture: prune, never invent. - - `page_keys` is the set of TOP-LEVEL keys this session may see; a placement of an unknown - or invisible key is dropped (it can return when the grant does — placement is cosmetic, - so pruning is loss-free). Unknown folder refs drop the placement, not the folder. - - `keep_unknown_ut` is the STORE-BLIP escape hatch — see `_placeable_top_keys`. When the - `ut_*` listing could not be read, a `ut_` key absent from `page_keys` is KEPT rather than - pruned: keeping a placement whose database may since have been deleted is cosmetically - harmless, while pruning a live one is the silent data loss this function just stopped - causing. Never widened to non-`ut_` keys — those come from the compiled registry, which - cannot fail to enumerate. - """ - raw = raw if isinstance(raw, dict) else {} - folders, seen = [], set() - for f in (raw.get("folders") or [])[:_MAX_NAV_FOLDERS]: - if not isinstance(f, dict): - continue - fid = str(f.get("id") or "").strip()[:40] - name = " ".join(str(f.get("name") or "").split())[:40] - if not fid or fid in seen or not name: - continue - seen.add(fid) - folders.append({"id": fid, "name": name}) - ids = {f["id"] for f in folders} - placement = {} - src = raw.get("placement") - if isinstance(src, dict): - for k, v in src.items(): - k, v = str(k)[:60], str(v)[:40] - if v not in ids: - continue # the folder is gone; the placement goes with it - if page_keys is None or k in page_keys: - placement[k] = v - elif keep_unknown_ut and k.startswith("ut_"): - placement[k] = v - return {"folders": folders, "placement": placement} - - -def _placeable_top_keys(session): - """`(keys, enumerated)` — the keys a placement may name, INCLUDING this tenant's databases. - - ⛔ THIS IS THE ITEM-6 FIX (wave 27, contract C1), and the bug it closes was invisible by - construction. The old version read `perms.nav_pages` alone — the compiled REGISTRY — and - `perms.py` contains no `ut_` or `user_tables` reference at all, because a tenant's - user-created databases are merged into the nav payload by `nav()` BELOW the permission wall. - So every `ut_*` key was absent from this set, and `_clean_nav_prefs` pruned every placement - naming one — on READ and on WRITE. - - The symptom was "a database I drag into a folder falls out of it later", never "it does not - work": the write answered **200 OK** having stored `{}`, the client kept its optimistic copy - (`Shell.commitPrefs` reverts only on `!ok`), and the folder held for the rest of the session. - The next reload served the pruned copy and the database was back at root. Built-in modules - (`customer_data`, `product_data`) ARE in the registry and persisted fine, which is exactly - why it read as intermittent rather than as a missing feature. - - ⚠ THE SAME MISTAKE, ALREADY MADE AND ALREADY FIXED ONE FUNCTION AWAY. `nav()`'s recents - block (see its `allowed` comment) hit this in wave 23 and solved it by pruning against the - ASSEMBLED page list. This is that lesson applied to the second consumer, which the wave-23 - fix did not reach. - - `enumerated` is False when the `ut_` listing raised — the caller must then NOT treat this set - as authoritative for `ut_` keys (`keep_unknown_ut`). Pruning a person's whole arrangement - because the store blinked would be the original defect wearing a different cause. - """ - pages = perms.nav_pages(session.user) or [] - keys = {p["key"] for p in pages if not p.get("parent")} - try: - import core.user_tables as user_tables - # The SAME listing `nav()` renders from — `may_open`-filtered, so a placement can never - # name a database this session cannot see, and the nav and this door agree by sharing - # one predicate rather than by two lists being kept in step. - # ⭐ W31-T10 (C1/D-175): `nav_entries` lends its own read to `may_open`, so this door is - # ONE document copy rather than `1 + N`. It is not a footnote here — `Shell.tsx` fires - # `/nav/prefs` CONCURRENTLY with `/nav` on the same `Store._lock`, and it measured - # **8,807 ms live / 17,945 ms in-process** on tenant #0, i.e. roughly half the wait the - # owner reports as "the Automation row arrives ten seconds late". - for e in user_tables.nav_entries(viewer=session.uname, is_admin=session.admin, - st=session.runtime): - keys.add(str(e.get("key") or "")) - except Exception: - return keys, False - return keys, True - - -def _clean_icon(raw): - """A stored/incoming icon, or None. Prune, never invent — `clean_folders`' posture.""" - if not isinstance(raw, dict): - return None - shape, tone = raw.get("shape"), raw.get("tone") - if shape not in _ICON_SHAPES or tone not in _ICON_TONES: - return None - return {"shape": shape, "tone": tone} - - -def _read_nav_meta(runtime): - """The tenant's `nav_meta`, validated on the way OUT as well as in. - - Re-validating a read looks redundant and is not: the bucket outlives this code, an older - build may have written a shape this one no longer accepts, and the whitelist is mirrored - from a vocabulary that lives in another language. A row whose icon does not survive - validation renders the default mark — never a crash, and never a half-drawn glyph. - """ - try: - stored = runtime.get(_NAV_META_KEY) or {} - except Exception: - return {} # a store blip must not take the nav down with it - if not isinstance(stored, dict): - return {} - out = {} - for key, meta in stored.items(): - if not isinstance(meta, dict): - continue - entry = {} - icon = _clean_icon(meta.get("icon")) - if icon: - entry["icon"] = icon - # ⛔ THE `ut_` RULE IS RE-APPLIED ON READ, not trusted from the record. The write - # route refuses a name for a built-in key, but a record written by an older build (or - # by hand) could still carry one — and honouring it would let the store rename - # `customer_data` on screen while `core/registry.py` and every other reader went on - # calling it Customer. A name that could not be written today is not served today. - name = meta.get("name") - if str(key).startswith("ut_") and isinstance(name, str) and name.strip(): - entry["name"] = " ".join(name.split())[:_MAX_NAV_NAME] - if entry: - out[str(key)] = entry - return out - - -def _read_recents(runtime, uname, allowed): - """This user's recents, newest first, PRUNED to what they may currently see. - - ⚠ PRUNED ON READ, never on write, and both halves of that are deliberate: - - · the WRITE happens on every route open — it is the one hot path this file has — so it - must not build the whole nav to validate one key; - · a key whose grant was REVOKED must stop being offered without anyone running a - migration, and must come back if the grant does. That is `_clean_nav_prefs`' own - prune-never-invent posture, applied to a second bucket for the same reason. - - A key that is not in `allowed` therefore reaches the store and never reaches a screen — - which also means a stuffed key cannot be used to discover what exists: it comes back only - if the session could already see it. - """ - try: - stored = (runtime.get(_NAV_RECENTS_KEY) or {}).get(uname) or {} - except Exception: - return [] # a store blip must not take the nav down with it - if not isinstance(stored, dict): - return [] - out = [] - for key, at in stored.items(): - key = str(key) - if allowed is not None and key not in allowed: - continue - try: - at = int(at) - except (TypeError, ValueError): - continue # a stamp this build cannot read is not a stamp - if at <= 0: - continue - out.append({"key": key, "at": at}) - out.sort(key=lambda r: r["at"], reverse=True) - return out[:_MAX_RECENTS] - - -# ── W34-T10 (ruling R1, contract C1): the "mark important" counts, per database ────────────── - -#: A wall-clock ceiling on the WHOLE counting block, checked between databases. -#: -#: ⛔ IT IS A COLD-START BOUND, NOT A PERFORMANCE BUDGET, and the measurement is the reason it -#: exists at all. Every database's views live in its OWN store bucket (`_table_workspace`), -#: never in the `user_tables` document this route already holds — so this block is `N` reads on a -#: route D-175 spent a wave reducing to one. What makes it affordable is that those buckets are -#: TINY. Measured on tenant #0, 2026-08-16: twelve of them cost **6.2 ms WARM in total**, and the -#: largest (`customer_data`) is 45,035 bytes against the 28.6 MB `user_tables` document; ten of the -#: twelve are under 1 KB. COLD is the other half of the truth: the same twelve cost 7,331 ms of -#: first fetch. So this budget bounds the FIRST request after a container starts, and a database it -#: does not reach is reported through `degraded` rather than quietly carrying no number. -#: -#: ⚠ THE NUMBER IS CHOSEN AGAINST THE CLIENT'S DEADLINE, NOT AGAINST A FEELING. `nav.ts`'s -#: `NAV_TIMEOUT_MS` is 20 s and a cold `/nav` already spends most of that on the `user_tables` -#: download; letting this block run unbounded (measured: 7,331 ms for twelve cold buckets) would -#: convert a slow success into a manufactured failure, which is the exact mistake that deadline's -#: own comment warns about. 2.5 s is a bound the route can afford to lose. -_IMPORTANT_BUDGET_S = 2.5 - -#: Suffix `core.view_templates.workspace_key` appends. Stripping it is how a page key becomes the -#: TOPIC key the cohort bucket is named from — `customer_data` -> `customer_table_workspace` -> -#: `customer` -> `customer_cohorts`. Derived rather than re-listed on purpose: `_WS_KEYS` is -#: already the one place `customer_data`/`product_data` are mapped to their topic, and a second -#: copy here would be free to drift from it ([[one-question-two-normalizers]]). -_WS_SUFFIX = "_table_workspace" - - -def _visible_views(doc, uname, is_admin, granted_ids): - """Every view on ONE database that THIS caller can see, from an already-read workspace doc. - - ⛔ PER-CALLER, NOT TENANT-WIDE, and this is the half a server-side count is most likely to get - wrong. `_table_workspace` is `{username: {views, fields, overlays}, '__shared__': {...}}` - — one home per view, never both — so "how many views are marked important here" has a - DIFFERENT answer for every account. Measured on tenant #0: `leadership` has one marked view and - `admin` has none, on the same database. A tenant-wide count would put a number in the owner's - rail that no view sidebar they can open would ever add up to. - - ⚠ `table_store._may_see` is imported rather than re-expressed. It is private, and reaching for - it is still the right call: the alternative is `TableOps.shared_views`, which re-reads the whole - bucket per database (the N whole reads this block exists to avoid), and the only other option is - a second copy of a permission predicate. A wrong copy of `_may_see` widens what a user is told - exists; a private import cannot. - """ - import core.table_store as table_store - out = {} - for vid, view in ((doc.get(uname) or {}).get("views") or {}).items(): - if isinstance(view, dict): - out[str(vid)] = view - for vid, view in ((doc.get(table_store.SHARED_KEY) or {}).get("views") or {}).items(): - if isinstance(view, dict) and table_store._may_see(view, uname, is_admin): - out.setdefault(str(vid), view) - # The wave-21 named-user grants. The ids come from ONE tenant-wide bucket read once for the - # whole request; the RECORD is already in hand, in whichever stratum its owner keeps it. - if granted_ids: - for stratum, blob in doc.items(): - if stratum == uname or not isinstance(blob, dict): - continue - for vid, view in (blob.get("views") or {}).items(): - if str(vid) in granted_ids and isinstance(view, dict): - out.setdefault(str(vid), view) - return out - - -def _view_record_count(cfg, cohorts): - """How many RECORDS this view resolves to, or None when that cannot be answered for free. - - ⛔ `None` IS AN ANSWER AND IT IS THE IMPORTANT ONE. Two of the three shapes below are exact - because the view CARRIES its row set; the third — an ordinary filtered view — can only be - counted by running its filters over the records, and the records are the one thing this route - must never read (`D-175`/`D-185`: `/nav` is a rows-free projection, and reaching for `rows` - here raises by that projection's own contract). So a filtered view is reported as UNCOUNTED and - the database's `partial` flag says so, which is the whole of R6's second sentence applied to a - badge: a limit that cannot be removed is REPORTED with its cause, never papered over with a - number that is short by an unknown amount. - - ⚠ THIS IS ALSO WHY THE SERVER DOES NOT SIMPLY MIRROR THE CLIENT. `CustomerGrid::alertCounts` - counts a filtered view fine and gives up on a SERVER-WINDOWED one (`D-205`'s `Important 0+`); - this end is the exact inverse — it has no rows at all and no window either. The two are honest - about different halves, which is why `partial` had to be on the wire rather than derived. - """ - if not isinstance(cfg, dict): - return None - # A cohort-locked view IS its cohort: the lock and the id are the same fact (`grid_events` - # re-stamps it on every write), and a cohort's membership is a stored pid LIST, not a query. - lock = str(cfg.get("cohortLock") or "").strip() - if lock: - n = cohorts.get(lock) - return int(n) if isinstance(n, int) else None - # A curated row set carries its own count. - pids = cfg.get("memberPids") - if isinstance(pids, list) and pids: - return len(pids) - return None - - -def _important_counts(session, keys): - """`({key: {marked, counted, partial}}, unread)` for the databases in `keys`. - - `marked` = views this caller can see on that database whose `config.important is True`. - `counted` = the SUM of those views' record counts — a record matching two marked views - contributes twice, because that is what the badge the owner is moving has always - meant (`CustomerGrid::importantTotal`: *"a sum of per-view counts, which is what - was asked"*), and a distinct-record total would disagree with the per-view numbers - a user can read off the sidebar and add up themselves. - `partial` = at least one marked view could not be counted. - - `unread` is the set of keys whose bucket did not answer — a store blip or the budget above. - They are reported through `degraded`, never as a confident zero. - """ - import core.shares as shares - import core.view_templates as view_templates - import modules.cohort as cohort_mod - - uname, is_admin = session.uname, session.admin - out, unread = {}, set() - try: - # ⚠ THE ROLE FILTER IS NOT BELT-AND-BRACES. `shared_with` answers "is there an entry naming - # me", and `grid_events._granted_views` — the reader whose answer this badge has to agree - # with — then requires `role_for(...) in ('view','edit')`. Dropping that second test would - # count a view the sidebar does not list, i.e. a badge one higher than anything a person can - # add up. One tenant-wide bucket, read once and cached, so it costs a dict lookup per id. - granted = {str(v) for v in - ((shares.shared_with(uname, kind="view", st=session.runtime) or {}) - .get("view") or []) - if shares.role_for("view", str(v), uname, - st=session.runtime) in ("view", "edit")} - except Exception: - granted = set() # no grants is the fail-closed answer: a narrower count, never wider - cohort_cache = {} - started = time.perf_counter() - for key in keys: - if key in out or key in unread: - continue # the caller's order may repeat a key; a repeat must not respend - ws_key = view_templates.workspace_key(key) - if not ws_key: - continue # not a table at all (a module surface, a folder head) - if time.perf_counter() - started > _IMPORTANT_BUDGET_S: - unread.add(key) - continue - try: - # ⛔ PROJECTED, and the two dropped keys are the whole reason this is affordable. - # `overlays` is per-record field values and `fields` is the schema stratum; neither - # says anything about a view. On `customer_data` they are most of the bucket. - doc = session.runtime.get_projection(ws_key, drop=("overlays", "fields")) or {} - except Exception: - unread.add(key) - continue - marked = [v for v in _visible_views(doc, uname, is_admin, granted).values() - if ((v.get("config") or {}).get("important") is True)] - if not marked: - out[key] = {"marked": 0, "counted": 0, "partial": False} - continue - scope = ws_key[:-len(_WS_SUFFIX)] if ws_key.endswith(_WS_SUFFIX) else key - if scope not in cohort_cache: - try: - # ⚠ Read through `session.runtime`, NOT `modules.cohort`'s own module-level - # helpers: those call `core.store` directly, so they carry no tenant namespace. - # The MODULE is asked for the bucket NAME (it owns that rule) and this route does - # the reading, which is the only tenant-correct combination. - bucket = session.runtime.get(cohort_mod.key_for(scope)) or {} - # ⛔ THIS CALLER'S OWN COHORTS ONLY, and the consequence is deliberate: a SHARED - # view locked to a cohort somebody else owns finds no id here, so it is reported - # UNCOUNTED (`partial`) rather than counted from a stratum this session cannot - # see. Widening the read to every user's cohorts would make the badge disclose the - # SIZE of another person's private list, which is a leak wearing a bug fix. - mine = bucket.get(uname) or {} - cohort_cache[scope] = { - str(cid): len(c.get("members") or []) - for cid, c in mine.items() if isinstance(c, dict) - } - except Exception: - cohort_cache[scope] = {} - counted, partial = 0, False - for view in marked: - n = _view_record_count(view.get("config"), cohort_cache[scope]) - if n is None: - partial = True - else: - counted += n - out[key] = {"marked": len(marked), "counted": counted, "partial": partial} - return out, unread - - -@router.get("/nav") -def nav(session: Session = Depends(require_session)): - """`{pages: [{key,label,source?,chrome}], landing}`. - - Note what this does NOT do: it never returns an empty `pages` list as a way of saying "you - are not allowed". A session that may open nothing at all is a misconfigured account, and it - gets an explicit 403 — an empty 200 is indistinguishable from "the registry is empty" and is - how a permission bug hides in plain sight (X2's never-an-empty-200 rule). - """ - pages = list(perms.nav_pages(session.user) or []) - # Wave 18 C1-TENANT: the REGISTRY is the product's module catalogue — tenant #0's world. - # A tenant record may enable a subset (`modules: [...]`); absent/'all' means everything - # (royal-imports and the compiled builders). A blank tenant enables NOTHING: its nav is - # its own databases, which is what "different set of databases at later waves" means. - tcfg = getattr(session.runtime.tenant, "config", None) or {} - tmods = tcfg.get("modules", "all") - # ⭐⭐ W31-T11 (owner item 6b) — WHAT THE CATALOGUE FILTER REMOVED, SAID OUT LOUD. - # - # Every provisioned tenant carries a restricted list today (`gtmlab`/`loopable`/`nurilab` all - # `['analyst','automation']` — census in `proto/nav-omission-census.md`), so this filter runs - # on every request that is not tenant #0's. It is the INTENDED catalogue, not a defect. But a - # row it removes and a row a store failure dropped are the SAME absence on the wire, and the - # client renders `null` for both — pixel-identical to still-loading. Naming the removals is - # what lets the shell tell "not part of this workspace" from "we could not read it". - _omitted = [] - if tmods != "all": - allowed = {str(k) for k in (tmods or [])} - _omitted = sorted({str(p.get("key")) for p in pages - if p.get("key") not in allowed - and (p.get("parent") or "") not in allowed}) - pages = [p for p in pages - if p.get("key") in allowed or (p.get("parent") or "") in allowed] - # Wave 18 C3-UT: this tenant's user-created databases, merged AFTER the registry rows — - # the host does the same at app.py:7796. Filtered by the per-table wall (`may_open`), so - # the nav cannot offer a row the table routes would refuse. - # ⭐⭐ W31-T10 (contract C1, D-175) — THE DOCUMENT IS READ ONCE FOR THE WHOLE REQUEST. - # - # This route was `2 + N` full deep copies of a 35.8 MB-ceiling document: `nav_entries` took one - # and then `may_open` took another PER TABLE, and the `manage`/`canDelete`/`locked` loop below - # took a SECOND independent one. Median `GET /nav` on tenant #0: **9,548 ms live, 24,741 ms - # in-process** — the second figure is the one that matters, because with the network gone and - # the document resident this route and `/nav/prefs` are the ONLY two that stay slow while every - # other collapses to 20–161 ms. That is per-call CPU, and this is it. - # ⚠ `_ut_defs` is read HERE, above the merge, so the two former reads become one; the locked/ - # manage loop below consumes this same dict. - _ut_defs, _ut_locked_mode, _degraded = {}, "", [] - try: - import core.user_tables as user_tables - # ⭐⭐ W32-T02 (R8/D-175/D-185) — AND THAT ONE READ IS A PROJECTION NOW. The block below - # reads `label`, `source`, `createdBy` and `recordMode`; `may_open` reads `createdBy` and - # the shares registry. Nothing on this route has ever opened a row — and on tenant #0 the - # rows are **99.89% of the document** (28,551,441 bytes; the definitions are 31,220 of - # them). Measured: `all_tables` 1,750 ms -> `all_defs` 1.4 ms, and `GET /nav` 1,921 ms - # median -> see the ticket. THAT is owner item 5: the rail rows did not arrive seconds - # apart because of CSS or ordering, they arrived when their route finished copying. - # ⛔ `all_defs` REFUSES `rows` rather than answering `{}` — if you add something here that - # needs a row, it raises with the reason instead of painting an empty grid. Use - # `all_tables` for that, and know you are buying the whole copy back. - _ut_defs = user_tables.all_defs(st=session.runtime) or {} - # ⛔ NEVER DEFAULT THIS TO `None`. `_ut_defs.get(k)` is `{}` for an unknown key, so its - # `.get("recordMode")` is None too — and `None == None` would mark EVERY database locked - # the moment this import failed. A sentinel that can equal real data is not a sentinel. - _ut_locked_mode = str(user_tables.AUTOMATION_RECORD_MODE) - _lent = user_tables.lend(session.runtime, **{user_tables.STORE_KEY: _ut_defs}) - for e in user_tables.nav_entries(viewer=session.uname, is_admin=session.admin, - st=_lent): - pages.append({"key": e["key"], "label": e["label"], - "source": e.get("source") or "Blank", "chrome": "main"}) - except Exception: - # ⭐⭐ W31-T11 (owner item 6b) — STILL SWALLOWED, NO LONGER SILENT. - # - # A store blip must not take the whole nav down with it, and that half stands. What - # changed is that it used to answer **200 OK with every database missing** and say - # nothing — so a client cannot tell "this tenant has no databases" from "we could not - # read them", and the rail renders the same complete-looking thing either way. That is - # the owner's *"Connectors and Automation module still disappears"* class of report: - # the payload is a claim about what exists, and a claim it could not verify has to be - # marked as such. `degraded` is that mark; the shell renders it in the affected slot. - _degraded.append("databases") - pass - if not pages: - if tmods != "all": - # A provisioned tenant with no modules and no databases YET is a legitimate empty - # state, not a misconfigured account — the client renders "create your first - # database", and X2's never-an-empty-200 rule is honoured by saying WHY it is - # empty rather than leaving 200-[] ambiguous. - return {"pages": [], "landing": None, "empty": "no_databases"} - raise err(403, "no_surfaces", - "your account has no dashboards assigned. Ask an administrator.") - # WAVE 19 (R8 / C1): the tenant's name + icon overrides, merged LAST — after the registry - # rows, after the tenant module filter, after the user tables. Merged HERE rather than - # applied by the client for one reason: `label` is what every reader of this payload shows, - # including the Settings modal's `moduleLabels` map, so a client-side merge would put the - # override in one door and the registry label in the other. - # - # ⛔ THIS MUTATES `pages` IN PLACE, WHICH IS SAFE ONLY BECAUSE `perms.nav_pages` BUILDS A - # FRESH `row = {...}` PER CALL (perms.py:194) and the ut_ rows are built fresh here. If - # either ever starts handing back cached or module-level dicts, this loop would write one - # tenant's chosen label into the object the NEXT tenant's request reads — a cross-tenant - # leak with no symptom until two tenants rename the same registry key. Copy the rows before - # merging on the day that invariant changes. - meta = _read_nav_meta(session.runtime) - # ⭐⭐ W34-T10 (ruling R1, contract C1) — THE MARK-IMPORTANT NUMBERS, COMPUTED ONCE. - # - # ⛔ COMPUTED HERE RATHER THAN INSIDE THE LOOP BELOW, AND THAT PLACEMENT IS THE SAFETY - # ARGUMENT, not tidiness. The `for p in pages:` loop sits OUTSIDE the try/except that wraps - # the `all_defs()` read, so anything raising inside it takes the whole nav down for every - # user — a 500 where the store-blip path is careful to answer 200 with `degraded`. One call, - # one guard, and the loop stays a dict lookup. - # - # ⛔⛔ AND THE ORDER IS LOAD-BEARING, WHICH IS THE ONE THING THE FIRST DRAFT GOT WRONG. - # A budget spent in whatever order the pages happen to arrive is a lottery: measured cold on - # tenant #0, the block spent all 1.5 s of its first draft on ten EMPTY Odoo buckets and was cut - # off two rows before `customer_data`, which holds the only marked view in the tenant. The - # feature would have shipped, been correct, and shown nothing on a cold container. Counting in - # the user's own RECENTS order first fixes that with a fact this route already holds: a mark - # lives on a database somebody works in, and `nav_recents` is exactly the list of those, - # newest first. - _page_keys = {str(p.get("key", "")) for p in pages} - recents = _read_recents(session.runtime, session.uname, _page_keys) - _recent_first = [r["key"] for r in recents] - _seen_first = set(_recent_first) - _recent_first += [k for k in (str(p.get("key", "")) for p in pages) - if k not in _seen_first] - _important, _imp_unread = {}, set() - try: - _important, _imp_unread = _important_counts(session, _recent_first) - except Exception: - _imp_unread = set(_page_keys) - if _imp_unread: - # The SAME honest-absence channel W31-T11 built for the `ut_*` merge. A database whose - # count could not be read must not be indistinguishable from one with nothing marked: - # both would render as no badge, and only one of them is true. - _degraded.append("important") - # Wave 21 (C3): the definitions, once — `manage`/`canDelete` below answer from `createdBy`. - # WAVE 27 (C9): `locked` answers from `recordMode`, off the same one read. - # ⭐ W31-T10: that read is now the SAME one the merge above did — it used to be a second, - # independent `all_tables`, which is why D-175 called this route `2 + N` rather than `1 + N`. - # ⚠ The fail-closed defaults still hold: both are initialised before the try above, so an - # import or store failure leaves `_ut_locked_mode` empty and nothing is marked locked. - for p in pages: - # `manage` (R14): may THIS session change this row's icon/name? Answered HERE because - # the server is the only end that knows — the client cannot see who created a user - # table. Additive and FAIL-CLOSED (absent reads as "no"), so the rail offers a control - # only where the write would actually land, and the route re-checks it regardless. - # - # ⛔ WAVE 21 (C3/W-5): the "presence IS the answer" shortcut DIED in wave 20 — the - # de5037f share-grant admission widened `may_open`, so a row's presence now includes - # databases merely SHARED to this viewer. `manage` (rename/icon) and `canDelete` are - # therefore answered from the DEFINITION: creator-or-admin strictly, matching the - # walls the PATCH and DELETE routes actually enforce. A grantee sees the row and no - # controls — the honest shape (the old code offered rename to users the route 403'd). - key = str(p.get("key", "")) - if key.startswith("ut_"): - _creator = str((_ut_defs.get(key) or {}).get("createdBy") or "") - _mine = bool(session.admin or (_creator and _creator == session.uname)) - p["manage"] = _mine - p["canDelete"] = _mine - # ⭐ WAVE 27 item 3 (contract C9) — A LOCKED DATABASE SAYS SO IN THE RAIL. - # - # "Locked" is the owner's item-4 vocabulary and it means exactly ONE thing (DESIGN.md - # §4, THE THREE LOCKS): RECORDS cannot be added, deleted or edited — **fields still - # can**. The automation-owned IG child datasets (posts, comments, snapshots) are the - # live example; their rows arrive from the engine, so a "+" row there could only - # refuse, which R8 calls a fake affordance. - # - # ⚠ THE AUTHORITY IS `user_tables.records_mutable`, NOT THIS LINE. The comparison is - # inlined only because `_ut_defs` is already in hand — calling the predicate per row - # would be one store read per database on every nav request — and the VALUE comes - # from the module's own constant rather than a copied string, so the two cannot drift - # to different answers. If that predicate ever grows a second condition, this must - # become a call. - # - # ⚠ ABSENT READS AS UNLOCKED, and that is the safe direction here even though it is - # the opposite of `manage`'s fail-closed: the lock ICON is an affordance hint, while - # the actual refusal is `routes_tables._records_or_refuse`'s 403. A store blip costs - # a missing hint, never a write that should not have landed. - if (_ut_locked_mode - and (_ut_defs.get(key) or {}).get("recordMode") == _ut_locked_mode): - p["locked"] = True - elif session.admin: - p["manage"] = True - # ⭐ W34-T10 / C1 — always emitted for a database this route could READ, including when - # nothing is marked (`{marked: 0, counted: 0, partial: false}`). That is the same rule - # `omitted`/`degraded` follow at the bottom of this function and for the same reason: a key - # a consumer has to test for is a key a consumer forgets to test for. ABSENT here means - # "not a database, or we could not read it" — the second case is named in `degraded`. - if key in _important: - p["important"] = _important[key] - entry = meta.get(p.get("key")) if meta else None - if not entry: - continue - if entry.get("icon"): - p["icon"] = entry["icon"] - if entry.get("name"): - p["label"] = entry["name"] - landing = perms.landing_page(session.user) - if landing and not any(p.get("key") == landing for p in pages): - landing = pages[0].get("key") - # WAVE 23 (C10 / R7) — the Home landing's recents, on the payload the client already asks - # for. A second round trip for a list this short, computed from a bucket this route is - # already holding the store open for, would be a request per page load for no gain. - # - # ⛔ `allowed` IS THE ASSEMBLED PAGE LIST — the rows this route just built, `ut_*` databases - # included. Pruning against `perms.nav_pages` alone would silently drop every user database - # from Home's recents: the exact surface R7 is about, invisible, with every gate green. - # - # ⚠ WAVE 27 (item 6): the OTHER consumer of that narrower set — the folder-placement door — - # had the identical bug and nobody connected the two for four waves. It is fixed at the - # source now (`_placeable_top_keys`, which merges the same `may_open`-filtered listing), so - # both doors finally agree on what a placeable key is. This block keeps using the assembled - # list because it already holds it: re-enumerating here would be a second store read for an - # answer sitting in a local variable. - # - # ⚠ W34-T10 MOVED THE READ, NOT THE RULE. `recents` is now computed ABOVE the enrichment loop, - # because the important-count block spends its budget in RECENTS ORDER (see there). It is still - # ONE read of `nav_recents`, still pruned against the assembled page list, and it is used here - # unchanged — a second `_read_recents` call would be the extra store read this comment forbids. - # ⭐ W31-T11 — TWO KINDS OF ABSENCE, NAMED SEPARATELY, and both keys are ALWAYS PRESENT. - # `omitted` — this workspace's catalogue does not include these modules. Deliberate. - # `degraded` — a part of this payload could not be read. NOT deliberate, and the shell says - # so in the affected slot instead of rendering a confident nothing. - # ⚠ A key a consumer has to test for is a key a consumer forgets to test for; both ship as - # `[]` rather than being omitted when empty, which is the same rule `limits` follows. - return {"pages": pages, "landing": landing, "recents": recents, - "omitted": _omitted, "degraded": _degraded} - - -@router.post("/nav/opened") -def nav_opened(body: dict = Body(default=None), - session: Session = Depends(require_session)): - """WAVE 23 (C10) — stamp a page as JUST OPENED. Fire-and-forget from the client. - - The client calls this on every route commit, so this is the only write in this file on a - hot path, and three things follow from that: - - · `flush='async'` — the coalescing mode ([[store-async-flush]]). A blocking upload per - page open against an HF-Dataset-backed store would put a network round trip inside every - navigation. `nav_prefs`/`nav_meta` stay `sync` because a folder rename is not a hot path; - this is. - · NO VALIDATION OF THE KEY against the nav. Building the page list to check one string - would make the stamp cost more than the navigation that triggered it — and it would buy - nothing, because the READ prunes to what the session may currently see. An unknown or - revoked key is stored and never served back. - · THE MAP IS CAPPED HERE TOO. Read-side capping alone would let a hostile or buggy client - grow one user's document without bound; `_MAX_RECENTS` entries survive, oldest first to - go, which is the same rule the read applies. - """ - body = body if isinstance(body, dict) else {} - key = str(body.get("key") or "").strip()[:60] - if not key: - raise err(400, "bad_request", "no page was named") - if not session.runtime.available(): - raise err(503, "store_unavailable", - "the tenant store is unavailable. Nothing was recorded.") - stamp, uname = _now(), session.uname - - def _up(data): - data = data if isinstance(data, dict) else {} - mine = dict(data.get(uname) or {}) if isinstance(data.get(uname), dict) else {} - mine[key] = stamp - if len(mine) > _MAX_RECENTS: - # Oldest first. `int(v)` guarded: a stamp an older build wrote in another shape - # sorts as 0 and is the first thing evicted, which is the right answer for a value - # this route can no longer read. - def _at(item): - try: - return int(item[1]) - except (TypeError, ValueError): - return 0 - mine = dict(sorted(mine.items(), key=_at, reverse=True)[:_MAX_RECENTS]) - data[uname] = mine - return data - - try: - session.runtime.update(_NAV_RECENTS_KEY, _up, flush='async') - except Exception: - raise err(503, "store_unavailable", - "the tenant store refused the write. Nothing was recorded.") - return {"key": key, "at": stamp} - - -@router.post("/nav/meta") -def save_nav_meta(body: dict = Body(default=None), - session: Session = Depends(require_session)): - """WAVE 19 (R8 / C1) — set one database's icon and/or name, tenant-wide. - - A PATCH OF ONE KEY, not the wholesale replace `/nav/prefs` uses two routes up, and the - asymmetry is deliberate. Prefs are one user's complete picture of their own rail, so - replacing the document whole is what makes a deleted folder stay deleted. This bucket is - shared by every admin in the tenant: a wholesale write here means whoever saves last - silently erases what the other one named while their tab was open. - - THREE WALLS, all fail-closed: - · the SESSION must be able to open the key at all (the same predicate the nav uses, so - the rail and this route cannot disagree about what exists); - · the WRITE is admin-only — renaming a database is a change every user in the tenant - sees, which is the definition of an administrative act here; - · `name` is refused outright for a non-`ut_` key. Built-in labels are compiled registry - literals: honouring an override would leave this payload and `core/registry.py` - calling the same module two different things, and the wave-16 lesson is that the - client must not be the place that decides what a payload meant. - - `icon: null` CLEARS. An absent field is untouched — which is what makes a rename and an - icon change two independent writes rather than a race between them. - """ - body = body if isinstance(body, dict) else {} - key = str(body.get("key") or "").strip() - if not key: - raise err(400, "bad_request", "no database was named") - # ── THE WALL (ruling R14, 2026-08-04) ──────────────────────────────────────────────────── - # TWO DOORS, because a `ut_` database and a built-in module are owned by different people. - # - # · `ut_*` — the table's CREATOR or a tenant admin, which is exactly what - # `user_tables.may_open` already means. A database you made is yours to name; requiring - # an admin for that was the C1 consequence B flagged and R14 resolved. `session.require` - # is deliberately NOT used here: it is the MODULE gate and would 403 every ut_ key, - # because a user table is not a module. Same split `/nav/schema/{key}` makes. - # · everything else — admin only, and icon only. There is no owner of `customer_data` to - # defer to, and its label is a compiled registry literal (refused below regardless). - if key.startswith("ut_"): - import core.user_tables as user_tables - if not user_tables.get(key, st=session.runtime) or not user_tables.may_open( - key, session.uname, session.admin, st=session.runtime): - raise err(403, "forbidden", "that database belongs to another user") - else: - if not session.admin: - raise err(403, "forbidden", - "only an administrator can change a built-in database's icon") - session.require(key) - - patch = {} - if "icon" in body: - icon = _clean_icon(body.get("icon")) - if body.get("icon") is not None and icon is None: - # Loud, not silent. A shape this build does not know is a CLIENT that has drifted - # from this whitelist, and answering 200 to a write that stored nothing is how - # that drift stays invisible until a user reports "my icon keeps resetting". - raise err(400, "bad_icon", "that icon is not one of the available shapes and tones") - patch["icon"] = icon - if "name" in body: - if not key.startswith("ut_"): - raise err(400, "name_not_allowed", - "only a database you created can be renamed. This one's name comes " - "from the module registry") - name = " ".join(str(body.get("name") or "").split())[:_MAX_NAV_NAME] - if not name: - raise err(400, "bad_request", "a database needs a name") - patch["name"] = name - if not patch: - raise err(400, "bad_request", "nothing to change") - - if not session.runtime.available(): - raise err(503, "store_unavailable", - "the tenant store is unavailable. Nothing was saved.") - - def _up(data): - data = data if isinstance(data, dict) else {} - entry = dict(data.get(key) or {}) if isinstance(data.get(key), dict) else {} - for field, value in patch.items(): - if value is None: - entry.pop(field, None) # an explicit null CLEARS - else: - entry[field] = value - # An entry with nothing left in it is removed rather than stored empty: the read side - # skips empties anyway, and a bucket that accumulates `{}` per key is a document that - # grows forever and says nothing. - if entry: - data[key] = entry - else: - data.pop(key, None) - return data - - try: - session.runtime.update(_NAV_META_KEY, _up) - except Exception: - raise err(503, "store_unavailable", - "the change was not saved: the store refused the write.") - return {"key": key, "meta": _read_nav_meta(session.runtime).get(key, {})} - - -@router.get("/nav/prefs") -def nav_prefs(session: Session = Depends(require_session)): - """This user's database-list folders, re-validated at serve time against what they may - currently see (a revoked page's placement vanishes with the page, and returns with it).""" - try: - stored = (session.runtime.get(_NAV_PREFS_KEY) or {}).get(session.uname) or {} - except Exception: - stored = {} - keys, enumerated = _placeable_top_keys(session) - return {"prefs": _clean_nav_prefs(stored, keys, keep_unknown_ut=not enumerated)} - - -@router.post("/nav/prefs") -def save_nav_prefs(body: dict = Body(default=None), - session: Session = Depends(require_session)): - """Wholesale replace, like the table folder stratum — the client sent its complete - picture, validated here; a partial merge would resurrect deleted folders forever.""" - keys, enumerated = _placeable_top_keys(session) - clean = _clean_nav_prefs(body or {}, keys, keep_unknown_ut=not enumerated) - if not session.runtime.available(): - raise err(503, "store_unavailable", - "the tenant store is unavailable. Nothing was saved.") - - def _up(data): - data = data if isinstance(data, dict) else {} - if clean["folders"] or clean["placement"]: - data[session.uname] = clean - else: - data.pop(session.uname, None) - return data - - try: - session.runtime.update(_NAV_PREFS_KEY, _up) - except Exception: - raise err(503, "store_unavailable", - "the folder change was not saved: the store refused the write.") - return {"prefs": clean} - - -@router.get("/nav/schema/{key}") -def nav_schema(key: str, session: Session = Depends(require_session)): - """The database's schema drawer payload: its field contract + the semantic measures this - session may build with. Fail-closed on the SAME predicate as the nav — a key the session - may not open answers 403, never a redacted schema.""" - # Wave 18 C3-UT: a user table's schema is its own definition, walled by ITS predicate - # (`may_open`) rather than the module grant machinery — `session.require` would 403 every - # ut key because a user table is deliberately not a module. - if key.startswith("ut_"): - import core.user_tables as user_tables - defn = user_tables.get(key, st=session.runtime) - if not defn or not user_tables.may_open(key, session.uname, session.admin, - st=session.runtime): - raise err(403, "forbidden", "that database belongs to another user") - # WAVE 19 (R8) — the drawer wears the RENAMED name. A rail that says one thing and a - # schema panel opened from it that says another is the drift a rename is supposed to - # remove, not create. - return {"key": key, - "label": (_read_nav_meta(session.runtime).get(key, {}).get("name") - or defn.get("label") or key), - "source": defn.get("source") or "Blank", - "fields": [{"key": f["key"], "label": f["label"], "type": f["type"], - "source": f.get("source") or "overlay", - "description": str(f.get("description") or "")} - for f in (defn.get("fields") or [])], - "measures": []} - session.require(key) - pages = perms.nav_pages(session.user) or [] - page = next((p for p in pages if p.get("key") == key), None) - if page is None: - raise err(404, "unknown_page", f"{key!r} is not a database this session can see") - fields = [] - if key in ("customer_data", "cohort", "customers"): - try: - import aios_grid - for f in aios_grid.FIELDS: - entry = {"key": f["key"], "label": f["label"], "type": f["type"], - "source": f["source"], - "description": str(f.get("description") or "")} - if f.get("options"): - entry["options"] = list(f["options"]) - fields.append(entry) - except Exception: - fields = [] - measures = [] - try: - from core import measure_resolve - team_id = perms.scope_team_id(session.user) - for m in measure_resolve.offer(team_id) or []: - measures.append({"key": str(m.get("key") or ""), - "label": str(m.get("label") or m.get("key") or ""), - "type": str(m.get("type") or "")}) - except Exception: - measures = [] - out = {"key": key, "label": page.get("label") or key, - "source": page.get("source") or "", "fields": fields, "measures": measures} - if not fields: - # Honest, never a mock: a database whose contract is not yet published says so. - out["note"] = "This database has not published a field contract yet." - return out +"""routes_nav.py — X2's `GET /api/v1/nav`: the registry, filtered by what this session may open. + +SERVER-FILTERED, not client-filtered. The client renders what it is given and never decides who +may see what — a nav that hides a link the API would still serve is a UI courtesy, not a +permission. `core.perms.nav_pages` is the single predicate (shared with `may_open`'s page gate), +so the nav and the 403 can never disagree about a grant. + +The full rule set — archived is invisible to everyone, `group_only` rows are excluded so no +`parent` reference can dangle, `nav: False` surfaces ship as `chrome: 'utility'`, and the +`prefs.json` Library preference is deliberately NOT applied — is documented on +`core.perms.nav_pages`, which is where it belongs: one place, both callers. +""" +import time + +from fastapi import APIRouter, Body, Depends + +from deps import Session, err, perms, require_session + +router = APIRouter(prefix="/api/v1") + +#: Wave 2026-08-02 (C-SCHEMA): per-user folders over the database list, the view-folder +#: pattern applied to the nav. Cosmetic per-user state — placement never grants or hides a +#: surface (the server-filtered nav still decides what exists). +_NAV_PREFS_KEY = "nav_prefs" +_MAX_NAV_FOLDERS = 16 + +#: WAVE 19 (R8 / contract C1): the database's NAME and ICON overrides. +#: +#: ⚠ TENANT-WIDE, which is the whole difference from `nav_prefs` above and the reason it is a +#: separate bucket rather than another field in that one. Folders and placement are one +#: person's arrangement of their own rail — per-user by definition. What a database is CALLED +#: and what it looks like are facts about the database: a workspace where two people call the +#: same table different things has no shared vocabulary left to discuss it in. Same store, two +#: buckets, because they answer to two different owners. +#: +#: Shape: {"": {"icon": {"shape": , "tone": }, +#: "name": ""}} +_NAV_META_KEY = "nav_meta" + +#: The grid's folder-icon vocabulary, mirrored — 12 shapes x 5 tones. The CLIENT imports these +#: from `customer-grid/types` rather than redefining them; this end cannot import TypeScript, +#: so it is the one place the list is written twice. +#: +#: ⛔ WHAT AN UNKNOWN VALUE MUST NOT DO IS BE STORED. `FolderMark` indexes its path table by +#: shape and maps the result, so a shape this whitelist let through and the renderer does not +#: know is `undefined.map()` — a blank rail, from a stored preference, for every user in the +#: tenant until somebody edits the store by hand. Refusing at the door is the cheap end of +#: that. If the two lists ever drift the symptom is an icon that silently reverts to the +#: default, which is the loudest SAFE failure available here. +_ICON_SHAPES = frozenset({"folder", "star", "flag", "tag", "bookmark", "grid", + "chart", "map", "users", "clock", "heart", "bolt"}) +_ICON_TONES = frozenset({"neutral", "blue", "green", "yellow", "red"}) +_MAX_NAV_NAME = 60 + +#: WAVE 23 (contract C10 / ruling R7) — the Home landing's RECENTS. +#: +#: PER-USER, like `nav_prefs` two buckets up and unlike `nav_meta`: what I opened last is +#: nobody else's business, and a tenant-wide "recently opened" would be a surveillance feature +#: rather than a convenience one. +#: +#: ⛔ A MAP KEYED BY PAGE, NOT AN APPEND LOG, and the difference is the whole feature. +#: `{username: {pageKey: }}` — re-opening a database OVERWRITES its stamp. An +#: append-only list capped at 50 fills with fifty copies of the same ten databases inside one +#: working session, and the cap then evicts OLDEST-FIRST: the tenth database you touched falls +#: off the list while forty slots hold repeat visits to the first. Keying by page makes +#: "recent" mean what the word means, and makes the cap bound the number of DATABASES +#: remembered rather than the number of clicks. +_NAV_RECENTS_KEY = "nav_recents" +_MAX_RECENTS = 50 + +#: ⚠ EPOCH SECONDS (UTC by definition), never a formatted stamp — and this is a correction of a +#: precedent, not a preference. `user_tables.create` writes +#: `datetime.now().strftime('%Y-%m-%dT%H:%M:%S')`: naive LOCAL time, no offset. A browser parses +#: that string as its OWN local time, so on a UTC host read by a non-UTC reader "opened 30 +#: minutes ago" renders as "opened 7 hours ago" and the Today / Past-7-days buckets misfile — +#: with nothing to go red, because both ends are internally consistent. An integer instant has +#: no such reading. (D-18 made the same correction for notification stamps AFTER the defect +#: shipped; this is that lesson applied before it.) +def _now() -> int: + return int(time.time()) + + +def _clean_nav_prefs(raw, page_keys, *, keep_unknown_ut=False): + """Validated wholesale replacement, the `clean_folders` posture: prune, never invent. + + `page_keys` is the set of TOP-LEVEL keys this session may see; a placement of an unknown + or invisible key is dropped (it can return when the grant does — placement is cosmetic, + so pruning is loss-free). Unknown folder refs drop the placement, not the folder. + + `keep_unknown_ut` is the STORE-BLIP escape hatch — see `_placeable_top_keys`. When the + `ut_*` listing could not be read, a `ut_` key absent from `page_keys` is KEPT rather than + pruned: keeping a placement whose database may since have been deleted is cosmetically + harmless, while pruning a live one is the silent data loss this function just stopped + causing. Never widened to non-`ut_` keys — those come from the compiled registry, which + cannot fail to enumerate. + """ + raw = raw if isinstance(raw, dict) else {} + folders, seen = [], set() + for f in (raw.get("folders") or [])[:_MAX_NAV_FOLDERS]: + if not isinstance(f, dict): + continue + fid = str(f.get("id") or "").strip()[:40] + name = " ".join(str(f.get("name") or "").split())[:40] + if not fid or fid in seen or not name: + continue + seen.add(fid) + folders.append({"id": fid, "name": name}) + ids = {f["id"] for f in folders} + placement = {} + src = raw.get("placement") + if isinstance(src, dict): + for k, v in src.items(): + k, v = str(k)[:60], str(v)[:40] + if v not in ids: + continue # the folder is gone; the placement goes with it + if page_keys is None or k in page_keys: + placement[k] = v + elif keep_unknown_ut and k.startswith("ut_"): + placement[k] = v + return {"folders": folders, "placement": placement} + + +def _placeable_top_keys(session): + """`(keys, enumerated)` — the keys a placement may name, INCLUDING this tenant's databases. + + ⛔ THIS IS THE ITEM-6 FIX (wave 27, contract C1), and the bug it closes was invisible by + construction. The old version read `perms.nav_pages` alone — the compiled REGISTRY — and + `perms.py` contains no `ut_` or `user_tables` reference at all, because a tenant's + user-created databases are merged into the nav payload by `nav()` BELOW the permission wall. + So every `ut_*` key was absent from this set, and `_clean_nav_prefs` pruned every placement + naming one — on READ and on WRITE. + + The symptom was "a database I drag into a folder falls out of it later", never "it does not + work": the write answered **200 OK** having stored `{}`, the client kept its optimistic copy + (`Shell.commitPrefs` reverts only on `!ok`), and the folder held for the rest of the session. + The next reload served the pruned copy and the database was back at root. Built-in modules + (`customer_data`, `product_data`) ARE in the registry and persisted fine, which is exactly + why it read as intermittent rather than as a missing feature. + + ⚠ THE SAME MISTAKE, ALREADY MADE AND ALREADY FIXED ONE FUNCTION AWAY. `nav()`'s recents + block (see its `allowed` comment) hit this in wave 23 and solved it by pruning against the + ASSEMBLED page list. This is that lesson applied to the second consumer, which the wave-23 + fix did not reach. + + `enumerated` is False when the `ut_` listing raised — the caller must then NOT treat this set + as authoritative for `ut_` keys (`keep_unknown_ut`). Pruning a person's whole arrangement + because the store blinked would be the original defect wearing a different cause. + """ + pages = perms.nav_pages(session.user) or [] + keys = {p["key"] for p in pages if not p.get("parent")} + try: + import core.user_tables as user_tables + # The SAME listing `nav()` renders from — `may_open`-filtered, so a placement can never + # name a database this session cannot see, and the nav and this door agree by sharing + # one predicate rather than by two lists being kept in step. + # ⭐ W31-T10 (C1/D-175): `nav_entries` lends its own read to `may_open`, so this door is + # ONE document copy rather than `1 + N`. It is not a footnote here — `Shell.tsx` fires + # `/nav/prefs` CONCURRENTLY with `/nav` on the same `Store._lock`, and it measured + # **8,807 ms live / 17,945 ms in-process** on tenant #0, i.e. roughly half the wait the + # owner reports as "the Automation row arrives ten seconds late". + for e in user_tables.nav_entries(viewer=session.uname, is_admin=session.admin, + st=session.runtime): + keys.add(str(e.get("key") or "")) + except Exception: + return keys, False + return keys, True + + +def _clean_icon(raw): + """A stored/incoming icon, or None. Prune, never invent — `clean_folders`' posture.""" + if not isinstance(raw, dict): + return None + shape, tone = raw.get("shape"), raw.get("tone") + if shape not in _ICON_SHAPES or tone not in _ICON_TONES: + return None + return {"shape": shape, "tone": tone} + + +def _read_nav_meta(runtime): + """The tenant's `nav_meta`, validated on the way OUT as well as in. + + Re-validating a read looks redundant and is not: the bucket outlives this code, an older + build may have written a shape this one no longer accepts, and the whitelist is mirrored + from a vocabulary that lives in another language. A row whose icon does not survive + validation renders the default mark — never a crash, and never a half-drawn glyph. + """ + try: + stored = runtime.get(_NAV_META_KEY) or {} + except Exception: + return {} # a store blip must not take the nav down with it + if not isinstance(stored, dict): + return {} + out = {} + for key, meta in stored.items(): + if not isinstance(meta, dict): + continue + entry = {} + icon = _clean_icon(meta.get("icon")) + if icon: + entry["icon"] = icon + # ⛔ THE `ut_` RULE IS RE-APPLIED ON READ, not trusted from the record. The write + # route refuses a name for a built-in key, but a record written by an older build (or + # by hand) could still carry one — and honouring it would let the store rename + # `customer_data` on screen while `core/registry.py` and every other reader went on + # calling it Customer. A name that could not be written today is not served today. + name = meta.get("name") + if str(key).startswith("ut_") and isinstance(name, str) and name.strip(): + entry["name"] = " ".join(name.split())[:_MAX_NAV_NAME] + if entry: + out[str(key)] = entry + return out + + +def _read_recents(runtime, uname, allowed): + """This user's recents, newest first, PRUNED to what they may currently see. + + ⚠ PRUNED ON READ, never on write, and both halves of that are deliberate: + + · the WRITE happens on every route open — it is the one hot path this file has — so it + must not build the whole nav to validate one key; + · a key whose grant was REVOKED must stop being offered without anyone running a + migration, and must come back if the grant does. That is `_clean_nav_prefs`' own + prune-never-invent posture, applied to a second bucket for the same reason. + + A key that is not in `allowed` therefore reaches the store and never reaches a screen — + which also means a stuffed key cannot be used to discover what exists: it comes back only + if the session could already see it. + """ + try: + stored = (runtime.get(_NAV_RECENTS_KEY) or {}).get(uname) or {} + except Exception: + return [] # a store blip must not take the nav down with it + if not isinstance(stored, dict): + return [] + out = [] + for key, at in stored.items(): + key = str(key) + if allowed is not None and key not in allowed: + continue + try: + at = int(at) + except (TypeError, ValueError): + continue # a stamp this build cannot read is not a stamp + if at <= 0: + continue + out.append({"key": key, "at": at}) + out.sort(key=lambda r: r["at"], reverse=True) + return out[:_MAX_RECENTS] + + +# ── VIEW READERS — shared with `routes_starred` (W35-T39/T42) ──────────────────────────────── +# +# ⛔⛔ W35-T43 (rulings R4/R7) — THE "MARK IMPORTANT" COUNTS ARE GONE FROM THIS ROUTE, and the +# block that computed them (`_important_counts`) went with them. It read one store bucket PER +# DATABASE on the route D-175 spent a whole wave reducing to a single read: MEASURED on a 13-database +# fixture, `GET /nav` performed **14 per-database view-bucket reads** and stamped `important` on +# **14** pages. It is **0 and 0** now. +# +# Those reads were bounded by a 2.5 s wall clock (`_IMPORTANT_BUDGET_S`) whose own note conceded the +# cost: twelve buckets are 6.2 ms WARM and **7,331 ms COLD**, so the budget existed to stop a cold +# container converting a slow success into a manufactured failure against `nav.ts`'s 20 s deadline. +# A budget that can be exhausted is a number that can be silently short, which is why it also had to +# publish a `degraded` entry. R7 removes the whole apparatus by moving the question to +# `GET /starred/counts`, called AFTER the page paints — where a slow answer costs a late badge +# instead of a late rail. Closes D-288 and D-289. +# +# ⚠ WHAT SURVIVES AND WHY: `_visible_views`, `_view_record_count` and `_granted_view_ids` are the +# READERS, and `routes_starred` calls all three. They were never the cost — the per-database store +# read was — and duplicating them into the new route would have put two answers behind one badge. + +#: Suffix `core.view_templates.workspace_key` appends. Stripping it is how a page key becomes the +#: TOPIC key the cohort bucket is named from — `customer_data` -> `customer_table_workspace` -> +#: `customer` -> `customer_cohorts`. Derived rather than re-listed on purpose: `_WS_KEYS` is +#: already the one place `customer_data`/`product_data` are mapped to their topic, and a second +#: copy here would be free to drift from it ([[one-question-two-normalizers]]). +_WS_SUFFIX = "_table_workspace" + + +def _visible_views(doc, uname, is_admin, granted_ids): + """Every view on ONE database that THIS caller can see, from an already-read workspace doc. + + ⛔ PER-CALLER, NOT TENANT-WIDE, and this is the half a server-side count is most likely to get + wrong. `_table_workspace` is `{username: {views, fields, overlays}, '__shared__': {...}}` + — one home per view, never both — so "how many views are marked important here" has a + DIFFERENT answer for every account. Measured on tenant #0: `leadership` has one marked view and + `admin` has none, on the same database. A tenant-wide count would put a number in the owner's + rail that no view sidebar they can open would ever add up to. + + ⚠ `table_store._may_see` is imported rather than re-expressed. It is private, and reaching for + it is still the right call: the alternative is `TableOps.shared_views`, which re-reads the whole + bucket per database (the N whole reads this block exists to avoid), and the only other option is + a second copy of a permission predicate. A wrong copy of `_may_see` widens what a user is told + exists; a private import cannot. + """ + import core.table_store as table_store + out = {} + for vid, view in ((doc.get(uname) or {}).get("views") or {}).items(): + if isinstance(view, dict): + out[str(vid)] = view + for vid, view in ((doc.get(table_store.SHARED_KEY) or {}).get("views") or {}).items(): + if isinstance(view, dict) and table_store._may_see(view, uname, is_admin): + out.setdefault(str(vid), view) + # The wave-21 named-user grants. The ids come from ONE tenant-wide bucket read once for the + # whole request; the RECORD is already in hand, in whichever stratum its owner keeps it. + if granted_ids: + for stratum, blob in doc.items(): + if stratum == uname or not isinstance(blob, dict): + continue + for vid, view in (blob.get("views") or {}).items(): + if str(vid) in granted_ids and isinstance(view, dict): + out.setdefault(str(vid), view) + return out + + +def _view_record_count(cfg, cohorts): + """How many RECORDS this view resolves to, or None when that cannot be answered for free. + + ⛔ `None` IS AN ANSWER AND IT IS THE IMPORTANT ONE. Two of the three shapes below are exact + because the view CARRIES its row set; the third — an ordinary filtered view — can only be + counted by running its filters over the records, and the records are the one thing this route + must never read (`D-175`/`D-185`: `/nav` is a rows-free projection, and reaching for `rows` + here raises by that projection's own contract). So a filtered view is reported as UNCOUNTED and + the database's `partial` flag says so, which is the whole of R6's second sentence applied to a + badge: a limit that cannot be removed is REPORTED with its cause, never papered over with a + number that is short by an unknown amount. + + ⚠ THIS IS ALSO WHY THE SERVER DOES NOT SIMPLY MIRROR THE CLIENT. `CustomerGrid::alertCounts` + counts a filtered view fine and gives up on a SERVER-WINDOWED one (`D-205`'s `Important 0+`); + this end is the exact inverse — it has no rows at all and no window either. The two are honest + about different halves, which is why `partial` had to be on the wire rather than derived. + """ + if not isinstance(cfg, dict): + return None + # A cohort-locked view IS its cohort: the lock and the id are the same fact (`grid_events` + # re-stamps it on every write), and a cohort's membership is a stored pid LIST, not a query. + lock = str(cfg.get("cohortLock") or "").strip() + if lock: + n = cohorts.get(lock) + return int(n) if isinstance(n, int) else None + # A curated row set carries its own count. + pids = cfg.get("memberPids") + if isinstance(pids, list) and pids: + return len(pids) + return None + + +def _visible_database_keys(session): + """`(keys, enumerated)` — every DATABASE this session may open that HAS a view bucket. + + ⭐ ADDED BY W35-T39 because neither existing enumeration in this file answers this question: + + · `_placeable_top_keys` applies the ACCOUNT grant and `may_open`, and **not the TENANT + CATALOGUE** — correctly, because a folder placement is cosmetic and pruning one is loss. + A star scan cannot borrow that: it would open the view bucket of a database this workspace's + catalogue does not include. `nav()` applies the catalogue filter; that helper never has. + · `nav()`'s own `_page_keys` is assembled from the page DICTS it is building for the wire, so + it cannot be reached from another route without rebuilding the payload. + + So this is the KEY-SET question on its own, with the same three walls `nav()` applies in the same + order: the account grant (`perms.nav_pages`), the TENANT catalogue, and `may_open` for the + tenant's own databases. `view_templates.workspace_key` is the last filter and it is what makes + the answer honest for a caller about to read a view bucket: a module surface with no workspace + (`sales`, `ar`) is not a database and has no views to star. + + ⚠ THE `parent` CLAUSE BELOW IS A NO-OP TODAY AND IS KEPT ONLY TO MIRROR `nav()`. `perms.nav_pages` + excludes `group_only` rows and therefore **emits no `parent` at all** (its own docstring says so: + a dangling reference would be worse than a flat list), so `customer_data` arrives here FLAT even + though the registry gives it `parent: 'customers'`. Stated because the opposite is the obvious + reading — this ticket's first draft assumed the clause was excluding children and wrote a + docstring around a defect that does not exist. + + ⚠ `enumerated` is False when the `ut_` listing raised — the caller must not treat the set as + authoritative for `ut_` keys, exactly as `_placeable_top_keys` requires. + """ + import core.view_templates as view_templates + keys, enumerated = set(), True + tcfg = getattr(session.runtime.tenant, "config", None) or {} + tmods = tcfg.get("modules", "all") + allowed = None if tmods == "all" else {str(k) for k in (tmods or [])} + for p in (perms.nav_pages(session.user) or []): + key = str(p.get("key") or "") + if allowed is not None and key not in allowed \ + and str(p.get("parent") or "") not in allowed: + continue + if view_templates.workspace_key(key): + keys.add(key) + try: + import core.user_tables as user_tables + # The SAME `may_open`-filtered listing `nav()` renders from, over the ROWS-FREE projection + # (W32-T02) — so this costs ~0.1% of the document rather than a 28.6 MB deep copy. + for e in user_tables.nav_entries(viewer=session.uname, is_admin=session.admin, + st=session.runtime): + k = str(e.get("key") or "") + if k: + keys.add(k) + except Exception: + enumerated = False + return keys, enumerated + + +def _granted_view_ids(session): + """Every view id a wave-21 NAMED-USER GRANT lets this caller see. Fail-closed to `set()`. + + ⭐ EXTRACTED (W35-T39) SO THE STAR AND THE COUNT SHARE ONE ANSWER. `_visible_views` above + takes this set as its third stratum, and it had exactly one caller (`_important_counts`) with + the derivation inline — which W35-T43 deletes. `routes_starred` needs the identical set to + answer "which views has this person starred", so the derivation moves here beside the reader + it feeds rather than being copied into a second file ([[one-evaluator-per-question]]). + + ⚠ THE ROLE FILTER IS NOT BELT-AND-BRACES. `shared_with` answers "is there an entry naming + me", and `grid_events._granted_views` — the reader whose answer any consumer of this has to + agree with — then requires `role_for(...) in ('view','edit')`. Dropping that second test + would admit a view the sidebar does not list. One tenant-wide bucket, read once per call, so + it costs a dict lookup per id. + ⚠ Fail-closed: no grants is a NARROWER answer, never a wider one. + """ + import core.shares as shares + try: + return {str(v) for v in + ((shares.shared_with(session.uname, kind="view", st=session.runtime) or {}) + .get("view") or []) + if shares.role_for("view", str(v), session.uname, + st=session.runtime) in ("view", "edit")} + except Exception: + return set() + + +@router.get("/nav") +def nav(session: Session = Depends(require_session)): + """`{pages: [{key,label,source?,chrome}], landing}`. + + Note what this does NOT do: it never returns an empty `pages` list as a way of saying "you + are not allowed". A session that may open nothing at all is a misconfigured account, and it + gets an explicit 403 — an empty 200 is indistinguishable from "the registry is empty" and is + how a permission bug hides in plain sight (X2's never-an-empty-200 rule). + """ + pages = list(perms.nav_pages(session.user) or []) + # Wave 18 C1-TENANT: the REGISTRY is the product's module catalogue — tenant #0's world. + # A tenant record may enable a subset (`modules: [...]`); absent/'all' means everything + # (royal-imports and the compiled builders). A blank tenant enables NOTHING: its nav is + # its own databases, which is what "different set of databases at later waves" means. + tcfg = getattr(session.runtime.tenant, "config", None) or {} + tmods = tcfg.get("modules", "all") + # ⭐⭐ W31-T11 (owner item 6b) — WHAT THE CATALOGUE FILTER REMOVED, SAID OUT LOUD. + # + # Every provisioned tenant carries a restricted list today (`gtmlab`/`loopable`/`nurilab` all + # `['analyst','automation']` — census in `proto/nav-omission-census.md`), so this filter runs + # on every request that is not tenant #0's. It is the INTENDED catalogue, not a defect. But a + # row it removes and a row a store failure dropped are the SAME absence on the wire, and the + # client renders `null` for both — pixel-identical to still-loading. Naming the removals is + # what lets the shell tell "not part of this workspace" from "we could not read it". + _omitted = [] + if tmods != "all": + allowed = {str(k) for k in (tmods or [])} + _omitted = sorted({str(p.get("key")) for p in pages + if p.get("key") not in allowed + and (p.get("parent") or "") not in allowed}) + pages = [p for p in pages + if p.get("key") in allowed or (p.get("parent") or "") in allowed] + # Wave 18 C3-UT: this tenant's user-created databases, merged AFTER the registry rows — + # the host does the same at app.py:7796. Filtered by the per-table wall (`may_open`), so + # the nav cannot offer a row the table routes would refuse. + # ⭐⭐ W31-T10 (contract C1, D-175) — THE DOCUMENT IS READ ONCE FOR THE WHOLE REQUEST. + # + # This route was `2 + N` full deep copies of a 35.8 MB-ceiling document: `nav_entries` took one + # and then `may_open` took another PER TABLE, and the `manage`/`canDelete`/`locked` loop below + # took a SECOND independent one. Median `GET /nav` on tenant #0: **9,548 ms live, 24,741 ms + # in-process** — the second figure is the one that matters, because with the network gone and + # the document resident this route and `/nav/prefs` are the ONLY two that stay slow while every + # other collapses to 20–161 ms. That is per-call CPU, and this is it. + # ⚠ `_ut_defs` is read HERE, above the merge, so the two former reads become one; the locked/ + # manage loop below consumes this same dict. + _ut_defs, _ut_locked_mode, _degraded = {}, "", [] + try: + import core.user_tables as user_tables + # ⭐⭐ W32-T02 (R8/D-175/D-185) — AND THAT ONE READ IS A PROJECTION NOW. The block below + # reads `label`, `source`, `createdBy` and `recordMode`; `may_open` reads `createdBy` and + # the shares registry. Nothing on this route has ever opened a row — and on tenant #0 the + # rows are **99.89% of the document** (28,551,441 bytes; the definitions are 31,220 of + # them). Measured: `all_tables` 1,750 ms -> `all_defs` 1.4 ms, and `GET /nav` 1,921 ms + # median -> see the ticket. THAT is owner item 5: the rail rows did not arrive seconds + # apart because of CSS or ordering, they arrived when their route finished copying. + # ⛔ `all_defs` REFUSES `rows` rather than answering `{}` — if you add something here that + # needs a row, it raises with the reason instead of painting an empty grid. Use + # `all_tables` for that, and know you are buying the whole copy back. + _ut_defs = user_tables.all_defs(st=session.runtime) or {} + # ⛔ NEVER DEFAULT THIS TO `None`. `_ut_defs.get(k)` is `{}` for an unknown key, so its + # `.get("recordMode")` is None too — and `None == None` would mark EVERY database locked + # the moment this import failed. A sentinel that can equal real data is not a sentinel. + _ut_locked_mode = str(user_tables.AUTOMATION_RECORD_MODE) + _lent = user_tables.lend(session.runtime, **{user_tables.STORE_KEY: _ut_defs}) + for e in user_tables.nav_entries(viewer=session.uname, is_admin=session.admin, + st=_lent): + pages.append({"key": e["key"], "label": e["label"], + "source": e.get("source") or "Blank", "chrome": "main"}) + except Exception: + # ⭐⭐ W31-T11 (owner item 6b) — STILL SWALLOWED, NO LONGER SILENT. + # + # A store blip must not take the whole nav down with it, and that half stands. What + # changed is that it used to answer **200 OK with every database missing** and say + # nothing — so a client cannot tell "this tenant has no databases" from "we could not + # read them", and the rail renders the same complete-looking thing either way. That is + # the owner's *"Connectors and Automation module still disappears"* class of report: + # the payload is a claim about what exists, and a claim it could not verify has to be + # marked as such. `degraded` is that mark; the shell renders it in the affected slot. + _degraded.append("databases") + pass + if not pages: + if tmods != "all": + # A provisioned tenant with no modules and no databases YET is a legitimate empty + # state, not a misconfigured account — the client renders "create your first + # database", and X2's never-an-empty-200 rule is honoured by saying WHY it is + # empty rather than leaving 200-[] ambiguous. + return {"pages": [], "landing": None, "empty": "no_databases"} + raise err(403, "no_surfaces", + "your account has no dashboards assigned. Ask an administrator.") + # WAVE 19 (R8 / C1): the tenant's name + icon overrides, merged LAST — after the registry + # rows, after the tenant module filter, after the user tables. Merged HERE rather than + # applied by the client for one reason: `label` is what every reader of this payload shows, + # including the Settings modal's `moduleLabels` map, so a client-side merge would put the + # override in one door and the registry label in the other. + # + # ⛔ THIS MUTATES `pages` IN PLACE, WHICH IS SAFE ONLY BECAUSE `perms.nav_pages` BUILDS A + # FRESH `row = {...}` PER CALL (perms.py:194) and the ut_ rows are built fresh here. If + # either ever starts handing back cached or module-level dicts, this loop would write one + # tenant's chosen label into the object the NEXT tenant's request reads — a cross-tenant + # leak with no symptom until two tenants rename the same registry key. Copy the rows before + # merging on the day that invariant changes. + meta = _read_nav_meta(session.runtime) + # ⛔⛔ W35-T43 (rulings R4/R7) — THE MARK-IMPORTANT COUNTS ARE NO LONGER COMPUTED HERE. + # + # W34-T10 put them on this payload with a 2.5 s budget and a `degraded` entry, spent in RECENTS + # ORDER so a cold container would reach the databases somebody actually works in first. Every + # one of those was a correct answer to the wrong question: the block was `N` per-database store + # reads on the route D-175 spent a wave reducing to ONE, and no ordering makes an N-read block + # free. MEASURED on a 13-database fixture: **14 view-bucket reads and 14 `important` stamps per + # `/nav`** before this ticket, **0 and 0** after. + # + # R7 moves the question to `GET /starred/counts`, called AFTER the page paints. A slow answer + # there is a late badge; a slow answer here was a late RAIL. Closes D-288 and D-289. + # ⚠ `recents` STAYS and is still ONE read — it is the Home landing's own payload (C10/R7 of wave + # 23), and it was only computed this early so the deleted budget could spend itself in its order. + _page_keys = {str(p.get("key", "")) for p in pages} + recents = _read_recents(session.runtime, session.uname, _page_keys) + # Wave 21 (C3): the definitions, once — `manage`/`canDelete` below answer from `createdBy`. + # WAVE 27 (C9): `locked` answers from `recordMode`, off the same one read. + # ⭐ W31-T10: that read is now the SAME one the merge above did — it used to be a second, + # independent `all_tables`, which is why D-175 called this route `2 + N` rather than `1 + N`. + # ⚠ The fail-closed defaults still hold: both are initialised before the try above, so an + # import or store failure leaves `_ut_locked_mode` empty and nothing is marked locked. + for p in pages: + # `manage` (R14): may THIS session change this row's icon/name? Answered HERE because + # the server is the only end that knows — the client cannot see who created a user + # table. Additive and FAIL-CLOSED (absent reads as "no"), so the rail offers a control + # only where the write would actually land, and the route re-checks it regardless. + # + # ⛔ WAVE 21 (C3/W-5): the "presence IS the answer" shortcut DIED in wave 20 — the + # de5037f share-grant admission widened `may_open`, so a row's presence now includes + # databases merely SHARED to this viewer. `manage` (rename/icon) and `canDelete` are + # therefore answered from the DEFINITION: creator-or-admin strictly, matching the + # walls the PATCH and DELETE routes actually enforce. A grantee sees the row and no + # controls — the honest shape (the old code offered rename to users the route 403'd). + key = str(p.get("key", "")) + if key.startswith("ut_"): + _creator = str((_ut_defs.get(key) or {}).get("createdBy") or "") + _mine = bool(session.admin or (_creator and _creator == session.uname)) + p["manage"] = _mine + p["canDelete"] = _mine + # ⭐ WAVE 27 item 3 (contract C9) — A LOCKED DATABASE SAYS SO IN THE RAIL. + # + # "Locked" is the owner's item-4 vocabulary and it means exactly ONE thing (DESIGN.md + # §4, THE THREE LOCKS): RECORDS cannot be added, deleted or edited — **fields still + # can**. The automation-owned IG child datasets (posts, comments, snapshots) are the + # live example; their rows arrive from the engine, so a "+" row there could only + # refuse, which R8 calls a fake affordance. + # + # ⚠ THE AUTHORITY IS `user_tables.records_mutable`, NOT THIS LINE. The comparison is + # inlined only because `_ut_defs` is already in hand — calling the predicate per row + # would be one store read per database on every nav request — and the VALUE comes + # from the module's own constant rather than a copied string, so the two cannot drift + # to different answers. If that predicate ever grows a second condition, this must + # become a call. + # + # ⚠ ABSENT READS AS UNLOCKED, and that is the safe direction here even though it is + # the opposite of `manage`'s fail-closed: the lock ICON is an affordance hint, while + # the actual refusal is `routes_tables._records_or_refuse`'s 403. A store blip costs + # a missing hint, never a write that should not have landed. + if (_ut_locked_mode + and (_ut_defs.get(key) or {}).get("recordMode") == _ut_locked_mode): + p["locked"] = True + elif session.admin: + p["manage"] = True + # ⛔ W35-T43 — `important` IS NO LONGER STAMPED HERE. W34-T10 emitted it for every database + # this route could read, including `{marked: 0, counted: 0, partial: false}`, so a consumer + # never had to test for the key. R4 retires the count from the rail and the flyout entirely + # (A's W35-T07 is the client half) and R7 moves the numbers to `GET /starred/counts`. + # ⚠ `nav.ts::NavPage` still DECLARES `important` on the client until A's ticket lands; an + # absent key reads as `undefined` there, which is the same thing the optional field already + # meant for a non-database row. Flagged to A rather than assumed harmless. + entry = meta.get(p.get("key")) if meta else None + if not entry: + continue + if entry.get("icon"): + p["icon"] = entry["icon"] + if entry.get("name"): + p["label"] = entry["name"] + landing = perms.landing_page(session.user) + if landing and not any(p.get("key") == landing for p in pages): + landing = pages[0].get("key") + # WAVE 23 (C10 / R7) — the Home landing's recents, on the payload the client already asks + # for. A second round trip for a list this short, computed from a bucket this route is + # already holding the store open for, would be a request per page load for no gain. + # + # ⛔ `allowed` IS THE ASSEMBLED PAGE LIST — the rows this route just built, `ut_*` databases + # included. Pruning against `perms.nav_pages` alone would silently drop every user database + # from Home's recents: the exact surface R7 is about, invisible, with every gate green. + # + # ⚠ WAVE 27 (item 6): the OTHER consumer of that narrower set — the folder-placement door — + # had the identical bug and nobody connected the two for four waves. It is fixed at the + # source now (`_placeable_top_keys`, which merges the same `may_open`-filtered listing), so + # both doors finally agree on what a placeable key is. This block keeps using the assembled + # list because it already holds it: re-enumerating here would be a second store read for an + # answer sitting in a local variable. + # + # ⚠ W34-T10 MOVED THE READ, NOT THE RULE. `recents` is now computed ABOVE the enrichment loop, + # because the important-count block spends its budget in RECENTS ORDER (see there). It is still + # ONE read of `nav_recents`, still pruned against the assembled page list, and it is used here + # unchanged — a second `_read_recents` call would be the extra store read this comment forbids. + # ⭐ W31-T11 — TWO KINDS OF ABSENCE, NAMED SEPARATELY, and both keys are ALWAYS PRESENT. + # `omitted` — this workspace's catalogue does not include these modules. Deliberate. + # `degraded` — a part of this payload could not be read. NOT deliberate, and the shell says + # so in the affected slot instead of rendering a confident nothing. + # ⚠ A key a consumer has to test for is a key a consumer forgets to test for; both ship as + # `[]` rather than being omitted when empty, which is the same rule `limits` follows. + return {"pages": pages, "landing": landing, "recents": recents, + "omitted": _omitted, "degraded": _degraded} + + +@router.post("/nav/opened") +def nav_opened(body: dict = Body(default=None), + session: Session = Depends(require_session)): + """WAVE 23 (C10) — stamp a page as JUST OPENED. Fire-and-forget from the client. + + The client calls this on every route commit, so this is the only write in this file on a + hot path, and three things follow from that: + + · `flush='async'` — the coalescing mode ([[store-async-flush]]). A blocking upload per + page open against an HF-Dataset-backed store would put a network round trip inside every + navigation. `nav_prefs`/`nav_meta` stay `sync` because a folder rename is not a hot path; + this is. + · NO VALIDATION OF THE KEY against the nav. Building the page list to check one string + would make the stamp cost more than the navigation that triggered it — and it would buy + nothing, because the READ prunes to what the session may currently see. An unknown or + revoked key is stored and never served back. + · THE MAP IS CAPPED HERE TOO. Read-side capping alone would let a hostile or buggy client + grow one user's document without bound; `_MAX_RECENTS` entries survive, oldest first to + go, which is the same rule the read applies. + """ + body = body if isinstance(body, dict) else {} + key = str(body.get("key") or "").strip()[:60] + if not key: + raise err(400, "bad_request", "no page was named") + if not session.runtime.available(): + raise err(503, "store_unavailable", + "the tenant store is unavailable. Nothing was recorded.") + stamp, uname = _now(), session.uname + + def _up(data): + data = data if isinstance(data, dict) else {} + mine = dict(data.get(uname) or {}) if isinstance(data.get(uname), dict) else {} + mine[key] = stamp + if len(mine) > _MAX_RECENTS: + # Oldest first. `int(v)` guarded: a stamp an older build wrote in another shape + # sorts as 0 and is the first thing evicted, which is the right answer for a value + # this route can no longer read. + def _at(item): + try: + return int(item[1]) + except (TypeError, ValueError): + return 0 + mine = dict(sorted(mine.items(), key=_at, reverse=True)[:_MAX_RECENTS]) + data[uname] = mine + return data + + try: + session.runtime.update(_NAV_RECENTS_KEY, _up, flush='async') + except Exception: + raise err(503, "store_unavailable", + "the tenant store refused the write. Nothing was recorded.") + return {"key": key, "at": stamp} + + +@router.post("/nav/meta") +def save_nav_meta(body: dict = Body(default=None), + session: Session = Depends(require_session)): + """WAVE 19 (R8 / C1) — set one database's icon and/or name, tenant-wide. + + A PATCH OF ONE KEY, not the wholesale replace `/nav/prefs` uses two routes up, and the + asymmetry is deliberate. Prefs are one user's complete picture of their own rail, so + replacing the document whole is what makes a deleted folder stay deleted. This bucket is + shared by every admin in the tenant: a wholesale write here means whoever saves last + silently erases what the other one named while their tab was open. + + THREE WALLS, all fail-closed: + · the SESSION must be able to open the key at all (the same predicate the nav uses, so + the rail and this route cannot disagree about what exists); + · the WRITE is admin-only — renaming a database is a change every user in the tenant + sees, which is the definition of an administrative act here; + · `name` is refused outright for a non-`ut_` key. Built-in labels are compiled registry + literals: honouring an override would leave this payload and `core/registry.py` + calling the same module two different things, and the wave-16 lesson is that the + client must not be the place that decides what a payload meant. + + `icon: null` CLEARS. An absent field is untouched — which is what makes a rename and an + icon change two independent writes rather than a race between them. + """ + body = body if isinstance(body, dict) else {} + key = str(body.get("key") or "").strip() + if not key: + raise err(400, "bad_request", "no database was named") + # ── THE WALL (ruling R14, 2026-08-04) ──────────────────────────────────────────────────── + # TWO DOORS, because a `ut_` database and a built-in module are owned by different people. + # + # · `ut_*` — the table's CREATOR or a tenant admin, which is exactly what + # `user_tables.may_open` already means. A database you made is yours to name; requiring + # an admin for that was the C1 consequence B flagged and R14 resolved. `session.require` + # is deliberately NOT used here: it is the MODULE gate and would 403 every ut_ key, + # because a user table is not a module. Same split `/nav/schema/{key}` makes. + # · everything else — admin only, and icon only. There is no owner of `customer_data` to + # defer to, and its label is a compiled registry literal (refused below regardless). + if key.startswith("ut_"): + import core.user_tables as user_tables + if not user_tables.get(key, st=session.runtime) or not user_tables.may_open( + key, session.uname, session.admin, st=session.runtime): + raise err(403, "forbidden", "that database belongs to another user") + else: + if not session.admin: + raise err(403, "forbidden", + "only an administrator can change a built-in database's icon") + session.require(key) + + patch = {} + if "icon" in body: + icon = _clean_icon(body.get("icon")) + if body.get("icon") is not None and icon is None: + # Loud, not silent. A shape this build does not know is a CLIENT that has drifted + # from this whitelist, and answering 200 to a write that stored nothing is how + # that drift stays invisible until a user reports "my icon keeps resetting". + raise err(400, "bad_icon", "that icon is not one of the available shapes and tones") + patch["icon"] = icon + if "name" in body: + if not key.startswith("ut_"): + raise err(400, "name_not_allowed", + "only a database you created can be renamed. This one's name comes " + "from the module registry") + name = " ".join(str(body.get("name") or "").split())[:_MAX_NAV_NAME] + if not name: + raise err(400, "bad_request", "a database needs a name") + patch["name"] = name + if not patch: + raise err(400, "bad_request", "nothing to change") + + if not session.runtime.available(): + raise err(503, "store_unavailable", + "the tenant store is unavailable. Nothing was saved.") + + def _up(data): + data = data if isinstance(data, dict) else {} + entry = dict(data.get(key) or {}) if isinstance(data.get(key), dict) else {} + for field, value in patch.items(): + if value is None: + entry.pop(field, None) # an explicit null CLEARS + else: + entry[field] = value + # An entry with nothing left in it is removed rather than stored empty: the read side + # skips empties anyway, and a bucket that accumulates `{}` per key is a document that + # grows forever and says nothing. + if entry: + data[key] = entry + else: + data.pop(key, None) + return data + + try: + session.runtime.update(_NAV_META_KEY, _up) + except Exception: + raise err(503, "store_unavailable", + "the change was not saved: the store refused the write.") + return {"key": key, "meta": _read_nav_meta(session.runtime).get(key, {})} + + +@router.get("/nav/prefs") +def nav_prefs(session: Session = Depends(require_session)): + """This user's database-list folders, re-validated at serve time against what they may + currently see (a revoked page's placement vanishes with the page, and returns with it).""" + try: + stored = (session.runtime.get(_NAV_PREFS_KEY) or {}).get(session.uname) or {} + except Exception: + stored = {} + keys, enumerated = _placeable_top_keys(session) + return {"prefs": _clean_nav_prefs(stored, keys, keep_unknown_ut=not enumerated)} + + +@router.post("/nav/prefs") +def save_nav_prefs(body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Wholesale replace, like the table folder stratum — the client sent its complete + picture, validated here; a partial merge would resurrect deleted folders forever.""" + keys, enumerated = _placeable_top_keys(session) + clean = _clean_nav_prefs(body or {}, keys, keep_unknown_ut=not enumerated) + if not session.runtime.available(): + raise err(503, "store_unavailable", + "the tenant store is unavailable. Nothing was saved.") + + def _up(data): + data = data if isinstance(data, dict) else {} + if clean["folders"] or clean["placement"]: + data[session.uname] = clean + else: + data.pop(session.uname, None) + return data + + try: + session.runtime.update(_NAV_PREFS_KEY, _up) + except Exception: + raise err(503, "store_unavailable", + "the folder change was not saved: the store refused the write.") + return {"prefs": clean} + + +@router.get("/nav/schema/{key}") +def nav_schema(key: str, session: Session = Depends(require_session)): + """The database's schema drawer payload: its field contract + the semantic measures this + session may build with. Fail-closed on the SAME predicate as the nav — a key the session + may not open answers 403, never a redacted schema.""" + # Wave 18 C3-UT: a user table's schema is its own definition, walled by ITS predicate + # (`may_open`) rather than the module grant machinery — `session.require` would 403 every + # ut key because a user table is deliberately not a module. + if key.startswith("ut_"): + import core.user_tables as user_tables + defn = user_tables.get(key, st=session.runtime) + if not defn or not user_tables.may_open(key, session.uname, session.admin, + st=session.runtime): + raise err(403, "forbidden", "that database belongs to another user") + # WAVE 19 (R8) — the drawer wears the RENAMED name. A rail that says one thing and a + # schema panel opened from it that says another is the drift a rename is supposed to + # remove, not create. + return {"key": key, + "label": (_read_nav_meta(session.runtime).get(key, {}).get("name") + or defn.get("label") or key), + "source": defn.get("source") or "Blank", + "fields": [{"key": f["key"], "label": f["label"], "type": f["type"], + "source": f.get("source") or "overlay", + "description": str(f.get("description") or "")} + for f in (defn.get("fields") or [])], + "measures": []} + session.require(key) + pages = perms.nav_pages(session.user) or [] + page = next((p for p in pages if p.get("key") == key), None) + if page is None: + raise err(404, "unknown_page", f"{key!r} is not a database this session can see") + fields = [] + if key in ("customer_data", "cohort", "customers"): + try: + import aios_grid + for f in aios_grid.FIELDS: + entry = {"key": f["key"], "label": f["label"], "type": f["type"], + "source": f["source"], + "description": str(f.get("description") or "")} + if f.get("options"): + entry["options"] = list(f["options"]) + fields.append(entry) + except Exception: + fields = [] + measures = [] + try: + from core import measure_resolve + team_id = perms.scope_team_id(session.user) + for m in measure_resolve.offer(team_id) or []: + measures.append({"key": str(m.get("key") or ""), + "label": str(m.get("label") or m.get("key") or ""), + "type": str(m.get("type") or "")}) + except Exception: + measures = [] + out = {"key": key, "label": page.get("label") or key, + "source": page.get("source") or "", "fields": fields, "measures": measures} + if not fields: + # Honest, never a mock: a database whose contract is not yet published says so. + out["note"] = "This database has not published a field contract yet." + return out diff --git a/api/routes_query.py b/api/routes_query.py index 0323ba425d55fb98d96db93494d65130f9b945db..cfa0e6b44d6d1b7ddebde412fd4d18e7886b0f8f 100644 --- a/api/routes_query.py +++ b/api/routes_query.py @@ -355,13 +355,24 @@ def _from_chat(answer, provider): return answer, None, provider, None -def _call_model(question, snapshot, model=MODEL_AUTO, chat=None, history=None): +def _call_model(question, snapshot, model=MODEL_AUTO, chat=None, history=None, session=None): """Return ``(spec, said, provider, reason)`` without any source-data fallback. ``said`` is what the assistant SAID: the whole answer when it did not build a view, and the sentence beside the view when it built one and talked as well. ``reason`` is set only when something went wrong, so a prose ANSWER and a transport FAILURE are distinguishable one layer up rather than both arriving as a bare sentence. + + ⭐⭐ W35-T31 · CONTRACT C7 (R9) — ``session`` IS HERE ONLY SO THE METER CAN BE TOLD WHOSE CALL + THIS WAS, and PRD amendment A1 is why it is an argument rather than ambient state: a + ``ContextVar`` bound in ``deps.require_session`` reads back ``None`` inside this function on the + SAME thread, so a ledger built on one would have recorded nothing and shown a dashboard of + zeros indistinguishable from a quiet week. + + ⚠ OPTIONAL, DELIBERATELY. ``verify_query`` calls this function directly with an injected + ``chat`` and no session, and a required argument would have made every one of those calls a + signature change. An omitted session is COUNTED in ``usage_ledger.UNATTRIBUTED`` and reported + to an admin, never dropped. """ requested = str(model or MODEL_AUTO).strip().lower() if requested not in model_choices(): @@ -384,6 +395,27 @@ def _call_model(question, snapshot, model=MODEL_AUTO, chat=None, history=None): return None, sentence, None, "model_unavailable" if requested != MODEL_AUTO else "not_configured" import requests + + import usage_ledger + + def _book(provider, body): + """⭐⭐ C7 — ONE LEDGER LINE PER PROVIDER RESPONSE THIS FUNCTION READS. + + ⛔ HERE, INSIDE THE LADDER, AND NOT AT THE RETURN. The ladder tries providers in order and + a failed one has already SPENT tokens at that vendor; booking only the winner would report + a week cheaper than it was, which is the cost-surprise R13 cited. Every 200 this loop reads + gets a line, including the one whose answer turned out to be empty. + ⚠ `tokens_from` returns `(None, None)` when the provider did not say, and `None` is carried + through rather than coerced to 0 — a call booked at zero is an unmeasured call presented as + a free one. `usage_ledger` counts it as UNMEASURED so the total reads as a floor. + ⚠ `record` never raises, by its own contract, so this cannot break the assistant. + """ + ins, outs = usage_ledger.tokens_from(body) + usage_ledger.record( + "assistant", provider["name"], provider["model"], ins, outs, + total=usage_ledger.total_from(body), + st=getattr(session, "runtime", None), user=getattr(session, "uname", "")) + last = None for provider in providers: try: @@ -417,7 +449,12 @@ def _call_model(question, snapshot, model=MODEL_AUTO, chat=None, history=None): last = f"{provider['name']}: HTTP {response.status_code}" continue try: - answer = response.json()["choices"][0]["message"] + body = response.json() + # C7: booked from the RAW body, before anything below can raise on its shape. A + # provider that answered 200 and billed for it has spent tokens whether or not this + # function can read what it said. + _book(provider, body) + answer = body["choices"][0]["message"] said = _said(answer.get("content")) calls = answer.get("tool_calls") or [] if calls: @@ -667,6 +704,11 @@ def _citation_complete(citation): def _public_view(view): source = view["source"] + # ⭐⭐ W35-T25 · CONTRACT C4 — `edited` IS DERIVED, NEVER STORED, and that is deliberate. + # A stored boolean beside two specs is a third source of truth that can disagree with both; + # the only honest answer to "has this been changed" is "compare it". It also means a Revert + # that restores the spec clears the badge by construction rather than by remembering to. + original = view.get("original_spec") return { "id": view["id"], "viewId": view["id"], "scope": source["database"], "name": view["name"], "description": str(view.get("description") or ""), @@ -674,6 +716,12 @@ def _public_view(view): "explain": view["explain"], "threadId": view["threadId"], "createdAt": view["createdAt"], "virtual": True, "source": _safe(source), "view": _safe(view["view"]), "citationIds": list(view["citationIds"]), "numeric": _safe(view["numeric"]), + # ⚠ AN ARTEFACT MADE BEFORE THIS WAVE HAS NO ORIGINAL, so it reports `edited: false` and + # sends no `original_spec` — and the client shows neither the badge nor Revert. C4 is + # explicit that this is the right answer: a Revert with nothing to revert to is worse + # than an absent one, and it is the case an NC in `verify_query` covers. + "original_spec": _safe(original) if isinstance(original, dict) else None, + "edited": bool(isinstance(original, dict) and _safe(view["view"]) != _safe(original)), } @@ -744,14 +792,88 @@ def list_queries(session: Session = Depends(require_session)): return _public_state(_state(session), session) +# ⭐⭐ W35-T25 · CONTRACT C4 (owner item 4 / R3) — WHICH SPEC MEMBERS AN EDIT MAY MOVE. +# +# ⛔ AN ALLOW-LIST, AND IT IS THE SECURITY BOUNDARY OF THIS WHOLE TICKET. The client sends a +# `SavedView`, a shape it composes itself, and merging it wholesale would let a caller rewrite +# `kind` (which decides the renderer and the citation's own claim), `aggregation` (the number this +# artefact CITED) or `name`. R3 opens the view SPEC: *"filters, sort, group, visible columns, +# widths"*, plus the row height and column order that carry them. Nothing else. +# +# ⚠ `aggregation` IS DELIBERATELY ABSENT even though a person can change it on an ordinary grid. +# The artefact's citation records `contributing_record_count` and an op computed from THIS +# aggregation; letting an edit move it would leave a cited number describing a calculation the +# artefact no longer performs, which is the one thing every provenance rule in this module exists +# to prevent. +# +# ⚠ A COMMENT, NOT A BARE MODULE-LEVEL STRING. `verify_prose` reads a free-floating string literal +# as candidate copy, so writing this as a `"""..."""` above the constant put an em dash into the +# gate's own census as a 296th finding. A `#` block is out of scope by construction. +QUERY_EDITABLE_SPEC = ("visible", "order", "widths", "filters", "filterConj", "sorts", + "groupBy", "rowHeightMode", "frozenCount", "colorBy", "display") + + +def _clean_spec_edit(config, spec, fields): + """Merge a client view config onto the artefact's spec, cleaned against ITS OWN fields. + + ⛔ THE FIELD KEYS COME FROM THE STORED SOURCE, never from the request. The artefact carries + the snapshot's field list, so this needs no database read and cannot be widened by a caller + naming a column the snapshot did not have. + """ + keys = {str(field.get("key")) for field in (fields or []) if isinstance(field, dict)} + out = copy.deepcopy(spec) if isinstance(spec, dict) else {} + if not isinstance(config, dict): + return out + for member in QUERY_EDITABLE_SPEC: + if member not in config: + continue + value = config[member] + if member in ("visible", "order"): + cleaned = [str(key) for key in value if str(key) in keys] if isinstance(value, list) else [] + # ⚠ An EMPTY visible list is refused rather than stored: `_validate` already treats + # "no columns" as a refusal at creation, and a grid showing nothing is not an edit + # somebody meant to make. + if member == "visible" and not cleaned: + continue + out[member] = cleaned[:MAX_VISIBLE] + elif member == "widths": + out[member] = {str(key): int(width) for key, width in value.items() + if str(key) in keys and isinstance(width, (int, float)) + and 0 < float(width) <= 2000} if isinstance(value, dict) else {} + elif member == "filters": + out[member] = _grid().clean_filter_tree( + [item for item in value if isinstance(item, dict)], keys) if isinstance(value, list) else [] + elif member == "filterConj": + out[member] = "or" if value == "or" else "and" + elif member == "sorts": + out[member] = [{"colId": item["colId"], "dir": "desc" if item.get("dir") == "desc" else "asc"} + for item in value if isinstance(item, dict) and item.get("colId") in keys][:3] \ + if isinstance(value, list) else [] + elif member in ("groupBy", "colorBy"): + out[member] = value if value in keys else None + elif member == "rowHeightMode": + out[member] = value if value in ("short", "medium", "tall", "extra") else None + elif member == "frozenCount": + out[member] = max(0, min(6, int(value))) if isinstance(value, (int, float)) else 0 + elif member == "display": + out[member] = _grid()._clean_display(value, keys) if isinstance(value, dict) else out.get("display") + return out + + @router.post("/query/{qid}/events") def mutate_query_workspace(qid: str, body: dict = Body(default=None), session: Session = Depends(require_session)): """The only grid-mutation transport for a virtual Query workspace. - Query artefacts are immutable snapshots: creation and changes belong to an Assistant prompt, - never a source-native grid workspace. Deleting the caller's personal artefact is the one - permitted mutation. The binding key, not a client-supplied source scope, identifies it. + ⭐⭐ W35-T25 (owner item 4 / R2) — `view_upsert` ON THE ARTEFACT ITSELF IS NOW ACCEPTED. It + used to 409 `query_workspace_immutable` for everything but a delete, on the reading that an AI + artefact is a snapshot. R2 replaces that reading: the SPEC is the reader's to shape, the + CITATION and the numbers behind it are not. `QUERY_EDITABLE_SPEC` above is where that line is + drawn, and `original_spec` is what makes the change undoable. + + Creating a SECOND view inside the workspace is still refused — an artefact holds exactly one — + and deleting the caller's personal artefact still removes it. The binding key, not a + client-supplied source scope, identifies it. """ qid = str(qid) body = body if isinstance(body, dict) else {} @@ -772,9 +894,40 @@ def mutate_query_workspace(qid: str, body: dict = Body(default=None), raise err(404, "unknown_query", "that Query artefact does not exist") event_type = str(event["type"]) + if event_type == "view_upsert": + sent = event.get("view") + sent = sent if isinstance(sent, dict) else {} + # ⛔ THE ID IS CHECKED THE SAME WAY THE DELETE'S IS. An upsert naming a different view is + # a create wearing an update's name, and a Query workspace holds exactly one view. + if str(sent.get("id") or "") != qid: + raise err(400, "query_workspace_mismatch", + "a Query view edit must name the same virtual artefact as its binding") + spec = _clean_spec_edit(sent.get("config"), view.get("view"), (view.get("source") or {}).get("fields")) + + def apply_edit(raw): + current = copy.deepcopy(raw) if isinstance(raw, dict) else _blank_state() + row = (current.get("views") or {}).get(qid) + if not isinstance(row, dict): + return current + # ⚠ BACKFILLED HERE, and only when absent: an artefact created before this wave has no + # original, and the FIRST edit is the last moment its pre-edit spec still exists. Not + # backfilling would leave it permanently unrevertable; backfilling unconditionally + # would make Revert restore the latest edit. + if not isinstance(row.get("original_spec"), dict): + row["original_spec"] = _safe(row.get("view")) + row["view"] = _safe(spec) + current["views"][qid] = row + return current + + if not session.runtime.available(): + raise err(503, "store_unavailable", "the tenant store is unavailable; nothing was saved") + session.runtime.update(_namespace_key(session), apply_edit, flush="async") + return {"workspaceBinding": {"kind": "query", "key": qid}, "event": event_type, + "view": _public_view(apply_edit(_state(session))["views"][qid])} + if event_type != "view_delete": raise err(409, "query_workspace_immutable", - "AI-created Query views change only through a new Assistant prompt") + "a Query workspace holds one view, so it cannot take another") if str(event.get("viewId") or "") != qid: raise err(400, "query_workspace_mismatch", "a Query delete must name the same virtual artefact as its binding") @@ -784,6 +937,43 @@ def mutate_query_workspace(qid: str, body: dict = Body(default=None), "event": event_type, **deleted} +@router.post("/query/{qid}/revert") +def revert_query(qid: str, session: Session = Depends(require_session)): + """⭐⭐ CONTRACT C4 (R3) — restore the AI's original SPEC, and only the spec. + + ⛔ WHAT THIS DOES NOT DO, which the client's confirm says out loud BEFORE it acts: it does not + undo anything the reader changed in the SOURCE database. A Query view is live now, so a cell + edit made through it is a real write to a real record — reverting a view's filters cannot and + must not walk those back. R3 is explicit that the dialog states this before it acts, because a + Revert that silently leaves data changed is worse than one that never offered. + + ⚠ 409, not 404, when there is no original: the artefact exists and is readable, and the caller + asked for something that does not exist FOR IT. A 404 would say the artefact is gone. + """ + qid = str(qid) + # ⚠ `_artifact_or_404` returns `(state, row)`, not the row. Read it as a pair. + _state_now, view = _artifact_or_404(session, qid) + original = view.get("original_spec") + if not isinstance(original, dict): + raise err(409, "no_original_spec", + "this view was created before the assistant kept an original, so there is " + "nothing to revert to") + + def restore(raw): + current = copy.deepcopy(raw) if isinstance(raw, dict) else _blank_state() + row = (current.get("views") or {}).get(qid) + if not isinstance(row, dict): + return current + row["view"] = _safe(row.get("original_spec")) + current["views"][qid] = row + return current + + if not session.runtime.available(): + raise err(503, "store_unavailable", "the tenant store is unavailable; nothing was reverted") + session.runtime.update(_namespace_key(session), restore, flush="async") + return _public_view(restore(_state(session))["views"][qid]) + + def _sources(body, target, session): raw = body.get("sources") if isinstance(body, dict) else None sources = [str(item).strip() for item in raw] if isinstance(raw, list) else [] @@ -826,8 +1016,10 @@ def _submit(body, session, chat=None): assistant_message_id = _new_id("message") # R16: the turns already on screen go with the question. `state` was read above, BEFORE this # turn's own messages exist, so the replay is strictly the prior conversation. + # ⭐ C7/T31: `session` rides along so the meter can attribute this call. See `_call_model`'s + # docstring for why it is an argument and not ambient state (PRD amendment A1). spec, said, provider, reason = _call_model(question, snapshot, model=selected_model, chat=chat, - history=_history(state, thread_id)) + history=_history(state, thread_id), session=session) view, refusal, refusal_code = _validate(spec, snapshot["fields"]) if spec is not None else (None, said, reason) artifact = None @@ -850,6 +1042,15 @@ def _submit(body, session, chat=None): "question": question, "explain": _explain(view, snapshot["fields"]), "createdAt": now, "source": source, "view": _safe(view), "citationIds": [citation_id], "numeric": numeric, + # ⭐⭐ W35-T25 · CONTRACT C4 (R3) — THE AI'S OWN SPEC, WRITTEN ONCE, HERE. + # R2 makes a Query view editable, so `view` moves from now on. This is the copy + # "Revert to AI original" restores, and the thing `edited` is measured against. + # ⛔ WRITTEN AT CREATE AND NOWHERE ELSE. Re-stamping it on any later write would + # make Revert restore the most recent edit — a Revert that reverts to nothing, + # which C4 names as worse than no Revert at all. + # ⚠ `_safe(view)` twice, not the same object twice: `view` is mutable and a + # shared reference would let an edit rewrite the original through the alias. + "original_spec": _safe(view), "model": provider, "requestedModel": selected_model} assistant_message = { diff --git a/api/routes_starred.py b/api/routes_starred.py new file mode 100644 index 0000000000000000000000000000000000000000..c017e94c6788da13f98e01f1246786ea172a9c88 --- /dev/null +++ b/api/routes_starred.py @@ -0,0 +1,828 @@ +"""routes_starred.py — WAVE 35 (rulings R4/R5/R6/R7, contracts C2/C3/C5): THE STAR. + +Owner items 8, 9 and 12. **Star and "mark important" are ONE idea and ONE flag** (R4): on screen +the word is Star, and starring an object lands it on Home and in the new Starred module. Four +kinds are starrable — `database`, `agent`, `query`, `view` — and RECORDS get their own door +(C5, `/starred/records`), because a record's identity is `(database, pid)` rather than a bare id. + +⛔⛔ THE ONE DESIGN RULE THAT DECIDES EVERYTHING BELOW: **A VIEW'S STAR IS NOT STORED HERE.** +C2 is explicit — `kind: "view"` writes the EXISTING `config.important` in that database's own +`_table_workspace`, and never a second store. R4 says star and mark-important are one flag; +two stores for one flag is two flags that can disagree, and the badge inside the database and the +row on Home would then answer differently about the same view. So this file keeps THREE id lists +(`databases`, `agents`, `queries`) in its own per-user bucket, and for views it is a READER and a +WRITER of somebody else's bucket rather than an owner of anything. + +⚠ WHICH IS WHY `GET /starred` IS NOT FREE, AND EVERY CALLER MUST KNOW. Finding the starred views +means reading each visible database's view bucket — the same N reads `routes_nav._important_counts` +did before W35-T43 deleted it. Measured on tenant #0 (2026-08-16, recorded on that constant): +twelve buckets cost **6.2 ms WARM in total** and **7,331 ms of first fetch COLD**. So the scan is +budgeted, it is spent in the caller's own RECENTS order (a mark lives on a database somebody works +in), and a database it could not reach is named in `degraded` rather than quietly carrying no star. +That is R6's second sentence applied to a list: a limit that cannot be removed is REPORTED with its +cause, never silently enforced. **Call this AFTER first paint.** + +⚠ NO INDEX, DELIBERATELY. The obvious fix for the scan is a `starred.views` id list written beside +the three above. That is the second store C2 forbids: the moment the flag and the index can +disagree — a view deleted, a view unmarked from inside the database, a store write that lands on one +and not the other — the product has two answers to one question and no way to tell which is right. +The scan is the price of one flag, and it is paid off the critical path. +""" +import time + +from fastapi import APIRouter, Body, Depends + +import routes_nav +from deps import Session, err, require_session + +router = APIRouter(prefix="/api/v1") + +#: The per-user bucket holding the three ID-LIST kinds. `{username: {databases, agents, queries}}` +#: +#: ⚠ PER USER (R6), like `nav_recents` and unlike `nav_meta`: what I care about is not a fact about +#: the object, so a tenant-wide star would put one person's shortlist on everybody's Home. +_STARRED_KEY = "starred" + +#: The four kinds C2 declares. `view` is the odd one out — see the header. +KINDS = ("database", "agent", "query", "view") + +#: The three kinds this file's own bucket holds, and the payload key each answers under. +_LIST_KINDS = {"database": "databases", "agent": "agents", "query": "queries"} + +#: A bound on each id list. ⛔ REPORTED, NEVER SILENT: a POST that would cross it is refused with +#: the cause and the action (unstar something), rather than being accepted and truncated. A star +#: silently dropped is a star the user believes they set. +MAX_PER_KIND = 200 + +#: An id is a store key, a view id or an automation id — all of which are short. Bounded so a +#: hostile client cannot grow one user's document with one very long string. +_MAX_ID = 120 + +#: The wall-clock ceiling on the WHOLE view scan, checked between databases — the same number and +#: the same reason as the `_IMPORTANT_BUDGET_S` this replaces. It bounds the FIRST request after a +#: container starts; warm it is not reached. +VIEW_SCAN_BUDGET_S = 2.5 + + +def _clean_id(raw): + """One id, or "" — the only shape this file stores.""" + return str(raw or "").strip()[:_MAX_ID] + + +def _clean_list(raw): + """A stored id list, validated on the way OUT as well as in. + + Re-validating a read is not redundant: the bucket outlives this code and an older build (or a + hand edit) could have written a shape this one does not accept. Prune, never invent — the + posture `_clean_nav_prefs` takes one file over. + """ + if not isinstance(raw, list): + return [] + out, seen = [], set() + for item in raw[:MAX_PER_KIND]: + got = _clean_id(item) + if got and got not in seen: + seen.add(got) + out.append(got) + return out + + +def _read_lists(session): + """This caller's three id lists. A store blip answers empty rather than taking the page down.""" + try: + mine = (session.runtime.get(_STARRED_KEY) or {}).get(session.uname) or {} + except Exception: + return {name: [] for name in _LIST_KINDS.values()} + if not isinstance(mine, dict): + return {name: [] for name in _LIST_KINDS.values()} + return {name: _clean_list(mine.get(name)) for name in _LIST_KINDS.values()} + + +def _database_keys(session): + """`(keys, enumerated)` — every database this session may open that has a view bucket. + + ⚠ `routes_nav._visible_database_keys` IS REUSED RATHER THAN RE-DERIVED, and that is not + tidiness: it merges the compiled registry (behind the account grant AND the tenant catalogue) + with `user_tables.nav_entries`, the `may_open`-filtered listing over the rows-free projection. + Writing a second enumeration here is how the folder-drop defect happened (wave 27, item 6): one + door knew about `ut_*` keys and the other did not, and the disagreeing door answered 200 while + storing nothing. + ⛔ AND IT IS NOT `_placeable_top_keys`, WHICH NEVER APPLIES THE TENANT CATALOGUE — right for a + cosmetic folder placement, wrong for a scan that then opens each database's view bucket. See + that helper's own note for the second difference (module surfaces have no views to star). + """ + return routes_nav._visible_database_keys(session) + + +def _scan_order(session, keys): + """`keys`, RECENTS-FIRST — the order the scan budget is spent in. + + ⛔ THE ORDER IS LOAD-BEARING AND W34-T10 LEARNED IT THE EXPENSIVE WAY. A budget spent in + whatever order the keys happen to arrive is a lottery: measured cold on tenant #0, that block + spent all of its first draft's budget on ten EMPTY Odoo buckets and was cut off two rows before + `customer_data`, which holds the only marked view in the tenant. The feature would have been + correct and shown nothing. A star lives on a database somebody works in, and `nav_recents` is + exactly that list, newest first. + """ + recents = routes_nav._read_recents(session.runtime, session.uname, keys) + order = [r["key"] for r in recents] + seen = set(order) + order += [k for k in sorted(keys) if k not in seen] + return order + + +def _view_docs(session, order): + """`({key: workspace_doc}, unread)` — each database's view bucket, PROJECTED, within budget. + + ⛔ PROJECTED, and the two dropped keys are what makes this affordable: `overlays` is per-record + cell values and `fields` is the schema stratum, neither of which says anything about a view. On + `customer_data` they are most of the bucket. + ⚠ `unread` is a store blip OR the budget — reported, never a confident absence. + """ + import core.view_templates as view_templates + docs, unread = {}, [] + started = time.perf_counter() + for key in order: + ws_key = view_templates.workspace_key(key) + if not ws_key: + continue # a module surface or a folder head, not a table at all + if time.perf_counter() - started > VIEW_SCAN_BUDGET_S: + unread.append(key) + continue + try: + docs[key] = session.runtime.get_projection(ws_key, + drop=("overlays", "fields")) or {} + except Exception: + unread.append(key) + return docs, unread + + +def _refresh_one(session, cache, key): + """Re-read ONE database's bucket inside a cache the caller already holds. + + ⛔ THIS IS WHY A WRITE MAY REUSE A SCAN AT ALL. The star write changes exactly one bucket, so + every OTHER document in the cache is still true — but the one just written is stale by + definition, and serving it back would answer the POST with the flag's OLD value. That is a lost + write wearing the face of a failed read ([[refetch-eats-its-own-write]] in reverse), and it is + the trap that makes "just reuse the cache" wrong without this function. + """ + import core.view_templates as view_templates + order, docs, unread = cache + ws_key = view_templates.workspace_key(key) + if not ws_key: + return cache + try: + docs[key] = session.runtime.get_projection(ws_key, drop=("overlays", "fields")) or {} + except Exception: + docs.pop(key, None) + if key not in unread: + unread.append(key) + return (order, docs, unread) + + +def _scan_starred_views(session, keys=None, cache=None): + """`(rows, unread)` — every view this caller has starred, newest-worked database first. + + A row is `{"id", "database", "name", "config"}`. ⚠ `config` is carried so the COUNTS route does + not have to read every bucket a second time: `_view_docs` is the expensive half of this file + (7,331 ms cold for twelve buckets) and two consumers asking the same question twice per request + is the 1+N shape D-175 spent a wave removing from `/nav`. `_starred_views` below drops it for the + wire. + + ⛔ `cache` IS `(order, docs, unread)` FROM AN EARLIER PASS IN THE SAME REQUEST, and it exists + because the view-star WRITE otherwise scanned every bucket TWICE: once in `_find_view` to locate + the view, then again to build the answer. That is the same 1+N shape, in the same file, on the + same expensive read — introduced an hour after it was removed one function away. A caller that + has written must `_refresh_one` the database it wrote before passing its cache. + """ + if cache is not None: + order, docs, unread = cache + else: + if keys is None: + keys, _ = _database_keys(session) + order = _scan_order(session, keys) + docs, unread = _view_docs(session, order) + granted = routes_nav._granted_view_ids(session) + out = [] + for key in order: + doc = docs.get(key) + if doc is None: + continue + for vid, view in routes_nav._visible_views(doc, session.uname, session.admin, + granted).items(): + cfg = view.get("config") or {} + if cfg.get("important") is True: + out.append({"id": vid, "database": key, + "name": str(view.get("name") or vid)[:120], "config": cfg}) + return out, unread + + +def _starred_views(session, keys=None, cache=None): + """The C2 wire rows — `{"id", "database", "name"}`, with the config dropped. + + ⭐ `name` is a SUPERSET of C2, published to B as `NOTE E-1`: the scan already holds the record, + so the name is free, and without it Home would render a starred view as a bare id. + ⛔ THE CONFIG DOES NOT GO ON THE WIRE. It carries `memberPids` (a curated view's whole pid list) + and every filter leaf, which is somebody's row set travelling to a client that asked for a label. + """ + rows, unread = _scan_starred_views(session, keys, cache=cache) + return ([{"id": r["id"], "database": r["database"], "name": r["name"]} for r in rows], + unread) + + +def _query_names(session, qids): + """`({qid: name}, resolved)` for the caller's own Query artefacts. + + ⭐ ANSWERS `ASK B-1 (2)`. C2 declares `queries: [qid]` and there is no view/query id -> label map + anywhere in the client, so Home and Starred would render a starred Query as a bare id. B's only + alternative was `GET /query`, which drags threads, messages, citations and every saved view + across the wire to render two strings. + + ⚠ IT READS LANE C's BUCKET AND CALLS LANE C's HELPERS, WITHOUT EDITING LANE C's FILE. Query + state is ONE small per-user document (`query_user_`: threads, messages, citations, views — + no grid rows), and `_namespace_key`/`_state` are pure functions of the session. Re-deriving the + key here would be a second spelling of a hash, which is the one thing guaranteed to drift. + + ⛔ `resolved` IS FALSE WHEN THE READ FAILED, and the caller must not prune on it — a store blip + would otherwise delete somebody's whole starred-query list. Same posture as + `_clean_nav_prefs(keep_unknown_ut=...)`. + """ + if not qids: + return {}, True + try: + import routes_query + views = (routes_query._state(session) or {}).get("views") or {} + except Exception: + return {}, False + out = {} + for qid in qids: + row = views.get(qid) + if isinstance(row, dict): + out[qid] = str(row.get("name") or qid)[:120] + return out, True + + +def _payload(session, keys=None, cache=None): + """The C2 answer: the three id lists, the starred views, and what could not be read. + + ⚠ `degraded` SHIPS AS `[]` RATHER THAN BEING OMITTED WHEN EMPTY — the same rule `/nav`'s + `omitted`/`degraded` follow, for the same reason: a key a consumer has to test for is a key a + consumer forgets to test for. + ⚠ `cache` is a scan an earlier pass in THIS request already paid for — see `_scan_starred_views`. + """ + lists = _read_lists(session) + if keys is None: + keys, _ = _database_keys(session) + # ⚠ PRUNED ON READ, NEVER ON WRITE — `_read_recents`' posture, and both halves are deliberate. + # A database whose grant was revoked must stop being offered without anybody running a + # migration, and must come back if the grant does. A key that is not visible therefore reaches + # the store and never reaches a screen. + # ⚠ ONLY `databases` is pruned. An agent id or a query id would each cost another store read to + # validate, and pruning them here buys nothing: this list is the CALLER'S OWN writes, so it can + # disclose nothing they did not already know, and B/C render only objects they can already see + # (the way `groupRecents` drops an unreachable recent). Said out loud rather than left as a gap. + lists["databases"] = [k for k in lists["databases"] if k in keys] + # ⭐ ASK B-1 (2): the names, and — because the read that answers "what is it called" also answers + # "does it still exist" — the same prune the databases get. ⛔ ONLY WHEN THE READ SUCCEEDED: a + # store blip that returned no artefacts must not delete the whole starred-query list. + qnames, qresolved = _query_names(session, lists["queries"]) + if qresolved: + lists["queries"] = [q for q in lists["queries"] if q in qnames] + views, unread = _starred_views(session, keys, cache=cache) + out = {"databases": lists["databases"], "agents": lists["agents"], + "queries": lists["queries"], "queryNames": qnames, + "views": views, "degraded": unread} + if unread: + # R6's second sentence, in the payload: the CAUSE and what it means, not a bare list. + out["note"] = (f"{len(unread)} database(s) could not be read in time, so a view you " + f"starred there is missing from this list. Open one of them once and the " + f"next load will include it") + return out + + +@router.get("/starred") +def starred(session: Session = Depends(require_session)): + """C2 — everything this caller has starred. + + ⚠ NOT ON `/nav`'s PATH, and R7 is the ruling: `/nav` is a rows-free projection whose budget was + already the subject of D-175/D-185/D-288. This route is called AFTER Home/Starred paint. + """ + return _payload(session) + + +def _find_view(session, vid, database=""): + """`(key, ws_key, stratum, view, cache)` for a view this caller can SEE, else None. + + `stratum` is the username whose block holds it, or `table_store.SHARED_KEY`. `cache` is the + `(order, docs, unread)` this call paid for — but **only when it scanned everything**. With a + `database` hint it read ONE bucket, which is not a complete answer for the caller's whole + starred list, so it hands back None and the caller pays for a full pass. Handing back a partial + cache would silently drop every starred view on every OTHER database. + + ⚠ `database` IS AN OPTIONAL FAST PATH (a superset of C2, published to B and C): the client + almost always knows which database the view belongs to, and naming it turns an N-bucket scan + into one read. Absent, the scan runs — because C2's body carries only `kind`/`id`/`on`, and a + door that REQUIRED the hint would break the contract two lanes are building against. + """ + import core.table_store as table_store + import core.view_templates as view_templates + + keys, _ = _database_keys(session) + if database: + # An unknown or invisible key must not become a wider scan. It answers "not found", which + # is the same answer an id nobody holds gets — see the write door's note on why. + if database not in keys: + return None + order, cache = [database], None + else: + order = _scan_order(session, keys) + docs, unread = _view_docs(session, order) + if not database: + cache = (order, docs, unread) + granted = routes_nav._granted_view_ids(session) + for key in order: + doc = docs.get(key) + if doc is None: + continue + if str(vid) not in routes_nav._visible_views(doc, session.uname, session.admin, granted): + continue + ws_key = view_templates.workspace_key(key) + # WHICH stratum holds it decides the WALL, so it is resolved here rather than guessed. + # Personal first: a caller's own block is the common case and needs no grant. + if isinstance(doc.get(session.uname), dict) and \ + str(vid) in ((doc[session.uname].get("views") or {})): + return (key, ws_key, session.uname, doc[session.uname]["views"][str(vid)], cache) + shared = (doc.get(table_store.SHARED_KEY) or {}).get("views") or {} + if str(vid) in shared: + return (key, ws_key, table_store.SHARED_KEY, shared[str(vid)], cache) + for stratum, blob in doc.items(): + if stratum == session.uname or not isinstance(blob, dict): + continue + if str(vid) in (blob.get("views") or {}): + return (key, ws_key, str(stratum), blob["views"][str(vid)], cache) + return None + + +def _may_star_view(session, stratum, view): + """May this caller WRITE the star on this view? Fail-closed. + + ⛔ A VIEW ID THE CALLER CANNOT WRITE MUST REFUSE **LOUDLY**, and this is a standing scar: + `view_upsert` answers **200 while writing nothing** when the id belongs to a view the caller + cannot see, so a pinned id is effectively tenant-scoped and every symptom of it is silent + ([[a-view-id-another-user-holds-refuses-silently]]). This function is the reason the write door + below raises instead of shrugging. + + THREE HOMES, THREE WALLS: + · the caller's OWN stratum — theirs, no further question; + · `__shared__` — `table_store._may_edit`, which is the predicate the view door itself uses; + · another user's stratum, reached by a wave-21 grant — the grant must be `edit` or `owner`. + ⚠ A `view` ROLE IS NOT A WRITE GRANT. `_granted_view_ids` admits both roles because it + answers "can they SEE it"; this answers "may they CHANGE it", and collapsing the two would + let a read-only grantee mark somebody else's shared view. + """ + import core.shares as shares + import core.table_store as table_store + if stratum == session.uname: + return True + if stratum == table_store.SHARED_KEY: + return bool(table_store._may_edit(view, session.uname, session.admin)) + return shares.role_for("view", str((view or {}).get("id") or ""), session.uname, + is_admin=session.admin, + st=session.runtime) in ("edit", "owner") + + +def _write_view_star(session, ws_key, stratum, vid, on): + """Set `config.important` IN PLACE, in the view's own bucket. Returns True when it landed. + + ⛔⛔ IN PLACE, AND **NOT** THROUGH `grid_events.view_upsert` OR `table_store.save_view` — three + separate reasons, each of which has cost this repo something: + + 1. `view_upsert` REBUILDS `config` from a literal allowlist and filters `memberPids` against + `allowed_pids` and `cohortLock` against `cohort_ids`. Routing a flag toggle through it + means a cohort view whose pool this request did not assemble comes back with its curated + row set pruned — "the 40 accounts we agreed to call" quietly becoming fewer, under the + same name, with nothing going red. + 2. `table_store.save_view` re-runs `_unique_name` against every view name in the document. + A star has no business re-allocating a name; a collision would RENAME the starred view. + 3. Both are "write the whole view" doors. This writes one boolean. + + ⚠ SET UNCONDITIONALLY, never "only when True" — `grid_events` records the same rule beside its + own `important` line: a key written only when present means unstarring is silently a no-op and + the stored `true` survives, i.e. a mark you can set and never clear. + ⚠ `bool(on)`, so a JSON `"false"` cannot become a star. + ⚠ FLUSH IS THE DEFAULT (sync). This is a deliberate one-at-a-time act like a folder rename, not + the autosave hot path `table_store` uses `async` for — and a coalesced write is exactly what + made eighteen 200s land zero rows in wave 33. + """ + landed = {"ok": False} + + def _up(data): + data = data if isinstance(data, dict) else {} + block = data.get(stratum) + if not isinstance(block, dict): + return data + views = block.get("views") + if not isinstance(views, dict) or str(vid) not in views: + return data + view = views[str(vid)] + if not isinstance(view, dict): + return data + cfg = view.get("config") + if not isinstance(cfg, dict): + cfg = {} + view["config"] = cfg + cfg["important"] = bool(on) + landed["ok"] = True + return data + + session.runtime.update(ws_key, _up) + return landed["ok"] + + +@router.post("/starred") +def set_starred(body: dict = Body(default=None), + session: Session = Depends(require_session)): + """C2 — star or unstar ONE object, and answer with the caller's whole new list. + + Body: `{"kind": "database"|"agent"|"query"|"view", "id": str, "on": bool}`, plus an OPTIONAL + `"database"` when `kind` is `view` (the fast path — see `_find_view`). + + ⚠ THE WHOLE LIST COMES BACK, not an ack, because the client is optimistic-then-reconciled + (T15): an ack would leave the browser as the only place that knows what the new state is. + """ + body = body if isinstance(body, dict) else {} + kind = str(body.get("kind") or "").strip() + oid = _clean_id(body.get("id")) + if kind not in KINDS: + raise err(400, "bad_request", + f"kind must be one of {', '.join(KINDS)}") + if not oid: + raise err(400, "bad_request", "no object was named") + if "on" not in body or not isinstance(body.get("on"), bool): + # Explicit rather than defaulted: a toggle whose default is "star" turns a malformed + # unstar into a star, which is the one direction a user cannot undo by repeating it. + raise err(400, "bad_request", "`on` must be true or false") + on = bool(body["on"]) + if not session.runtime.available(): + raise err(503, "store_unavailable", + "the tenant store is unavailable. Nothing was saved.") + + if kind == "view": + found = _find_view(session, oid, _clean_id(body.get("database"))) + if not found: + # ⛔ 404 FOR BOTH "no such view" AND "not yours to see", deliberately: a distinct answer + # would turn this route into an oracle that sorts real view ids from invented ones. + raise err(404, "unknown_view", "that view is not available") + db_key, ws_key, stratum, view, cache = found + if not _may_star_view(session, stratum, view): + raise err(403, "forbidden", + "that view belongs to somebody else and you have read access only") + try: + landed = _write_view_star(session, ws_key, stratum, oid, on) + except Exception: + raise err(503, "store_unavailable", + "the star was not saved: the store refused the write.") + if not landed: + # The view moved between the read and the write (deleted, shared, unshared). Loud, + # because a 200 over a write that changed nothing is the defect this file's own + # `_may_star_view` note is about. + raise err(409, "view_moved", + "that view changed while the star was being saved. Try again") + # ⛔ REUSE THE SCAN, REFRESH THE ONE BUCKET WE WROTE. Without this the POST scans every view + # bucket TWICE — once to find the view, once to build the answer — which is the 1+N shape + # this file removed from `/starred/counts` an hour earlier, reintroduced by the write path. + # ⚠ `_refresh_one` is not optional: the written document is stale by definition and serving + # it back would answer the POST with the flag's OLD value. + if cache is not None: + cache = _refresh_one(session, cache, db_key) + return _payload(session, cache=cache) + + field = _LIST_KINDS[kind] + uname = session.uname + # ⛔ THE CAP IS ENFORCED INSIDE THE TRANSACTION, NOT BY A READ BEFORE IT, AND THAT IS NOT + # PEDANTRY. A pre-flight `len(current) >= MAX_PER_KIND` check is advisory: two concurrent stars + # both pass it, both append, and the document ends up at MAX+1 — where `_clean_list`'s + # `raw[:MAX_PER_KIND]` then drops one **on the next read**. That is a star the user set, saw + # confirmed, and cannot find, which is exactly the silent truncation R6's second sentence + # forbids. Refusing inside the read-modify-write makes the ceiling true rather than likely. + # ⚠ IT RECORDS THE REFUSAL RATHER THAN RAISING FROM INSIDE `_up`: an exception thrown through + # `store.update` would surface as this route's own 503 "the store refused the write", which is a + # different and wrong story about a limit the caller can act on. + refused = {"full": False} + + def _up(data): + data = data if isinstance(data, dict) else {} + mine = dict(data.get(uname) or {}) if isinstance(data.get(uname), dict) else {} + ids = _clean_list(mine.get(field)) + if on: + if oid not in ids: + if len(ids) >= MAX_PER_KIND: + refused["full"] = True + return data + ids.append(oid) + else: + ids = [i for i in ids if i != oid] + if ids: + mine[field] = ids + else: + mine.pop(field, None) + # A user with nothing starred is REMOVED rather than stored empty — a bucket that + # accumulates `{}` per account is a document that grows forever and says nothing. + if mine: + data[uname] = mine + else: + data.pop(uname, None) + return data + + try: + session.runtime.update(_STARRED_KEY, _up) + except Exception: + raise err(503, "store_unavailable", + "the star was not saved: the store refused the write.") + if refused["full"]: + # R6's second sentence: the cause and the action, never a 200 over a star that was dropped. + raise err(400, "starred_full", + f"you have starred the maximum of {MAX_PER_KIND} items of this kind. " + f"Unstar one to make room") + return _payload(session) + + +# ══════════════════════════════════════════════════════════════════ STARRED COUNTS (R7, C3) +# +# ⭐ R7: *"Row counts come from a NEW on-demand endpoint called after Home/Starred paint, never from +# `/nav`. A view whose count is not free shows the CAUSE, never an invented number."* +# +# ⛔ SO EVERY VID LANDS IN EXACTLY ONE OF `counts` AND `notes`, AND THAT IS THE CONTRACT (C3). A vid +# in neither is a client rendering nothing with no idea why; a vid in both is two answers to one +# question. `notes` is not a fallback for "we did not get round to it" — it is the CAUSE, in a +# sentence, with what to do about it, which is R6's second sentence applied to a badge. + + +def _cohort_sizes(session, key, cache): + """`{cohort_id: size}` for THIS caller's own cohorts on `key`'s topic. Memoised per request. + + ⛔ THIS CALLER'S OWN COHORTS ONLY, and the consequence is deliberate: a SHARED view locked to a + cohort somebody else owns finds no id here, so it is reported UNCOUNTED with that cause rather + than counted out of a stratum this session cannot see. Widening the read would make the number + disclose the SIZE of another person's private list, which is a leak wearing a bug fix. + ⚠ Read through `session.runtime`, NOT `modules.cohort`'s module-level helpers: those call + `core.store` directly and carry no tenant namespace. The MODULE is asked for the bucket NAME (it + owns that rule) and this route does the reading, which is the only tenant-correct combination. + """ + import core.view_templates as view_templates + import modules.cohort as cohort_mod + ws_key = view_templates.workspace_key(key) or "" + suffix = "_table_workspace" + scope = ws_key[:-len(suffix)] if ws_key.endswith(suffix) else key + if scope in cache: + return cache[scope] + try: + bucket = session.runtime.get(cohort_mod.key_for(scope)) or {} + mine = bucket.get(session.uname) or {} + cache[scope] = {str(cid): len(c.get("members") or []) + for cid, c in mine.items() if isinstance(c, dict)} + except Exception: + cache[scope] = {} + return cache[scope] + + +def _count_note(cfg, cohorts): + """WHY this view's size is not free, as one sentence naming a cause AND a next step. + + ⛔ R6's SECOND SENTENCE IS THE SPEC HERE, and a bare dash is the thing it forbids. D-253 was + booked once already for a limit whose explanation was hover-only, so this has to be a real + sentence a client can put in a `title` AND an `aria-label`. + ⚠ THREE DIFFERENT CAUSES, not one generic apology. A view locked to somebody else's list, a + filtered view and a whole-table view are three different facts, and only one of them is + something the reader can act on. + """ + cfg = cfg if isinstance(cfg, dict) else {} + lock = str(cfg.get("cohortLock") or "").strip() + if lock: + return ("This view is locked to a list owned by somebody else, so its size is not " + "something your account can read. Ask whoever shared it.") + if cfg.get("filters"): + return ("Counting a filtered view means reading every record, which this page does not " + "do. Open the database and the number is beside the view.") + return ("This view covers the whole database, so its size is the record count. Open the " + "database to see it.") + + +@router.get("/starred/counts") +def starred_counts(session: Session = Depends(require_session)): + """C3 — `{"counts": {vid: int}, "notes": {vid: str}, "degraded": [key]}`. + + ⛔ CALLED AFTER THE PAGE PAINTS, NEVER FROM `/nav` (R7). `/nav` is a rows-free projection whose + budget was the subject of D-175, D-185, D-288 and D-289; W35-T43 takes the last of that off it, + and putting a counting pass back on the nav's path would undo the whole point. + + ⚠ THE COUNTER IS `routes_nav._view_record_count`, CALLED — not a second implementation. It is the + one function that knows which shapes are free (a cohort lock resolves to a stored member list; a + curated `memberPids` carries its own length) and returns None for everything else. A second + counter here would be free to disagree with the badge inside the database. + """ + rows, unread = _scan_starred_views(session) + cache, counts, notes = {}, {}, {} + for row in rows: + cfg, key, vid = row["config"], row["database"], row["id"] + sizes = _cohort_sizes(session, key, cache) + n = routes_nav._view_record_count(cfg, sizes) + # ⛔ EXACTLY ONE OF THE TWO MAPS (C3). The `if/else` is what makes that structural rather + # than a rule somebody has to remember: there is no path that writes both and none that + # writes neither. + if isinstance(n, int): + counts[vid] = n + else: + notes[vid] = _count_note(cfg, sizes) + return {"counts": counts, "notes": notes, "degraded": unread} + + +# ══════════════════════════════════════════════════════════════════ RECORD STARS (R5/R6, C5) +# +# ⭐ R5: *"Records get their own star, feeding an undeletable 'Starred records' view beside +# 'All records' on every database."* R6: *"Record stars are PER USER, in the same per-username +# stratum views already live in."* +# +# ⛔⛔ THE ID LIST IS ITS OWN KEY IN THAT STRATUM AND IT DOES NOT TRAVEL THROUGH THE VIEW DOOR. +# D-170, measured: on a read-through grid a VIEW WRITE answers **409 `window_required`** — +# `/grid/events` -> `routes_tables.ut_write_ctx` -> `scoped_pids` -> `scoped_pool` -> +# `routes_odoo_tables.whole_pool` raises `TooBigToMaterialise`, because the shared read/write +# assembly insists on the pid set. `ut_odoo_order_lines` (255,286 rows) and `ut_odoo_gl_lines` +# (971,034) serve rows through a window correctly and refuse every view-config write. A record star +# stored inside a view record would therefore be IMPOSSIBLE on exactly the two biggest databases in +# the tenant. This writes one key in the workspace bucket and touches no pid set, so it answers 200 +# there. +# +# ⚠ AND FOR THE SAME REASON IT DOES NOT VALIDATE THE ID AGAINST THE ROW SET. Checking that a pid +# exists means materialising the pool, which is the 409 again. `/nav/opened` takes the identical +# posture and says why: the WRITE is cheap and unvalidated, the READ is what prunes — a client +# intersects `memberPids` with the rows it can see, exactly as a cohort already does +# (`aios_grid.workspace_wire` drops absent members and reports `missing`). + +#: The per-user key inside `_table_workspace`. BESIDE `views`, never inside one (R6). +_RECORDS_KEY = "starredRecords" + +#: The pinned id of the projected view. ⚠ C's rail pins this too (W35-T30) — it is a WIRE constant, +#: so it is exported rather than spelled twice. +STARRED_VIEW_ID = "starred-records" +STARRED_VIEW_NAME = "Starred records" + +#: A bound per (user, database). Reported on overflow, never silently enforced. +MAX_STARRED_RECORDS = 500 + + +def _record_ids(doc, uname): + """This caller's starred record ids on ONE database, validated on the way out. + + ⛔ INTS, AND A NON-DIGIT ID IS NOT STORED. Every record identity in this product is a digit + string — a pool `pid` is an int and a `ut_*` row key is `str(record_id).isdigit()` — and the + CLIENT's view engine reads `memberPids` as numbers (a cohort's members are ints). A mixed list + would make the projection match nothing on the rows it was built for, silently. + """ + raw = (doc.get(uname) or {}).get(_RECORDS_KEY) if isinstance(doc.get(uname), dict) else None + if not isinstance(raw, list): + return [] + out, seen = [], set() + for item in raw[:MAX_STARRED_RECORDS]: + try: + pid = int(item) + except (TypeError, ValueError): + continue + if pid not in seen: + seen.add(pid) + out.append(pid) + return out + + +def records_view(ids): + """The **"Starred records"** view, as a SavedView — or None when nothing is starred. + + ⭐ THE SERVER HANDS C THE WHOLE OBJECT (C5) rather than a bare id list, so the rail cannot grow + a second idea of what this view is. `kind: 'system'` and `locked: True` are what make it + undeletable, the same two facts that protect `all-customers`. + + ⛔ `memberPids` IS THE WHOLE MECHANISM: it makes this a CURATED ROW SET, which is the one shape + `routes_nav._view_record_count` can count for FREE — no rows read, no filter run. That is why C3 + can put a real number beside it and why R7's counts endpoint costs nothing here. + ⚠ None WHEN EMPTY, not an empty view: a view listing zero records under a name promising some is + the shape D-229 was booked for. C shows the row only when there is something in it. + """ + pids = [int(p) for p in (ids or [])] + if not pids: + return None + return { + "id": STARRED_VIEW_ID, + "name": STARRED_VIEW_NAME, + "kind": "system", + "locked": True, + "note": "The records you starred on this database. Only you can see this list.", + "config": {"filters": [], "filterConj": "and", "sorts": [], "groupBy": None, + "colorBy": None, "rowHeightMode": "short", "order": [], "visible": [], + "widths": {}, "memberPids": pids}, + } + + +def _database_or_refuse(session, key): + """The workspace bucket for a database this caller may OPEN, or a refusal. + + ⚠ THE SPLIT IS `routes_nav.nav_schema`'s, COPIED IN SHAPE RATHER THAN INVENTED: a `ut_*` + database is walled by `user_tables.may_open` (its creator or an admin or a grantee), and a + built-in module by `session.require`, which would 403 every `ut_` key because a user table is + deliberately not a module. + """ + import core.view_templates as view_templates + ws_key = view_templates.workspace_key(key) + if not ws_key: + raise err(404, "unknown_database", "that database is not available") + if key.startswith("ut_"): + import core.user_tables as user_tables + if not user_tables.may_open(key, session.uname, session.admin, st=session.runtime): + raise err(403, "forbidden", "that database belongs to another user") + else: + session.require(key) + return ws_key + + +def _records_payload(session, key, ws_key): + try: + doc = session.runtime.get_projection(ws_key, drop=("overlays", "fields")) or {} + except Exception: + doc = {} + ids = _record_ids(doc, session.uname) + return {"database": key, "ids": ids, "count": len(ids), "view": records_view(ids)} + + +@router.get("/starred/records") +def starred_records(database: str = "", session: Session = Depends(require_session)): + """C5 — the record ids THIS caller starred on ONE database, plus the view to render. + + ⚠ ONE DATABASE PER CALL, deliberately. Answering "every database" would be the N-bucket scan + the view half already pays for, on a route a grid calls every time it opens. + """ + key = _clean_id(database) + if not key: + raise err(400, "bad_request", "no database was named") + return _records_payload(session, key, _database_or_refuse(session, key)) + + +@router.post("/starred/records") +def set_starred_record(body: dict = Body(default=None), + session: Session = Depends(require_session)): + """C5 — star or unstar ONE record. Body `{"database": key, "id": str, "on": bool}`. + + ⛔ PER USER, IN THE PER-USERNAME STRATUM, AND IT MUST NEVER FALL BACK TO A TENANT-WIDE LIST + (R6). One person's stars becoming everyone's is the failure mode a shared column has here — a + custom field on these grids is written through a per-username stratum anyway, so a "shared" + answer would be a deliberate widening, not a shortcut. + """ + body = body if isinstance(body, dict) else {} + key = _clean_id(body.get("database")) + raw_id = _clean_id(body.get("id")) + if not key: + raise err(400, "bad_request", "no database was named") + if not raw_id.isdigit(): + # Named rather than coerced: see `_record_ids` on why a non-numeric id cannot be stored. + raise err(400, "bad_request", "a record id is a number") + if "on" not in body or not isinstance(body.get("on"), bool): + raise err(400, "bad_request", "`on` must be true or false") + pid, on = int(raw_id), bool(body["on"]) + ws_key = _database_or_refuse(session, key) + if not session.runtime.available(): + raise err(503, "store_unavailable", + "the tenant store is unavailable. Nothing was saved.") + current = _records_payload(session, key, ws_key)["ids"] + if on and pid not in current and len(current) >= MAX_STARRED_RECORDS: + raise err(400, "starred_full", + f"you have starred the maximum of {MAX_STARRED_RECORDS} records on this " + f"database. Unstar one to make room") + uname = session.uname + + def _up(data): + data = data if isinstance(data, dict) else {} + block = data.get(uname) + block = dict(block) if isinstance(block, dict) else {} + ids = _record_ids({uname: block}, uname) + if on: + if pid not in ids: + ids.append(pid) + else: + ids = [i for i in ids if i != pid] + if ids: + block[_RECORDS_KEY] = ids + else: + block.pop(_RECORDS_KEY, None) + # ⚠ THE STRATUM IS KEPT WHEN IT STILL HOLDS ANYTHING ELSE. Popping a username whose views + # and overlays live in the same block would delete somebody's whole workspace to clear one + # star — which is why this writes `block` back rather than replacing it. + if block: + data[uname] = block + else: + data.pop(uname, None) + return data + + try: + session.runtime.update(ws_key, _up) + except Exception: + raise err(503, "store_unavailable", + "the star was not saved: the store refused the write.") + return _records_payload(session, key, ws_key) diff --git a/api/routes_statements.py b/api/routes_statements.py index fb608f0da2cfb35c3dccd25d431893e6c8a1b141..771ac0a8a1b1326942145be58c132991bd3e66aa 100644 --- a/api/routes_statements.py +++ b/api/routes_statements.py @@ -31,6 +31,10 @@ from fastapi import APIRouter, Body, Depends from deps import Session, err from routes_admin import admin_gate +# W35-T35 (C8): the tenant predicate, held once. See `_royal_only` for why the import runs this +# way round. `main.py` already imports both modules, so this adds no load. +import automation_engine as engine + router = APIRouter(prefix="/api/v1") #: Cache the Odoo follow-up pull briefly. The Streamlit page used `@st.cache_data(ttl=1800)`; the @@ -48,8 +52,17 @@ def _royal_only(session: Session) -> Session: """⛔ See the module docstring, guardrail 3. The send client is env-credentialed, so it is tenant #0's and only tenant #0's. Refuse for anyone else rather than send as the wrong company. - Keyed on the runtime, never on a request field: a tenant is a property of the SESSION.""" - if getattr(session.runtime, "key", None) != "royal-imports": + Keyed on the runtime, never on a request field: a tenant is a property of the SESSION. + + ⭐⭐ WAVE 35 · T35 / CONTRACT C8 — THE TEST ITSELF NOW LIVES IN ONE PLACE. Statements became an + agent step this wave, so the automation engine has to answer the same question at two more + doors (which tenants see the action, which tenants may store it). C8's words are "the predicate + is imported, never re-expressed", and this is the direction that import can run: the engine + imports no route module and no FastAPI, so reaching `_royal_only` FROM it would drag `deps` and + `routes_admin` into a route-free module. The engine holds the boolean; this holds the refusal. + ⚠ WHAT STAYS HERE IS THE HTTP SHAPE, and that is deliberate — a 404 rather than a 403, so the + surface is invisible rather than forbidden. The engine must not know about status codes.""" + if not engine.is_statement_tenant(session.runtime): raise err(404, "not_found", "statements are not configured for this workspace") return session @@ -91,7 +104,9 @@ def statements(refresh: int = 0, session: Session = Depends(_gate)): "replyTo": cs.REPLY_TO, "company": cs.COMPANY}, "templates": {"subject": cs.DEFAULT_SUBJECT, "intro": cs.DEFAULT_INTRO, "footer": cs.DEFAULT_FOOTER}, - "tiers": ["A-Urgent", "B-Active", "C-Light", "Monitor"], + # W35-T35: ONE literal, read from the engine, so the sender's worklist and a statements + # agent's tier filter cannot come to mean different things. + "tiers": list(engine.STATEMENT_TIERS), } @@ -166,3 +181,89 @@ def send(body: dict = Body(default=None), session: Session = Depends(_gate)): failed.append({"customer": name, "error": str(e)[:200]}) return {"sent": sent, "failed": failed, "skipped": skipped, "safeMode": bool(cs.SAFE_MODE), "test": bool(override)} + + +# --------------------------------------------------------------------------------------------- +# WAVE 35 · T36 / CONTRACT C8 / OWNER RULING R10 — THE AGENT'S PARKED BATCH. +# +# ⛔⛔ THE AGENT ASSEMBLES; A PERSON CLICKS SEND. `automation_engine.run_statements` renders a batch +# and parks it on the automation definition, and it imports no mail path at all. THIS is the only +# door that releases one, and it is deliberately built out of the parts already here: +# · `_gate` — `admin_gate` THEN `_royal_only`, byte-for-byte the dependency the other three +# endpoints use. Not a copy of the checks: the same object. +# · `cs.queue_statement` — the same call `send()` above makes, so SAFE_MODE's allow-list refusal +# happens in the DATA LAYER, where no route, payload or UI can reach around it. +# ⇒ Nothing about the guardrails is re-expressed here, which is what makes "unchanged" checkable. +# +# ⚠ WHY THE BATCH IS RE-RENDERED FROM THE LIVE WORKLIST RATHER THAN SENT AS STORED. The parked +# `html` is what a person APPROVED and is what the review screen shows; but a balance can move +# between the parking and the click, and mailing a figure we know to be stale is worse than mailing +# a fresh one. So the parked batch decides WHO and WITH WHAT WORDS, and the live row decides the +# NUMBERS — the same split `preview` already makes. A customer who has left the worklist entirely +# (they paid) is reported as skipped rather than invoiced. + +@router.get("/admin/statements/agent/{auto_id}") +def agent_batch(auto_id: str, session: Session = Depends(_gate)): + """What is parked and waiting for a click, for ONE agent. `null` when nothing is.""" + defn = (engine.all_definitions(session.runtime) or {}).get(str(auto_id)) or {} + pending = defn.get("pendingStatements") + if not isinstance(pending, dict): + return {"pending": None} + # ⚠ The rendered `html` is NOT returned in the list — a 200-statement batch of rendered mail is + # megabytes, and the review screen needs the count and the names to decide. `preview` already + # renders ONE on demand. + return {"pending": { + "ts": pending.get("ts"), "count": int(pending.get("count") or 0), + "notes": list(pending.get("notes") or []), + "items": [{k: v for k, v in it.items() if k != "html"} + for it in (pending.get("items") or [])], + "safeMode": bool(_cs().SAFE_MODE)}} + + +@router.post("/admin/statements/agent/{auto_id}/send") +def agent_send(auto_id: str, body: dict = Body(default=None), + session: Session = Depends(_gate)): + """Release a parked batch. THE CLICK R10 REQUIRES — nothing else in the system calls this. + + ⛔ IT REFUSES AN EMPTY OR MISSING BATCH rather than answering 200 with nothing sent: a send + that reports success and mails nobody is the `view_upsert` failure mode (200 OK, zero writes) + arriving on the one route in this system that talks to a mail server. + """ + cs = _cs() + defn = (engine.all_definitions(session.runtime) or {}).get(str(auto_id)) or {} + pending = defn.get("pendingStatements") + items = list((pending or {}).get("items") or []) if isinstance(pending, dict) else [] + if not items: + raise err(404, "no_batch", "there is nothing parked for this agent to send") + # ⚠ An explicit subset is allowed (a person unticking a customer on the review screen) but it + # may only NARROW the parked batch. A name that was never parked cannot be introduced by the + # payload, or the review stops being what authorised the send. + want = {str(n) for n in (body or {}).get("customers") or []} + if want: + items = [it for it in items if str(it.get("customer")) in want] + if not items: + raise err(400, "bad_request", "none of those customers are in the parked batch") + rows, _ = _rows() + sent, failed, skipped = [], [], [] + for it in items: + name = str(it.get("customer") or "") + row = _find(rows, name) + if row is None: + skipped.append({"customer": name, + "reason": "no longer on the collection list, so nothing is owed"}) + continue + try: + # ⛔ THE SAME DATA-LAYER CALL `send()` MAKES. SafeModeBlocked is raised INSIDE + # `queue_statement`, so the guardrail cannot be argued with from here. + mid = cs.queue_statement(cs.Odoo(), row, + str(it.get("subject") or "") or cs.DEFAULT_SUBJECT, + cs.DEFAULT_INTRO, cs.DEFAULT_FOOTER) + sent.append({"customer": name, "to": row.get("Email"), "mailId": mid}) + except Exception as e: # noqa: BLE001 + failed.append({"customer": name, "error": str(e)[:200]}) + # ⚠ CLEARED ONLY WHEN NOTHING IS LEFT TO RETRY. A batch dropped while some of it failed would + # lose the list of who still needs a statement, and nobody would know to look. + if sent and not failed: + engine.clear_statements(session.runtime, auto_id) + return {"sent": sent, "failed": failed, "skipped": skipped, + "safeMode": bool(cs.SAFE_MODE), "cleared": bool(sent and not failed)} diff --git a/api/routes_tables.py b/api/routes_tables.py index 9be38e6e6c95d32ed698178ac95222089ae4e1e0..ed4db67e7a256237fb39d508b5dfcb7b8be6687e 100644 --- a/api/routes_tables.py +++ b/api/routes_tables.py @@ -1406,7 +1406,11 @@ def _fire_on_change(table_key, pid, changed, session): defn = _ut().get(table_key, st=session.runtime) or {} wanted = _ae.on_change_fields(defn, changed.keys()) for field in wanted: - _ae.run_field(table_key, field["key"], st=session.runtime, rows=[str(pid)]) + # ⭐ W35-T41 / C7 — `user` is the usage ledger's attribution. An on-change run is still + # somebody's edit spending somebody's tokens, so it is booked against the person who + # typed rather than left unattributed. + _ae.run_field(table_key, field["key"], st=session.runtime, rows=[str(pid)], + user=session.uname) except Exception: # noqa: BLE001 pass @@ -1450,7 +1454,9 @@ def enrich_field(table_key: str, fkey: str, body: dict = Body(default=None), report = _ae.run_field(table_key, fkey, st=session.runtime, rows=rows, policy=str((body or {}).get("scope") or "") or None, # A named row set through THIS door is a person asking. - manual=rows is not None) + manual=rows is not None, + # ⭐ W35-T41 / C7 — the usage ledger's attribution. + user=session.uname) if report.get("problem"): # A run that could not start at all is not a 200: nothing was attempted, nothing was # spent, and the reason is actionable (no provider configured, or the wrong column). diff --git a/api/routes_usage.py b/api/routes_usage.py new file mode 100644 index 0000000000000000000000000000000000000000..ae770793c62ea1ec928315c1bdd1b62ef2346abb --- /dev/null +++ b/api/routes_usage.py @@ -0,0 +1,51 @@ +"""routes_usage.py — WAVE 35 (ruling R9, contract C7): `GET /api/v1/usage`, the AI meter. + +Owner item 13's second door. R9: the dashboard REPORTS weekly usage per surface against an +allowance and does **not** cut anyone off this wave; over the allowance the product still works and +the bar is red. + +⛔ THIS ROUTE ENFORCES NOTHING AND MUST NOT START. It is the only reader of `usage_ledger`, and the +ledger has no ceiling in it (see that module's header). A `403 over_allowance` here would turn a +reporting feature into a wall the owner explicitly did not ask for this wave. + +⚠ TWO SCOPES, AND THE WIDER ONE IS ADMIN-ONLY. `?scope=me` (the default) is the caller's own week; +`?scope=tenant` folds every account and is refused for a non-admin — how much AI another employee +used is not a fact this product hands out sideways. +""" +from fastapi import APIRouter, Depends + +import usage_ledger +from deps import Session, err, require_session + +router = APIRouter(prefix="/api/v1") + + +@router.get("/usage") +def usage(scope: str = "me", session: Session = Depends(require_session)): + """`{week, resets, allowance, total, over, calls, unmeasured, scope, surfaces: [...]}`. + + Every one of the four surfaces is present whether or not it was used — see + `usage_ledger.usage`'s note on why a missing row and a quiet week must not look the same. + """ + wanted = str(scope or "me").strip().lower() + if wanted not in ("me", "tenant"): + raise err(400, "bad_request", "scope must be me or tenant") + if wanted == "tenant" and not session.admin: + raise err(403, "forbidden", + "only an administrator can see the whole workspace's AI usage") + out = usage_ledger.usage(session.runtime, session.uname, + tenant_wide=(wanted == "tenant")) + if session.admin and usage_ledger.UNATTRIBUTED["calls"]: + # ⛔ THE METER REPORTS ITS OWN BLIND SPOT (R6's second sentence). A call `record()` could not + # attribute — no store handle reached it, or the store refused — is absent from every number + # above, and an operator has to be able to see that rather than infer it. + # ⚠ ADMIN ONLY, and labelled as a PROCESS figure rather than a tenant one: the counter is a + # module global in this container, so it spans every tenant this process has served and is + # not a fact about the workspace being asked about. + out["unattributed"] = { + "calls": usage_ledger.UNATTRIBUTED["calls"], + "surfaces": dict(usage_ledger.UNATTRIBUTED["surfaces"]), + "note": ("these AI calls reached no ledger target in this container and are missing " + "from every figure above. It is a wiring gap, not usage"), + } + return out diff --git a/api/usage_ledger.py b/api/usage_ledger.py new file mode 100644 index 0000000000000000000000000000000000000000..4d42c06bc5bf7e6f06bddf11fbc908bf05a8be34 --- /dev/null +++ b/api/usage_ledger.py @@ -0,0 +1,281 @@ +"""usage_ledger.py — WAVE 35 (ruling R9, contract C7): ONE METER FOR EVERY AI SURFACE. + +R9: *"ONE meter for every AI surface. The dashboard REPORTS weekly usage against an allowance and +does NOT cut anyone off this wave; over the allowance it still works, shown red."* + +⛔ SO THIS FILE MEASURES AND NEVER ENFORCES. There is no ceiling here, no refusal, and no caller of +this module may branch on its answer. The one limit the product does enforce is per COLUMN and lives +where it was built (`ai_enrich.ceiling_report`); this is the tenant-wide REPORT, and mixing the two +would make an over-allowance week silently stop somebody's enrichment run. + +**THE FOUR SURFACES.** `assistant` (a person asking the Assistant a question) · `field_agent` (an AI +enrichment column filling cells) · `ai_review` (a model deciding a review stage) · `automation_draft` +(a sentence becoming a draft flow). They are the four LLM entry points the product has, and the gate +in `verify_api.py` DERIVES them from the code rather than reading this list, so a fifth entry point +is caught by the gate and not by somebody remembering to edit a tuple. + +⛔⛔ **`record()` MUST BE HANDED ITS TARGET, AND THAT IS A MEASUREMENT, NOT A STYLE CHOICE.** The +obvious design is to bind `(runtime, username)` into a `contextvars.ContextVar` inside +`deps.require_session` — every session-authed request passes through it — so `record()` could take +C7's five arguments and resolve the rest itself. **It does not work here and it fails SILENTLY.** +Measured 2026-08-17 with a FastAPI `TestClient`: a `ContextVar` set inside a sync dependency reads +back `None` in the handler *and in every function the handler calls*, on the SAME OS thread (17820 +both sides) — anyio copies the context per callable, so a `.set()` in the dependency's copy is +invisible to the endpoint's. An async endpoint reads `None` too. A ledger built that way would have +recorded nothing, answered no error, and shown a dashboard of zeros that looks exactly like a quiet +week ([[flag-shipped-without-its-writer]]). + +⚠ THEREFORE `st` AND `user` ARE KEYWORD ARGUMENTS AFTER C7'S FIVE, and a call that omits them is +COUNTED as unattributed and REPORTED rather than dropped — R6's second sentence applied to the meter +itself. A surface whose caller cannot reach a store handle is a gap somebody has to close; a meter +that hides the gap is worse than one that admits it. +""" +from __future__ import annotations + +import os +import time + +#: The store bucket. Small by construction — counters only, never a per-call log: a log would grow +#: without bound on a route that runs per row, and nothing in R9 asks a question a log answers. +STORE_KEY = "ai_usage" + +#: The four AI surfaces (C7). ⚠ A surface this list does not name is REFUSED rather than stored under +#: whatever string arrived, because a typo'd surface is a meter that reads low forever with a second +#: row nobody looks at. +SURFACES = ("assistant", "field_agent", "ai_review", "automation_draft") + +#: The weekly allowance the dashboard measures against (R9). ⛔ IT IS A REPORTING LINE, NOT A GATE. +#: Env-tunable so a deployment can set a real number without a release. +WEEKLY_ALLOWANCE = int(os.environ.get("AIOS_AI_WEEKLY_TOKENS") or 1_000_000) + +#: How many weeks stay in the bucket. A meter is about this week and the trend behind it; keeping +#: every week forever turns a counter document into an append-only log by another route. +MAX_WEEKS = 8 + +#: Calls this PROCESS could not attribute to a store and a user — see the header. Reported to an +#: admin by `GET /usage`, never silently zero. +UNATTRIBUTED = {"calls": 0, "surfaces": {}} + + +def week_of(when=None): + """The ISO week this instant belongs to, as `"2026-W33"`. + + ⚠ UTC, EXPLICITLY, and never a naive local stamp. `routes_nav` records the same correction for + its own stamps: a naive local time written on a UTC host and parsed by a non-UTC browser misfiles + into the wrong bucket with nothing to go red. `%G`/`%V` are the ISO year and ISO week, which is + the only pair that agrees with itself across a year boundary (`%Y-%W` does not). + """ + return time.strftime("%G-W%V", time.gmtime(when if when is not None else time.time())) + + +def resets_at(when=None): + """The UTC date the CURRENT week's counters roll over, as `"YYYY-MM-DD"`. + + R9 asks the page to state when the allowance resets, so the server answers it — a client that + computed "next Monday" itself would be a second implementation of the week boundary, and the two + would disagree for anybody whose clock is not UTC ([[one-question-two-normalizers]]). + """ + now = when if when is not None else time.time() + tm = time.gmtime(now) + # `tm_wday` is 0 for Monday, so days-until-next-Monday is 7 minus however far in we are. + return time.strftime("%Y-%m-%d", time.gmtime(now + (7 - tm.tm_wday) * 86400)) + + +def tokens_from(body): + """`(tokens_in, tokens_out)` for one provider response. Either may be None. + + ⛔ `None` MEANS THE PROVIDER DID NOT SAY, AND IT IS NOT ZERO. A ledger that books an unmeasured + call at zero reports a cheaper week than happened, which is exactly the cost-surprise complaint + R13 cited when the first token accounting was built. `ai_enrich._usage_tokens` has said this + since wave 34; this is that reader SPLIT, because C7 asks for input and output separately. + + ⚠ THE TOTAL-ONLY SHAPE IS REAL AND IS NOT A SPLIT. Some providers answer only `total_tokens`. + That is reported by `record(total=...)` rather than by inventing a split, because halving a + total would put two numbers on a dashboard that were never measured. + """ + usage = (body or {}).get("usage") + if not isinstance(usage, dict): + return (None, None) + + def _int(*keys): + for key in keys: + got = usage.get(key) + if isinstance(got, int) and not isinstance(got, bool): + return got + return None + + return (_int("prompt_tokens", "input_tokens"), + _int("completion_tokens", "output_tokens")) + + +def total_from(body): + """One call's TOTAL tokens, or None — the shape a caller with no use for the split wants. + + ⚠ Prefers the provider's own `total_tokens` over a sum, because a provider that reports both may + count cached or reasoning tokens in the total and in neither half. + """ + usage = (body or {}).get("usage") + if isinstance(usage, dict): + for key in ("total_tokens", "totalTokens"): + got = usage.get(key) + if isinstance(got, int) and not isinstance(got, bool): + return got + ins, outs = tokens_from(body) + if ins is None and outs is None: + return None + return (ins or 0) + (outs or 0) + + +def _int_or_none(value): + if isinstance(value, int) and not isinstance(value, bool): + return max(0, value) + return None + + +def record(surface, provider, model, tokens_in=None, tokens_out=None, *, + total=None, calls=1, st=None, user=""): + """Book ONE AI call. Returns True when the line landed durably, False when it could not. + + C7's signature is the five positional parameters; everything after `*` is this module's own + extension and each one has a reason: + + · `total` — for a caller that knows the total and not the split (`ai_enrich.run_field` + accumulates a per-run total through an `_ask` contract a gate injects against, and changing + that contract to carry a split would break the injected shape for no gain here). + · `calls` — so a RUN over many rows books one line with its real call count instead of one + store write per row. + · `st` / `user` — the target. Required in practice; see this module's header for the measured + reason a contextvar cannot supply them. + + ⛔ IT NEVER RAISES. A meter that can break the feature it measures is worse than no meter: a + store blip during an enrichment run would abort the run and lose the cells. + ⛔ AND IT NEVER SILENTLY SUCCEEDS. A call it cannot attribute increments `UNATTRIBUTED`, which + `GET /usage` shows an admin. + """ + key = str(surface or "").strip() + if key not in SURFACES: + # Refused rather than stored: a surface nobody named is a row nobody reads, and the meter + # would read low forever with no sign of why. + return False + ins, outs = _int_or_none(tokens_in), _int_or_none(tokens_out) + tot = _int_or_none(total) + if tot is None and (ins is not None or outs is not None): + tot = (ins or 0) + (outs or 0) + n = max(0, int(calls) if isinstance(calls, int) and not isinstance(calls, bool) else 1) + if not n: + return False + uname = str(user or "").strip().lower() + if st is None: + UNATTRIBUTED["calls"] += n + UNATTRIBUTED["surfaces"][key] = UNATTRIBUTED["surfaces"].get(key, 0) + n + return False + week = week_of() + prov, mdl = str(provider or "")[:40], str(model or "")[:80] + + def _up(data): + data = data if isinstance(data, dict) else {} + weeks = {w: v for w, v in data.items() if isinstance(v, dict)} + row = ((weeks.setdefault(week, {}) + .setdefault(uname or "-", {})) + .setdefault(key, {})) + row["calls"] = int(row.get("calls") or 0) + n + if tot is None: + # ⛔ "the call happened, the tokens are unknown" IS A STATE, and it has to be + # representable or the meter quietly under-reports. The dashboard shows it as a count of + # unmeasured calls beside the token total. + row["unmeasured"] = int(row.get("unmeasured") or 0) + n + else: + row["tokens"] = int(row.get("tokens") or 0) + tot + if ins is not None: + row["tokens_in"] = int(row.get("tokens_in") or 0) + ins + if outs is not None: + row["tokens_out"] = int(row.get("tokens_out") or 0) + outs + if prov: + row["provider"] = prov + if mdl: + row["model"] = mdl + # Oldest weeks first to go. Sorting ISO week strings sorts chronologically by construction, + # which is the second reason `%G-W%V` is the format rather than a prettier one. + for stale in sorted(weeks)[:-MAX_WEEKS]: + weeks.pop(stale, None) + return weeks + + try: + # ⚠ `flush='async'` — the coalescing mode ([[store-async-flush]]). This is the hot path of + # every AI surface in the product (an enrichment run books one line per RUN, but the + # assistant books one per question), and a blocking hub upload per call would put a network + # round trip inside the answer a person is waiting for. The mutation applies to the + # in-process cache immediately, so `GET /usage` reads its own writes. + st.update(STORE_KEY, _up, flush="async") + return True + except Exception: # noqa: BLE001 + UNATTRIBUTED["calls"] += n + UNATTRIBUTED["surfaces"][key] = UNATTRIBUTED["surfaces"].get(key, 0) + n + return False + + +def _blank_row(): + return {"calls": 0, "tokens": 0, "tokens_in": 0, "tokens_out": 0, "unmeasured": 0} + + +def _fold(rows): + out = _blank_row() + for row in rows: + if not isinstance(row, dict): + continue + for field in ("calls", "tokens", "tokens_in", "tokens_out", "unmeasured"): + got = row.get(field) + if isinstance(got, int) and not isinstance(got, bool): + out[field] += max(0, got) + return out + + +def usage(st, user="", *, tenant_wide=False, week=None): + """This week's meter. `{week, resets, allowance, total, over, surfaces: [...], ...}`. + + `tenant_wide=True` folds every account in the tenant — the admin view. Otherwise only `user`. + + ⛔ EVERY SURFACE IS PRESENT WHETHER OR NOT IT WAS USED, at zero. A surface a consumer has to test + for is a surface a consumer forgets to test for, and R9 says ONE meter for every AI surface: a + missing row and a quiet week would render identically, and only one of them is true. + ⛔ NOTHING HERE IS ESTIMATED. A surface with no ledger line reports zero, never a guess + ([[no-unverifiable-aggregates]]). + """ + wk = str(week or week_of()) + try: + stored = st.get(STORE_KEY) or {} + except Exception: # noqa: BLE001 + stored = {} + by_week = stored.get(wk) if isinstance(stored, dict) else {} + by_week = by_week if isinstance(by_week, dict) else {} + uname = str(user or "").strip().lower() + if tenant_wide: + buckets = [b for b in by_week.values() if isinstance(b, dict)] + else: + buckets = [by_week.get(uname or "-")] if isinstance(by_week.get(uname or "-"), dict) else [] + surfaces = [] + for key in SURFACES: + folded = _fold([b.get(key) for b in buckets]) + folded["surface"] = key + surfaces.append(folded) + total = sum(s["tokens"] for s in surfaces) + unmeasured = sum(s["unmeasured"] for s in surfaces) + out = { + "week": wk, + "resets": resets_at(), + "allowance": WEEKLY_ALLOWANCE, + "total": total, + # R9: over the allowance the product STILL WORKS and the bar goes red. This flag is the + # colour, never a wall — nothing in the API reads it. + "over": bool(WEEKLY_ALLOWANCE and total > WEEKLY_ALLOWANCE), + "calls": sum(s["calls"] for s in surfaces), + "unmeasured": unmeasured, + "scope": "tenant" if tenant_wide else "me", + "surfaces": surfaces, + } + if unmeasured: + # R6's second sentence: the number is short by an unknown amount and the payload says so + # rather than presenting a confident total. + out["note"] = (f"{unmeasured} call(s) this week did not report a token count, so the total " + f"above is a floor. The provider decides whether to report usage") + return out diff --git a/platform/core/keychain.py b/platform/core/keychain.py index c68126e22926bfee4f477fccfd0cd6d4b3712d0e..3bb97e549e64da02aeed59cce8f6e2af373c7bae 100644 --- a/platform/core/keychain.py +++ b/platform/core/keychain.py @@ -166,6 +166,52 @@ def read_fields(rt, entry_id): return None +def first_entry_of_type(rt, etype): + """The metadata row of the FIRST entry of `etype`, or None. Never decrypts anything. + + ⭐ W35-T45 — the same "first, by insertion-id sort" rule `_first_creds` resolves on, exposed as + an IDENTITY question rather than a credential one. `ensure_entry_of_type` needs to know whether + an entry exists WITHOUT unlocking it, and `list_entries` + a hand-rolled `next()` at each caller + is how two places come to disagree about which entry is "the" one. + """ + return next((e for e in list_entries(rt) if e.get('type') == str(etype or '')), None) + + +def ensure_entry_of_type(rt, etype, label, fields, username): + """Create ONE entry of `etype` only if the tenant has none. `(row, why)`; `row` is None on skip. + + ⭐ W35-T45 / R11 — the primitive tenant #0's env-to-keychain migration is built from, and the + reason it lives here is that only this module knows what "already has one" means: `odoo_creds` + resolves to the FIRST entry of a type, so a second one would be stored, invisible in every + resolver, and impossible to tell from the one that serves. + + ⛔ IDEMPOTENT BY TYPE, NOT BY A FLAG. A migration that keys idempotency on a marker it writes + itself is one lost marker away from running twice ([[a-migration-that-runs-on-the-next-write]]); + keying it on the thing it would create cannot double-apply, whatever else fails. + + ⛔ AND IT REFUSES ON A LOCKED KEYCHAIN RATHER THAN STORING A SECRET IN THE CLEAR. `add_entry` + would raise `KeychainLocked`; this answers with a REASON instead, because "there is no Fernet key + on this deployment" is an operator fact to report, not an exception to propagate out of a boot + thread where nobody reads it. + + ⚠ `why` IS RETURNED EVEN ON SUCCESS-BY-SKIP. "Already migrated" and "could not migrate" are + different operator actions and a bare None cannot tell them apart. + """ + etype = str(etype or '').strip().lower() + if etype not in ENTRY_TYPES: + return None, f'{etype!r} is not a keychain entry type' + existing = first_entry_of_type(rt, etype) + if existing: + return None, f'this tenant already has a {etype} entry ({existing["id"]})' + if not unlocked(): + return None, ('this deployment has no keychain key, and a credential must never be stored ' + 'in the clear') + try: + return add_entry(rt, label, etype, fields, username), '' + except Exception as exc: # noqa: BLE001 + return None, f'{type(exc).__name__}: {exc}' + + def odoo_creds(rt): """R3's resolver seam: the FIRST odoo-type entry's fields, else None (the caller falls back to the environment). Deterministic order = insertion-id sort, so 'first' is stable.""" diff --git a/web/src/account/FeedbackPage.tsx b/web/src/account/FeedbackPage.tsx new file mode 100644 index 0000000000000000000000000000000000000000..c301215e32d99d072e96cd1ca0e43cd4d8cd6f06 --- /dev/null +++ b/web/src/account/FeedbackPage.tsx @@ -0,0 +1,246 @@ +// --------------------------------------------------------------------------- +// account/FeedbackPage.tsx — WAVE 35 · W35-T18 (owner item 13, ruling R8, C6). +// +// One composer, one category, one send. R8 puts every submission on the +// platform-wide operator plane: no tenant sees another's, and the submitting +// tenant cannot edit or delete what it sent — so there is deliberately no "my +// feedback" list here, and the 201 IS the receipt. +// +// ⛔ THE SHAPE IS THE ASSISTANT'S CENTRED COLUMN, READ AND NOT IMPORTED. +// `assistant/assistant.css` belongs to another session this wave, so the anatomy +// is borrowed (a measure-bounded column; a rounded field with the control row +// inside it; a round send button that goes dark when there is something to send) +// while every rule here is `.acct-*` in this module's own stylesheet. Same system +// to the eye, no shared file to collide over. +// +// ⛔ FOUR STATES, AND THE FIRST TWO ARE THE ONES THAT GET SKIPPED. The category +// list is the SERVER'S (B-1/E-9) — there is no local copy to fall back to — so +// while it is in flight there is no form, and if it FAILS there is a sentence and +// a retry, never an empty dropdown and never an invented list. A page that offers +// a category the door refuses is worse than a page that says it cannot load. +// +// ⚠ THE SURFACE IS SPLIT FROM THE READ, and the reason is evidence rather than +// taste: `renderToStaticMarkup` never runs an effect, so a shot of the fetching +// component can only photograph its own spinner. With the whole state as ONE +// prop, every state this page has — loading, failed, ready, sending, sent, +// refused — can be rendered and looked at without a server, which is the only +// evidence a lane with no server can produce. +// --------------------------------------------------------------------------- + +import { useCallback, useEffect, useRef, useState } from "react"; + +import { loadFeedbackForm, sendFeedback } from "./accountApi"; +import type { FeedbackForm } from "./accountApi"; +import "./account.css"; + +/** The send arrow, in the same stroke vocabulary as the rest of the app's small glyphs. */ +function SendIcon() { + return ( + + ); +} + +export type FeedbackLoad = + | { phase: "loading" } + | { phase: "ready"; form: FeedbackForm } + | { phase: "failed"; message: string }; + +/** Everything this page holds, in one value, so the surface below is a function of it. */ +export interface FeedbackState { + load: FeedbackLoad; + category: string; + text: string; + sending: boolean; + /** The last send SUCCEEDED. Cleared the moment the person types again. */ + sent: boolean; + /** The last send was REFUSED, and this is the server's own sentence. */ + error: string; +} + +export const EMPTY_FEEDBACK: FeedbackState = { + load: { phase: "loading" }, + category: "", + text: "", + sending: false, + sent: false, + error: "", +}; + +export function FeedbackSurface({ + state, + onText, + onCategory, + onSend, + onRetry, +}: { + state: FeedbackState; + onText: (text: string) => void; + onCategory: (key: string) => void; + onSend: () => void; + onRetry: () => void; +}) { + const { load, category, text, sending, sent, error } = state; + const form = load.phase === "ready" ? load.form : null; + const tooLong = !!form && form.maxChars > 0 && text.length > form.maxChars; + const canSend = !!form && !!category && text.trim() !== "" && !tooLong && !sending; + + return ( +
+

Feedback

+
+ {load.phase === "loading" ? ( +
+ +
+ ) : load.phase === "failed" ? ( + // ⛔ NOT AN EMPTY FORM. The categories are the server's and there is no local copy, so + // a failed read means there is nothing to offer; drawing the composer anyway would + // invite somebody to write a paragraph into a control that cannot send it. +
+

{load.message}

+ +
+ ) : ( + <> + {sent ? ( + // The confirmation. It sits ABOVE the composer rather than replacing it, because + // the next thought is often the same person's, and a page that has to be navigated + // back to is a page that collects less of what R8 exists to collect. +

+ Thanks. That went to the Loopable team. +

+ ) : null} + {error ? ( +

+ {error} +

+ ) : null} + +
+