Deploy AIOS web (React glide grid + FastAPI slice)
Browse filesThis view is limited to 50 files because it contains too many changes. Β See raw diff
- RELEASES.json +7 -1
- VERSION +1 -1
- api/ai_review.py +58 -23
- api/automation_engine.py +83 -0
- api/connectors_tt.py +568 -568
- api/main.py +19 -0
- api/odoo_relational.py +0 -0
- api/providers.py +321 -4
- api/routes_admin.py +0 -0
- api/routes_agent_harness.py +455 -0
- api/routes_alerts.py +668 -668
- api/routes_automation.py +279 -8
- api/routes_grid.py +81 -0
- api/routes_keychain.py +0 -0
- api/routes_nav.py +0 -0
- api/routes_oauth.py +114 -114
- api/routes_odoo_tables.py +0 -0
- api/routes_publish.py +0 -0
- api/routes_query.py +0 -0
- api/routes_records.py +211 -211
- api/routes_script_views.py +329 -0
- api/routes_shares.py +393 -393
- api/routes_slack.py +28 -12
- api/routes_statements.py +31 -2
- api/routes_tables.py +0 -0
- api/routes_web_agent.py +5 -4
- platform/aios_grid.py +0 -0
- platform/core/perm_scope.py +349 -0
- platform/core/perms.py +31 -18
- platform/core/script_sandbox.py +519 -0
- platform/core/shares.py +19 -11
- platform/core/store.py +0 -0
- platform/core/table_store.py +746 -618
- platform/core/user_tables.py +0 -0
- platform/harness/meta_store.py +521 -521
- platform/model/topics/odoo_accounts.yml +60 -60
- platform/model/topics/odoo_agents.yml +59 -59
- platform/model/topics/odoo_bills.yml +86 -86
- platform/model/topics/odoo_customers.yml +213 -213
- platform/model/topics/odoo_invoices.yml +98 -98
- platform/model/topics/odoo_orders.yml +87 -87
- platform/model/topics/odoo_products.yml +147 -147
- platform/model/topics/odoo_vendors.yml +103 -103
- platform/modules/collections_send.py +7 -3
- platform/modules/product_data.py +804 -804
- platform/modules/products.py +737 -737
- web/public/sample_customers.json +558 -558
- web/src/alerts/alertsModel.ts +418 -418
- web/src/assistant/AssistantPage.tsx +11 -1
- web/src/automation/AutomationBuilder.tsx +557 -16
RELEASES.json
CHANGED
|
@@ -1,6 +1,12 @@
|
|
| 1 |
{
|
| 2 |
-
"current": "
|
| 3 |
"releases": [
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4 |
{
|
| 5 |
"version": "v28",
|
| 6 |
"sha": "2793e4b",
|
|
|
|
| 1 |
{
|
| 2 |
+
"current": "c9ad659 (c9ad659)",
|
| 3 |
"releases": [
|
| 4 |
+
{
|
| 5 |
+
"version": "v29",
|
| 6 |
+
"sha": "55ced50",
|
| 7 |
+
"date": "2026-08-18",
|
| 8 |
+
"subject": "v29: the build staging has been running since 2026-08-18 (55ced50)."
|
| 9 |
+
},
|
| 10 |
{
|
| 11 |
"version": "v28",
|
| 12 |
"sha": "2793e4b",
|
VERSION
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
c9ad659 (c9ad659)
|
api/ai_review.py
CHANGED
|
@@ -56,7 +56,9 @@ PROVIDERS = [
|
|
| 56 |
# labels it already has in front of it.
|
| 57 |
{"name": "anthropic", "env": "ANTHROPIC_API_KEY", "shape": "anthropic",
|
| 58 |
"url": "https://api.anthropic.com/v1/messages",
|
| 59 |
-
|
|
|
|
|
|
|
| 60 |
]
|
| 61 |
ANTHROPIC_VERSION = "2023-06-01"
|
| 62 |
TIMEOUT_SECONDS = float(os.environ.get("AIOS_AI_REVIEW_TIMEOUT") or 20)
|
|
@@ -277,13 +279,23 @@ MAX_DRAFT_ACTIONS = 12 # a draft a person reads in one screen; the engi
|
|
| 277 |
MAX_PROMPT_CHARS = 2000
|
| 278 |
|
| 279 |
|
| 280 |
-
def flow_providers():
|
| 281 |
-
"""The rungs usable here, in THIS module's refusal-first order. Empty = the feature is off.
|
| 282 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 283 |
by_name = {p["name"]: p for p in PROVIDERS}
|
| 284 |
-
|
| 285 |
-
return [by_name[n] for n in order
|
| 286 |
if n in by_name and (os.environ.get(by_name[n]["env"]) or "").strip()]
|
|
|
|
|
|
|
|
|
|
|
|
|
| 287 |
|
| 288 |
|
| 289 |
def flow_schema(kinds, trigger_keys, table_keys):
|
|
@@ -421,7 +433,7 @@ def _flow_from_tool_call(obj):
|
|
| 421 |
|
| 422 |
|
| 423 |
def draft_flow(*, prompt, catalog, required, triggers, tables, chat=None, timeout=None,
|
| 424 |
-
st=None, user=""):
|
| 425 |
"""A sentence -> `(draft, refusal_sentence, provider)`. β NOTHING IS SAVED HERE.
|
| 426 |
|
| 427 |
Exactly one of `draft` and `refusal_sentence` is truthy β the same contract
|
|
@@ -456,7 +468,7 @@ def draft_flow(*, prompt, catalog, required, triggers, tables, chat=None, timeou
|
|
| 456 |
draft, refusal = _flow_from_tool_call(chat(messages, tools))
|
| 457 |
return draft, refusal, "injected"
|
| 458 |
|
| 459 |
-
provs = flow_providers()
|
| 460 |
if not provs:
|
| 461 |
# β SAY SO. An AI feature that silently does nothing is indistinguishable from one that was
|
| 462 |
# never built [[flag-shipped-without-its-writer]].
|
|
@@ -464,30 +476,53 @@ def draft_flow(*, prompt, catalog, required, triggers, tables, chat=None, timeou
|
|
| 464 |
"be drafted from a description yet"), None
|
| 465 |
tmo = float(timeout or TIMEOUT_SECONDS)
|
| 466 |
problems = []
|
|
|
|
| 467 |
for p in provs:
|
| 468 |
-
|
| 469 |
-
|
| 470 |
-
|
| 471 |
-
|
| 472 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 473 |
try:
|
| 474 |
-
|
| 475 |
-
|
| 476 |
-
|
| 477 |
-
|
| 478 |
-
|
| 479 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 480 |
except Exception as e: # noqa: BLE001
|
| 481 |
problems.append(f"{p['name']}: {type(e).__name__}")
|
| 482 |
continue
|
| 483 |
if r.status_code != 200:
|
| 484 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 485 |
continue
|
| 486 |
try:
|
| 487 |
body = r.json()
|
| 488 |
-
|
| 489 |
-
|
| 490 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 491 |
except Exception as e: # noqa: BLE001
|
| 492 |
problems.append(f"{p['name']}: unreadable answer ({type(e).__name__})")
|
| 493 |
continue
|
|
|
|
| 56 |
# labels it already has in front of it.
|
| 57 |
{"name": "anthropic", "env": "ANTHROPIC_API_KEY", "shape": "anthropic",
|
| 58 |
"url": "https://api.anthropic.com/v1/messages",
|
| 59 |
+
# β AN ENV LEVER, NOT A NEW DEFAULT (W36-T35): haiku stays the choice for the reasons above,
|
| 60 |
+
# and a deployment whose side rungs are out of credit can raise the tier without a release.
|
| 61 |
+
"model": os.environ.get("AIOS_AI_REVIEW_ANTHROPIC_MODEL") or "claude-haiku-4-5"},
|
| 62 |
]
|
| 63 |
ANTHROPIC_VERSION = "2023-06-01"
|
| 64 |
TIMEOUT_SECONDS = float(os.environ.get("AIOS_AI_REVIEW_TIMEOUT") or 20)
|
|
|
|
| 279 |
MAX_PROMPT_CHARS = 2000
|
| 280 |
|
| 281 |
|
| 282 |
+
def flow_providers(pin=None):
|
| 283 |
+
"""The rungs usable here, in THIS module's refusal-first order. Empty = the feature is off.
|
| 284 |
+
|
| 285 |
+
β `pin` IS ASK D-18 (2026-08-18): the Agent chat's model toggle must configure something, and
|
| 286 |
+
the draft door used to read `prompt` off the body and nothing else β so the key the client sent
|
| 287 |
+
was accepted and dropped, and the picker was a control over nothing.
|
| 288 |
+
β AN UNKNOWN OR UNCONFIGURED PIN FALLS BACK TO THE LADDER rather than refusing. A model the
|
| 289 |
+
ladder stopped offering must not turn every later draft into an error; the caller is told which
|
| 290 |
+
rung actually answered, which is the honest half.
|
| 291 |
+
"""
|
| 292 |
by_name = {p["name"]: p for p in PROVIDERS}
|
| 293 |
+
live = [n for n in FLOW_PROVIDER_ORDER
|
|
|
|
| 294 |
if n in by_name and (os.environ.get(by_name[n]["env"]) or "").strip()]
|
| 295 |
+
wanted = str(pin or os.environ.get("AIOS_FLOW_PROVIDER") or "").strip().lower()
|
| 296 |
+
if wanted and wanted in live:
|
| 297 |
+
return [by_name[wanted]]
|
| 298 |
+
return [by_name[n] for n in live]
|
| 299 |
|
| 300 |
|
| 301 |
def flow_schema(kinds, trigger_keys, table_keys):
|
|
|
|
| 433 |
|
| 434 |
|
| 435 |
def draft_flow(*, prompt, catalog, required, triggers, tables, chat=None, timeout=None,
|
| 436 |
+
st=None, user="", model=None):
|
| 437 |
"""A sentence -> `(draft, refusal_sentence, provider)`. β NOTHING IS SAVED HERE.
|
| 438 |
|
| 439 |
Exactly one of `draft` and `refusal_sentence` is truthy β the same contract
|
|
|
|
| 468 |
draft, refusal = _flow_from_tool_call(chat(messages, tools))
|
| 469 |
return draft, refusal, "injected"
|
| 470 |
|
| 471 |
+
provs = flow_providers(model)
|
| 472 |
if not provs:
|
| 473 |
# β SAY SO. An AI feature that silently does nothing is indistinguishable from one that was
|
| 474 |
# never built [[flag-shipped-without-its-writer]].
|
|
|
|
| 476 |
"be drafted from a description yet"), None
|
| 477 |
tmo = float(timeout or TIMEOUT_SECONDS)
|
| 478 |
problems = []
|
| 479 |
+
import providers as _prov
|
| 480 |
for p in provs:
|
| 481 |
+
# ββ W36-T35 / R4 β THE OWNER QUOTED THIS LINE BACK AT US. It used to read
|
| 482 |
+
# `problems.append(f"{p['name']}: tool calls are not wired for this shape")` and `continue`,
|
| 483 |
+
# so the sentence *"anthropic: tool calls are not wired for this shape"* appeared under the
|
| 484 |
+
# Agent module verbatim. R4: *"Anthropic becomes the tool-calling path that always works."*
|
| 485 |
+
# It is wired now, through the ONE wire in `providers`, and it matters more than it looks:
|
| 486 |
+
# every side rung on this account is refusing today (cerebras 402, groq 404, openrouter
|
| 487 |
+
# 402), so without this branch the drafter cannot answer at all.
|
| 488 |
+
# β NO `effort` ON THIS DOOR β `output_config.effort` errors on Haiku 4.5, the tier this
|
| 489 |
+
# ladder runs. The parameter is the caller's to send, which is why the wire takes it.
|
| 490 |
try:
|
| 491 |
+
if p["shape"] == "anthropic":
|
| 492 |
+
req = _prov.anthropic_request(
|
| 493 |
+
model=p["model"], key=os.environ[p["env"]].strip(), system=None,
|
| 494 |
+
messages=messages, tools=tools, max_tokens=1500, tool_choice="required")
|
| 495 |
+
r = requests.post(req["url"], timeout=tmo,
|
| 496 |
+
headers=req["headers"], json=req["json"])
|
| 497 |
+
else:
|
| 498 |
+
r = requests.post(p["url"], timeout=tmo,
|
| 499 |
+
headers={"Authorization": f"Bearer {os.environ[p['env']].strip()}",
|
| 500 |
+
"Content-Type": "application/json"},
|
| 501 |
+
json={"model": p["model"], "messages": messages, "tools": tools,
|
| 502 |
+
"tool_choice": "required", "temperature": 0.1,
|
| 503 |
+
"max_tokens": 1500})
|
| 504 |
except Exception as e: # noqa: BLE001
|
| 505 |
problems.append(f"{p['name']}: {type(e).__name__}")
|
| 506 |
continue
|
| 507 |
if r.status_code != 200:
|
| 508 |
+
# β A SENTENCE, NOT `HTTP 402` (R4's third clause). The owner read the status codes off
|
| 509 |
+
# this very door. `refusal_sentence` also decides whether this was a CREDIT failure, and
|
| 510 |
+
# the memo makes the next turn SKIP the empty account instead of paying for it again.
|
| 511 |
+
if _prov.is_credit_failure(r.status_code, r.text):
|
| 512 |
+
_prov.mark_no_credit(p["name"])
|
| 513 |
+
problems.append(_prov.refusal_sentence(p["name"], r.status_code, r.text))
|
| 514 |
continue
|
| 515 |
try:
|
| 516 |
body = r.json()
|
| 517 |
+
if p["shape"] == "anthropic":
|
| 518 |
+
_text, args, _refused = _prov.anthropic_read(body)
|
| 519 |
+
if _refused:
|
| 520 |
+
problems.append(f"{p['name']}: {_refused}")
|
| 521 |
+
continue
|
| 522 |
+
else:
|
| 523 |
+
calls = (((body.get("choices") or [{}])[0].get("message") or {})
|
| 524 |
+
.get("tool_calls") or [])
|
| 525 |
+
args = json.loads(calls[0]["function"]["arguments"]) if calls else None
|
| 526 |
except Exception as e: # noqa: BLE001
|
| 527 |
problems.append(f"{p['name']}: unreadable answer ({type(e).__name__})")
|
| 528 |
continue
|
api/automation_engine.py
CHANGED
|
@@ -9219,6 +9219,36 @@ ACTION_CATALOG = [
|
|
| 9219 |
"ready": True,
|
| 9220 |
"detail": "Assemble this month's statements and park them for review. Nothing is sent until "
|
| 9221 |
"somebody opens the batch and clicks Send"},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 9222 |
{"kind": "slack", "label": "Send Slack message", "group": "Connected", "ready": False,
|
| 9223 |
"detail": "Needs the Slack connector"},
|
| 9224 |
# ββ 4. ADVANCED LOGIC ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
|
@@ -9306,6 +9336,33 @@ def _tenant_may_use(kind, rt):
|
|
| 9306 |
return True if gate is None else bool(rt is not None and gate(rt))
|
| 9307 |
|
| 9308 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 9309 |
def action_catalog(rt=None):
|
| 9310 |
"""The catalog as the wire carries it β a copy, because a caller that mutated the module
|
| 9311 |
constant would change every later reader's answer.
|
|
@@ -10831,6 +10888,32 @@ def apply_actions(rt, defn, table_key, row_ids, username="automation", log=print
|
|
| 10831 |
"and nothing was sent.")
|
| 10832 |
counts["webBlocked"] += 1
|
| 10833 |
continue
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 10834 |
elif kind == "ai_agent":
|
| 10835 |
# ββ W33-T56 (owner item 7, ruling R3) β THE FUZZY STEP, AT RUN TIME.
|
| 10836 |
#
|
|
|
|
| 9219 |
"ready": True,
|
| 9220 |
"detail": "Assemble this month's statements and park them for review. Nothing is sent until "
|
| 9221 |
"somebody opens the batch and clicks Send"},
|
| 9222 |
+
# ββ WAVE 36 Β· W36-T39 β D-277 CLOSED THE WAY THAT ROW ITSELF RECOMMENDED: *"an
|
| 9223 |
+
# `ACTION_CATALOG` row with `menu: false` β one line, reusing the door R18 built this same wave
|
| 9224 |
+
# to keep the five web kinds readable but unofferable. (a) looks right; it is E's file."*
|
| 9225 |
+
#
|
| 9226 |
+
# β THE DEFECT WAS A KIND IN NO CATALOG ON EITHER SIDE. `routes_automation._field_agent_rows`
|
| 9227 |
+
# emits `flow.actions[0].kind == "ai_enrich"` for every AI-enrichment column in the tenant, and
|
| 9228 |
+
# that string appeared ZERO times here and ZERO times in `aios-web/web/src/automation/`. The
|
| 9229 |
+
# builder resolves a stored step's caption with `catalog.find(c => c.kind === a.kind)` and falls
|
| 9230 |
+
# back to `a.kind`, so a field agent opened in the Agents module showed a RAW TOKEN.
|
| 9231 |
+
#
|
| 9232 |
+
# β `menu: False`, NEVER `ready: False`. `ready: False` renders as "coming soon" β a promise β
|
| 9233 |
+
# and `clean_actions` refuses the kind outright, which would 400 the synthetic row the moment
|
| 9234 |
+
# anything validated it. `menu: False` is the exact shape R18 built: withheld from the picker,
|
| 9235 |
+
# still resolvable as a caption, still valid.
|
| 9236 |
+
# β AND IT IS NOT ADDABLE BY HAND ON PURPOSE. An enrichment belongs to a COLUMN; the automation
|
| 9237 |
+
# canvas is not where one is created, which is why `patch_automation` already refuses a
|
| 9238 |
+
# `field:` id with "change its prompt, model or schedule on the column itself".
|
| 9239 |
+
# ββ AND A THIRD INSTANCE, FOUND BY W36-T39's OWN GATE ON ITS FIRST RUN. `_odoo_sync_row`
|
| 9240 |
+
# emits `flow.actions[0].kind == "odoo_sync"` for the connector's synthetic schedule agent, and
|
| 9241 |
+
# that kind was in no catalog either β the same raw token on the same screen as D-277, one row
|
| 9242 |
+
# down. Two known instances were enough to justify the check; the check then produced a third
|
| 9243 |
+
# nobody had booked, which is the difference between a gate and a regression test.
|
| 9244 |
+
{"kind": "odoo_sync", "label": "Sync from Odoo", "group": "Connected",
|
| 9245 |
+
"ready": True, "menu": False,
|
| 9246 |
+
"detail": "Pull the connected Odoo databases on a schedule. Configured on the connector, not "
|
| 9247 |
+
"here"},
|
| 9248 |
+
{"kind": "ai_enrich", "label": "Enrich this column with AI", "group": "Connected",
|
| 9249 |
+
"ready": True, "menu": False,
|
| 9250 |
+
"detail": "Fill an AI column for the records this flow walks. Configured on the column, not "
|
| 9251 |
+
"here"},
|
| 9252 |
{"kind": "slack", "label": "Send Slack message", "group": "Connected", "ready": False,
|
| 9253 |
"detail": "Needs the Slack connector"},
|
| 9254 |
# ββ 4. ADVANCED LOGIC ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
|
|
|
| 9336 |
return True if gate is None else bool(rt is not None and gate(rt))
|
| 9337 |
|
| 9338 |
|
| 9339 |
+
def catalog_kinds():
|
| 9340 |
+
"""Every action kind this module knows about β the LABEL vocabulary.
|
| 9341 |
+
|
| 9342 |
+
β D-277's WHOLE LESSON IN ONE SENTENCE: the client resolves a stored step's caption out of the
|
| 9343 |
+
catalog and falls back to the raw kind token, so a kind the server can EMIT and the catalog
|
| 9344 |
+
does not carry is a token on somebody's screen. This is the set that must cover every kind any
|
| 9345 |
+
server path can put into a `flow.actions` entry, whether or not a person may add it.
|
| 9346 |
+
"""
|
| 9347 |
+
return frozenset(str(row.get("kind") or "") for row in ACTION_CATALOG)
|
| 9348 |
+
|
| 9349 |
+
|
| 9350 |
+
def configurable_kinds():
|
| 9351 |
+
"""The kinds a PERSON may add from the picker, and must therefore be able to configure.
|
| 9352 |
+
|
| 9353 |
+
ββ W36-T39 β THE CONTRACT BETWEEN THE TWO TREES, DERIVED AND NEVER LISTED. `ready` alone is
|
| 9354 |
+
the wrong set: the five `web_*` kinds are ready and `menu: False` (R18), and `ai_enrich` is
|
| 9355 |
+
ready and `menu: False` (D-277) β all six are real, runnable, captioned, and unofferable. What
|
| 9356 |
+
a client must be able to CONFIGURE is exactly what a person can ADD.
|
| 9357 |
+
|
| 9358 |
+
β DERIVED FROM THE CATALOG, so a new row joins the contract by existing. A hand-kept list
|
| 9359 |
+
would be a third copy of the vocabulary, and the two copies this ticket exists to reconcile
|
| 9360 |
+
were already one too many.
|
| 9361 |
+
"""
|
| 9362 |
+
return frozenset(str(row.get("kind") or "") for row in ACTION_CATALOG
|
| 9363 |
+
if row.get("ready") and row.get("menu", True) is not False)
|
| 9364 |
+
|
| 9365 |
+
|
| 9366 |
def action_catalog(rt=None):
|
| 9367 |
"""The catalog as the wire carries it β a copy, because a caller that mutated the module
|
| 9368 |
constant would change every later reader's answer.
|
|
|
|
| 10888 |
"and nothing was sent.")
|
| 10889 |
counts["webBlocked"] += 1
|
| 10890 |
continue
|
| 10891 |
+
elif kind == "odoo_sync":
|
| 10892 |
+
# Same refusal, same reason as the arm below: a catalog row with no arm is walked,
|
| 10893 |
+
# counted, and reports success having done nothing. The connector owns this sync.
|
| 10894 |
+
if "odoo_sync_not_run_here" not in web_notes:
|
| 10895 |
+
web_notes.append("odoo_sync_not_run_here")
|
| 10896 |
+
log("[aios-auto] odoo_sync: the Odoo pull runs on the connector's own "
|
| 10897 |
+
"schedule, not from this canvas. Nothing was done")
|
| 10898 |
+
counts["webBlocked"] += 1
|
| 10899 |
+
continue
|
| 10900 |
+
elif kind == "ai_enrich":
|
| 10901 |
+
# ββ A REFUSAL ARM, AND IT EXISTS FOR THE REASON THE `send_statement` ARM ABOVE
|
| 10902 |
+
# STATES: `_walk` has no terminal `else`, so a catalog row whose arm is missing is
|
| 10903 |
+
# walked, COUNTED, reports the run `ok` and writes nothing β a step somebody
|
| 10904 |
+
# configured that succeeds at doing nothing. Adding the row (D-277) without this
|
| 10905 |
+
# arm would have opened exactly that window.
|
| 10906 |
+
# β AND THE REFUSAL IS THE TRUTH, not a stub. An AI column is filled by
|
| 10907 |
+
# `ai_enrich`, driven from the column's own editor; the synthetic agent row this
|
| 10908 |
+
# kind appears in is DERIVED from that column and is never stored, so nothing
|
| 10909 |
+
# reaches here through the ordinary path. If something ever does, it must say so
|
| 10910 |
+
# rather than report success.
|
| 10911 |
+
if "ai_enrich_not_run_here" not in web_notes:
|
| 10912 |
+
web_notes.append("ai_enrich_not_run_here")
|
| 10913 |
+
log("[aios-auto] ai_enrich: an AI column is filled from the column's own "
|
| 10914 |
+
"editor, not from this canvas. Nothing was done")
|
| 10915 |
+
counts["webBlocked"] += 1
|
| 10916 |
+
continue
|
| 10917 |
elif kind == "ai_agent":
|
| 10918 |
# ββ W33-T56 (owner item 7, ruling R3) β THE FUZZY STEP, AT RUN TIME.
|
| 10919 |
#
|
api/connectors_tt.py
CHANGED
|
@@ -1,568 +1,568 @@
|
|
| 1 |
-
"""connectors_tt.py β the TIKTOK connector (wave 29 Β· item 7 Β· DEBT D-9 Β· rulings R1 + R2).
|
| 2 |
-
|
| 3 |
-
Everything in this file knows what a VENDOR's TikTok row looks like. Nothing in it knows what an
|
| 4 |
-
automation is. That split is `connectors_ig.py`'s (wave 27 item 23) and it is the reason this file
|
| 5 |
-
exists at all rather than another thousand lines inside the engine.
|
| 6 |
-
|
| 7 |
-
β **EVERY VENDOR FIELD NAME HERE WAS PROBED, NOT GUESSED.** The whole schema β 40 profile / 43
|
| 8 |
-
post / 17 comment fields, each with the vendor's own type, description and `pii` flag β was read
|
| 9 |
-
live from `GET /datasets/{id}/metadata` for **$0.00** and written down in
|
| 10 |
-
`.claude/wiki/waves/wave29/proto/tiktok-schema.md` (promoted to `tiktok-capture.md` at
|
| 11 |
-
close-out). That document is the AUTHORITY: do not re-probe it, and do not invent a key. Where a
|
| 12 |
-
name below reads through a candidate list it is because the vendor has two names for one fact
|
| 13 |
-
(`biography`/`signature`, `region`/`country`), never because the name is uncertain.
|
| 14 |
-
|
| 15 |
-
β **NOTHING HERE EVER AUTHENTICATES TO TIKTOK.** No login, no cookie, no account to get banned β
|
| 16 |
-
public data through a supplier, exactly the rail `connectors_ig.py` states for Instagram. The
|
| 17 |
-
vendor key is a key to a SUPPLIER.
|
| 18 |
-
|
| 19 |
-
β **THE TRANSPORT IS `connectors_bd.py` β SHARED, VENDOR-NAMED, AND NO LONGER BORROWED FROM THE
|
| 20 |
-
OTHER PLATFORM'S CONNECTOR** (WAVE 30 Β· T09, DEBT D-128). `bd_call`, `bd_scrape` and
|
| 21 |
-
`bd_filter_start` take the dataset id as a PARAMETER β they are Bright Data's wire, not
|
| 22 |
-
Instagram's β and re-implementing them here would be a second copy of the deferral handling, the
|
| 23 |
-
truncation guard, the SSRF rail and the snapshot-progress reader, i.e. five places for one bug.
|
| 24 |
-
Until wave 30 the code was right and the NAME was wrong: this file imported thirteen symbols from
|
| 25 |
-
`connectors_ig`, which read as a dependency on Instagram and was really a dependency on a supplier.
|
| 26 |
-
β **This file now imports ZERO names from `connectors_ig`, and a gate check asserts that**, because
|
| 27 |
-
the sentence above is the kind that quietly stops being true.
|
| 28 |
-
|
| 29 |
-
β **WHAT $0 COULD NOT BUY, so nobody reads this file as more measured than it is:**
|
| 30 |
-
1. the real ROW shape β `/metadata` describes a DATASET, and Instagram's rows carry undeclared
|
| 31 |
-
envelope keys (`timestamp`, `input`) that no metadata call mentions;
|
| 32 |
-
2. that a declared field POPULATES β Bright Data's Instagram Reels *declares* `views: number`
|
| 33 |
-
and delivers an account-grain wrong number (Β§4e). **Declared is not delivered**, and the one
|
| 34 |
-
TikTok claim that matters most (`play_count`) is exactly a declaration.
|
| 35 |
-
"""
|
| 36 |
-
from __future__ import annotations
|
| 37 |
-
|
| 38 |
-
import automation_engine as engine
|
| 39 |
-
# β T09 β FOUR NAMES CAME OFF THIS LIST AND NOTHING BROKE, which is the point of deriving an
|
| 40 |
-
# import block from the AST both ways. `bd_call`, `bd_filter_start`, `bd_key` and `bd_ready` were
|
| 41 |
-
# imported here under a comment claiming they were *"re-exported for the runners"*; no runner ever
|
| 42 |
-
# read them off this module (the engine imports them from the transport itself), so they were four
|
| 43 |
-
# lines of dependency nobody was paying for. `[[artifact-with-no-importer]]` in its smallest form.
|
| 44 |
-
from connectors_bd import (
|
| 45 |
-
_bd_first_url,
|
| 46 |
-
_bd_flag,
|
| 47 |
-
_bd_list,
|
| 48 |
-
_bd_source_payload,
|
| 49 |
-
_first,
|
| 50 |
-
_ig_int,
|
| 51 |
-
bd_scrape,
|
| 52 |
-
)
|
| 53 |
-
|
| 54 |
-
# ---------------------------------------------------------------------------------------------
|
| 55 |
-
# THE DATASETS
|
| 56 |
-
# ---------------------------------------------------------------------------------------------
|
| 57 |
-
# β CATALOGUE PRESENCE IS NOT ENTITLEMENT β the same finding Instagram produced. `GET
|
| 58 |
-
# /datasets/list` returned 1,735 rows of which 12 are TikTok; the three below answer 200 with a
|
| 59 |
-
# full field list, and the two DISCOVERY halves answer **404 for our key**:
|
| 60 |
-
# `gd_lj71gn6l68bz7y9hc` (posts by profile) and `gd_lilwhto81z415d9mdl` (posts by keyword).
|
| 61 |
-
# β TikTok discovery routes through the PROFILES dataset's corpus filter, exactly as Instagram's
|
| 62 |
-
# does. A ticket that reaches for a by-keyword endpoint is reaching for a 404.
|
| 63 |
-
TT_DS_PROFILES = "gd_l1villgoiiidt09ci" # TikTok - Profiles. 40 fields, 152,000,000 records
|
| 64 |
-
TT_DS_POSTS = "gd_lu702nij2f790tmv9h" # TikTok - Posts. 43 fields
|
| 65 |
-
TT_DS_COMMENTS = "gd_lkf2st302ap89utw5k" # TikTok - Comments. 17 fields
|
| 66 |
-
|
| 67 |
-
#: The vendor's two post-type tokens, verbatim from the dataset's own `ai_description`
|
| 68 |
-
#: (*"strictly these two"*, video = 99.5% of rows). Ours are `image`/`video`/`carousel`.
|
| 69 |
-
TT_POST_TYPE_VIDEO = "video"
|
| 70 |
-
TT_POST_TYPE_CONTENT = "content"
|
| 71 |
-
|
| 72 |
-
#: What a TikTok profile URL looks like, for the runner that has a handle and needs a URL. Kept
|
| 73 |
-
#: beside the dataset ids because it is the same class of vendor fact.
|
| 74 |
-
#: β ONE SPELLING. `platform/core/user_tables.profile_url(handle, 'tiktok')` builds the identical
|
| 75 |
-
#: string from `_PROFILE_RULES` (contract C2, E's half) β this exists for the connector's own
|
| 76 |
-
#: batch calls, and the gate asserts the two agree rather than trusting that they do.
|
| 77 |
-
TT_PROFILE_URL = "https://www.tiktok.com/@{handle}"
|
| 78 |
-
|
| 79 |
-
|
| 80 |
-
def tt_profile_url(handle):
|
| 81 |
-
"""`nurilab` β `https://www.tiktok.com/@nurilab`. `''` for a blank handle, never a bare `@`."""
|
| 82 |
-
h = str(handle or "").strip().lstrip("@")
|
| 83 |
-
return TT_PROFILE_URL.format(handle=h) if h else ""
|
| 84 |
-
|
| 85 |
-
|
| 86 |
-
# ---------------------------------------------------------------------------------------------
|
| 87 |
-
# THE FIELD MAPS
|
| 88 |
-
# ---------------------------------------------------------------------------------------------
|
| 89 |
-
# Each function turns ONE vendor row into the cell dict for one of the `ut_tt_*` schemas declared
|
| 90 |
-
# in `automation_engine`. The rules they all obey, stated once:
|
| 91 |
-
#
|
| 92 |
-
# * **BLANK MEANS NOT READ, NEVER "THEY HAVE NONE".** A key the vendor did not send is OMITTED,
|
| 93 |
-
# so a later, richer pull fills it instead of being overwritten by this one's silence. `_first`
|
| 94 |
-
# returns `None` (never 0) when nothing matches, which is what makes that possible.
|
| 95 |
-
# * **A zero from the vendor is a MEASUREMENT and survives.** (Instagram's `_ig_zero_is_blank`
|
| 96 |
-
# rule is scoped to a paid rung whose zeros were proven fictional; nothing here has earned it.)
|
| 97 |
-
# * **Every unpromoted vendor key stays whole in `source_payload`.** A schema addition on the
|
| 98 |
-
# vendor's side is preserved rather than silently discarded while our column model catches up.
|
| 99 |
-
# * **Nothing here writes a session token.** `tt_chain_token`, `secu_id` (~85% null), `short_id`
|
| 100 |
-
# (100% null in the sample), `ftc` (100% null) and `relation` are deliberately unmapped; they
|
| 101 |
-
# ride in `source_payload` where they make no claim.
|
| 102 |
-
|
| 103 |
-
|
| 104 |
-
def _tt_str(node, *names):
|
| 105 |
-
"""The first non-empty string among `names`, or None when the vendor sent nothing.
|
| 106 |
-
|
| 107 |
-
β `None`, NOT `""`. The callers below drop `None` keys, which is what keeps a blank honest β
|
| 108 |
-
an empty string written into a cell claims "we looked and it is empty".
|
| 109 |
-
"""
|
| 110 |
-
v = _first(node, *names)
|
| 111 |
-
if v is None:
|
| 112 |
-
return None
|
| 113 |
-
s = str(v).strip()
|
| 114 |
-
return s or None
|
| 115 |
-
|
| 116 |
-
|
| 117 |
-
def _tt_pct(node, *names):
|
| 118 |
-
"""A vendor 0β1 engagement fraction β our stored 0β100 percentage, or None.
|
| 119 |
-
|
| 120 |
-
β THE Γ100 IS NOT COSMETIC (wave 26, amendment C1-a). Our `pct` renderer appends the sign to
|
| 121 |
-
the STORED number, so writing the vendor's raw 0.0656 would print a 6.6% creator as `0.0%` β
|
| 122 |
-
measured on the Instagram side, and TikTok sends the same shape on all three of its rates.
|
| 123 |
-
"""
|
| 124 |
-
v = _first(node, *names)
|
| 125 |
-
if v is None:
|
| 126 |
-
return None
|
| 127 |
-
out = engine._pct100(v)
|
| 128 |
-
return out or None
|
| 129 |
-
|
| 130 |
-
|
| 131 |
-
def _tt_day(node, *names):
|
| 132 |
-
"""A vendor stamp β `YYYY-MM-DD`, or None. Our `date` columns store a day."""
|
| 133 |
-
v = _first(node, *names)
|
| 134 |
-
if v is None:
|
| 135 |
-
return None
|
| 136 |
-
return engine._day(v) or None
|
| 137 |
-
|
| 138 |
-
|
| 139 |
-
def _drop_blanks(row):
|
| 140 |
-
"""The one place a mapped row loses its `None`s β see the BLANK MEANS NOT READ rule above."""
|
| 141 |
-
return {k: v for k, v in row.items() if v is not None and v != ""}
|
| 142 |
-
|
| 143 |
-
|
| 144 |
-
def normalize_profile(node, handle=""):
|
| 145 |
-
"""A TikTok Profiles row β the `ut_tt_profile` / `ut_tt_snapshots` cell shape.
|
| 146 |
-
|
| 147 |
-
β TWO FIELDS READ THROUGH A CANDIDATE PAIR, and both pairs are the vendor's, not a guess:
|
| 148 |
-
* `biography` is PRIMARY and `signature` the FALLBACK β the probe measured `signature`
|
| 149 |
-
populated on 85% of rows and they carry the same text;
|
| 150 |
-
* `region` is PRIMARY and `country` the FALLBACK β `region` is the one with a documented
|
| 151 |
-
two-letter-ISO description, `country` has no description at all.
|
| 152 |
-
β `videos_count` is mapped to `posts_count` with a caveat recorded rather than hidden: its
|
| 153 |
-
`ai_description` ranges 1-89, so it may be a WINDOW rather than a lifetime total. It is the
|
| 154 |
-
only count of its kind the dataset offers.
|
| 155 |
-
"""
|
| 156 |
-
node = node if isinstance(node, dict) else {}
|
| 157 |
-
account = _tt_str(node, "account_id") or str(handle or "").strip().lstrip("@")
|
| 158 |
-
return _drop_blanks({
|
| 159 |
-
"platform": engine.PLATFORM_TIKTOK,
|
| 160 |
-
"handle": account,
|
| 161 |
-
"full_name": _tt_str(node, "nickname"),
|
| 162 |
-
"tt_id": _tt_str(node, "id"),
|
| 163 |
-
"profile_url": _tt_str(node, "url") or (tt_profile_url(account) or None),
|
| 164 |
-
"bio": _tt_str(node, "biography", "signature"),
|
| 165 |
-
"external_url": _bd_first_url(_first(node, "bio_link")),
|
| 166 |
-
"verified": _bd_flag(node, "is_verified"),
|
| 167 |
-
"is_private": _bd_flag(node, "is_private"),
|
| 168 |
-
# β APPROXIMATE, and the word is the vendor's: `is_commerce_user` has *"many null values"*
|
| 169 |
-
# on their own description. It is the closest thing TikTok has to Instagram's
|
| 170 |
-
# `is_business_account`, and `_bd_flag` writes nothing at all when the key is absent β so
|
| 171 |
-
# the approximation only ever fills a cell the vendor actually answered.
|
| 172 |
-
"is_business": _bd_flag(node, "is_commerce_user"),
|
| 173 |
-
"followers": _ig_int(_first(node, "followers")),
|
| 174 |
-
"following": _ig_int(_first(node, "following")),
|
| 175 |
-
"posts_count": _ig_int(_first(node, "videos_count")),
|
| 176 |
-
# β `likes` on a PROFILE row is likes RECEIVED across the account's videos (18-110,200, no
|
| 177 |
-
# nulls). It is not a post-level number and it is not our `likes` column, which is why it
|
| 178 |
-
# is stored under a different name.
|
| 179 |
-
"likes_received": _ig_int(_first(node, "likes")),
|
| 180 |
-
"avg_engagement": _tt_pct(node, "awg_engagement_rate"),
|
| 181 |
-
"like_engagement": _tt_pct(node, "like_engagement_rate"),
|
| 182 |
-
"comment_engagement": _tt_pct(node, "comment_engagement_rate"),
|
| 183 |
-
"country_code": _tt_str(node, "region", "country"),
|
| 184 |
-
"region": _tt_str(node, "region"),
|
| 185 |
-
"predicted_lang": _tt_str(node, "predicted_lang"),
|
| 186 |
-
# β ACCOUNT AGE, NOT A MEASUREMENT STAMP β `create_time` on a profile is when the ACCOUNT
|
| 187 |
-
# was made. TikTok stamps nothing with "when this number was true", exactly like Instagram,
|
| 188 |
-
# which is why the append law dates a snapshot by when WE read it.
|
| 189 |
-
"account_created_at": _tt_day(node, "create_time"),
|
| 190 |
-
"source_payload": _bd_source_payload(node),
|
| 191 |
-
})
|
| 192 |
-
|
| 193 |
-
|
| 194 |
-
def tt_post_type(node):
|
| 195 |
-
"""A TikTok post row β one of OUR three type options, or None.
|
| 196 |
-
|
| 197 |
-
β THE FIRST OF THE PROBE DOC'S TWO NAMED BLOCKERS. The vendor's vocabulary is `"video"` /
|
| 198 |
-
`"content"`; ours is `image` / `video` / `carousel` and has no `"content"`. Writing the
|
| 199 |
-
vendor's token would fail `_clean_field`'s option check on the way in and would put an
|
| 200 |
-
untranslated API word in front of a user on the way out.
|
| 201 |
-
|
| 202 |
-
So: `video` is `video`, and `content` β TikTok's photo-mode post β is `image`, EXCEPT when the
|
| 203 |
-
row carries more than one `carousel_images` entry, which is what a carousel IS on either
|
| 204 |
-
network. β The multi-image branch is decided from the IMAGES, never from the type token: the
|
| 205 |
-
token cannot express it, so inferring `carousel` from the word would be inventing a fact.
|
| 206 |
-
β An UNKNOWN token returns None rather than defaulting to `video` (99.5% of rows are video, and
|
| 207 |
-
that is exactly what would make the wrong default invisible).
|
| 208 |
-
"""
|
| 209 |
-
raw = str((node or {}).get("post_type") or "").strip().lower()
|
| 210 |
-
if raw == TT_POST_TYPE_VIDEO:
|
| 211 |
-
return "video"
|
| 212 |
-
if raw != TT_POST_TYPE_CONTENT:
|
| 213 |
-
return None
|
| 214 |
-
images = (node or {}).get("carousel_images")
|
| 215 |
-
return "carousel" if isinstance(images, list) and len(images) > 1 else "image"
|
| 216 |
-
|
| 217 |
-
|
| 218 |
-
def normalize_post(node):
|
| 219 |
-
"""A TikTok Posts row β the `ut_tt_posts` cell shape. None when it carries no identity.
|
| 220 |
-
|
| 221 |
-
β THE SECOND NAMED BLOCKER, RESOLVED HERE AND NOWHERE ELSE: `play_count` is ONE number and
|
| 222 |
-
Instagram's schema has TWO columns for it (`plays` and `views`). On TikTok they are the same
|
| 223 |
-
fact β `play_count` IS the count TikTok displays under a video β so it maps to `views` and
|
| 224 |
-
`ut_tt_posts` HAS NO `plays` COLUMN. Copying one vendor number into two of our columns would
|
| 225 |
-
manufacture a second measurement that a rollup could average or double-count, which is a worse
|
| 226 |
-
outcome than the missing column it would paper over.
|
| 227 |
-
|
| 228 |
-
β `shortcode` reads `shortcode` then `post_id`: both are 19-digit numerics on this dataset and
|
| 229 |
-
the probe measured them as the same shape. That equality is what lets the commentsβposts link
|
| 230 |
-
join with NO normaliser, which the Instagram side never had.
|
| 231 |
-
β `num_share_count` (a number) is preferred over `share_count` (typed TEXT by the vendor).
|
| 232 |
-
β `commerce_info` is a business/commerce LOCATION per its own description β cities and
|
| 233 |
-
countries. It is NOT a paid-partnership flag, and nothing in this dataset is: TikTok declares
|
| 234 |
-
no equivalent, so `paid_partnership`/`partner` have no column on this family at all.
|
| 235 |
-
"""
|
| 236 |
-
node = node if isinstance(node, dict) else {}
|
| 237 |
-
shortcode = _tt_str(node, "shortcode", "post_id")
|
| 238 |
-
if not shortcode:
|
| 239 |
-
return None
|
| 240 |
-
return _drop_blanks({
|
| 241 |
-
"platform": engine.PLATFORM_TIKTOK,
|
| 242 |
-
"shortcode": shortcode,
|
| 243 |
-
# ββ `account_id`, NOT `profile_username` β MEASURED on a real Posts row 2026-08-12.
|
| 244 |
-
# The vendor's `profile_username` is the DISPLAY NAME (`"Dina"`), while `account_id` is the
|
| 245 |
-
# @handle (`"d1na_th"`) β the same field `normalize_profile` already reads for `handle`, so
|
| 246 |
-
# one name means one thing across both corpora. Reading the display name silently broke the
|
| 247 |
-
# only join this table has: `ut_tt_posts.influencer_key` -> `ut_tt_profile.handle` matched
|
| 248 |
-
# NOTHING, so a person could not filter posts by creator and a rollup would count zero.
|
| 249 |
-
# β NO FALLBACK TO `profile_username`, deliberately. It is not a degraded handle, it is a
|
| 250 |
-
# different fact, and filling a join key with it is worse than leaving it blank β a blank
|
| 251 |
-
# is visibly missing, a display name looks like an answer [[one-question-two-normalizers]].
|
| 252 |
-
# The URL is the honest second source: it carries the handle by construction.
|
| 253 |
-
"influencer_key": (_tt_str(node, "account_id")
|
| 254 |
-
or tt_handle(_tt_str(node, "profile_url", "url") or "") or None),
|
| 255 |
-
"posted_at": _tt_day(node, "create_time"),
|
| 256 |
-
"type": tt_post_type(node),
|
| 257 |
-
"caption": _tt_str(node, "description"),
|
| 258 |
-
"url": _tt_str(node, "url"),
|
| 259 |
-
"hashtags": _bd_list(node, "hashtags"),
|
| 260 |
-
"tagged_location": _tt_str(node, "commerce_info"),
|
| 261 |
-
"views": _ig_int(_first(node, "play_count")),
|
| 262 |
-
"likes": _ig_int(_first(node, "digg_count")),
|
| 263 |
-
"comments": _ig_int(_first(node, "comment_count")),
|
| 264 |
-
"shares": _ig_int(_first(node, "num_share_count")),
|
| 265 |
-
"saves": _ig_int(_first(node, "collect_count")),
|
| 266 |
-
"video_duration": _ig_int(_first(node, "video_duration")),
|
| 267 |
-
"source_payload": _bd_source_payload(node),
|
| 268 |
-
})
|
| 269 |
-
|
| 270 |
-
|
| 271 |
-
def normalize_comment(node):
|
| 272 |
-
"""A TikTok Comments row β the `ut_tt_comments` cell shape. None without a comment id.
|
| 273 |
-
|
| 274 |
-
β THE NAME COLLISION, RESOLVED: TikTok's `replies` is an ARRAY of reply objects and OUR
|
| 275 |
-
`replies` column is an INT count. The count comes from `num_replies`; the array stays whole in
|
| 276 |
-
`source_payload`. Reading the array's length instead would be a second, disagreeing answer to
|
| 277 |
-
a question the vendor already answers β and it would disagree, because a page of replies is not
|
| 278 |
-
all of them.
|
| 279 |
-
β `date_created` is typed `date` by this vendor, unlike Instagram's `comment_date` which is
|
| 280 |
-
text and needs defensive parsing. It still goes through `_tt_day` β one date path, so a vendor
|
| 281 |
-
that changes its mind cannot change ours.
|
| 282 |
-
β The comment TEXT and every identifiable commenter field (`commenter_user_name` is flagged
|
| 283 |
-
PII) stay in `source_payload` and are promoted to no column, which is the same posture the
|
| 284 |
-
Instagram comment schema takes for the same D-24 reason.
|
| 285 |
-
"""
|
| 286 |
-
node = node if isinstance(node, dict) else {}
|
| 287 |
-
comment_key = _tt_str(node, "comment_id")
|
| 288 |
-
if not comment_key:
|
| 289 |
-
return None
|
| 290 |
-
return _drop_blanks({
|
| 291 |
-
"platform": engine.PLATFORM_TIKTOK,
|
| 292 |
-
"comment_key": comment_key,
|
| 293 |
-
"shortcode": _tt_str(node, "post_id"),
|
| 294 |
-
# ββ OWNER RULING 2026-08-12: the comment's CONTENT gets a column. `comment_text` is the
|
| 295 |
-
# vendor's own key and `comment_text_only` its stripped variant (the probe recorded both);
|
| 296 |
-
# primary first, so a row carrying the rich form is not silently served the plain one.
|
| 297 |
-
# β The commenter's identity is deliberately NOT promoted β see `TT_COMMENT_FIELDS`.
|
| 298 |
-
"text": _tt_str(node, "comment_text", "comment_text_only"),
|
| 299 |
-
"commented_at": _tt_day(node, "date_created"),
|
| 300 |
-
"likes": _ig_int(_first(node, "num_likes")),
|
| 301 |
-
"replies": _ig_int(_first(node, "num_replies")),
|
| 302 |
-
"source_payload": _bd_source_payload(node),
|
| 303 |
-
})
|
| 304 |
-
|
| 305 |
-
|
| 306 |
-
#: β The three maps, addressable by name β so a gate (and the runners in T04-T06) can walk them
|
| 307 |
-
#: rather than naming three functions, and so adding a fourth dataset is one entry.
|
| 308 |
-
TT_NORMALIZERS = {
|
| 309 |
-
"tt_profile": normalize_profile,
|
| 310 |
-
"tt_post_metrics": normalize_post,
|
| 311 |
-
"tt_comments": normalize_comment,
|
| 312 |
-
}
|
| 313 |
-
|
| 314 |
-
|
| 315 |
-
# ---------------------------------------------------------------------------------------------
|
| 316 |
-
# THE FETCH β wave 30 Β· W30-T08 (carrying wave-29's dropped T05)
|
| 317 |
-
# ---------------------------------------------------------------------------------------------
|
| 318 |
-
|
| 319 |
-
def tt_handle(url):
|
| 320 |
-
"""A TikTok profile URL **or** a bare handle β the handle. `''` when it is neither.
|
| 321 |
-
|
| 322 |
-
Deliberately permissive about the input and strict about the output, because the two callers
|
| 323 |
-
hand it different things: an automation stores whatever a person typed in the profile column
|
| 324 |
-
(`@nurilab`, `nurilab`, or the full URL), while the discovery runner already holds a clean
|
| 325 |
-
`account_id`. One normaliser, so a row found by discovery and a row typed by hand cannot
|
| 326 |
-
resolve to two different handles.
|
| 327 |
-
"""
|
| 328 |
-
s = str(url or "").strip()
|
| 329 |
-
if not s:
|
| 330 |
-
return ""
|
| 331 |
-
if "tiktok.com" in s.lower():
|
| 332 |
-
# Everything after the first `@`, up to the next path segment or query.
|
| 333 |
-
tail = s.split("@", 1)[1] if "@" in s else ""
|
| 334 |
-
s = tail.split("/")[0].split("?")[0].split("#")[0]
|
| 335 |
-
s = s.strip().lstrip("@").strip()
|
| 336 |
-
# A handle is the vendor's `account_id` shape: alphanumerics, dots and underscores.
|
| 337 |
-
return s if s and all(c.isalnum() or c in "._" for c in s) else ""
|
| 338 |
-
|
| 339 |
-
|
| 340 |
-
def tt_post_urls(node, limit=0):
|
| 341 |
-
"""β WAVE 30 Β· T10 β the profile row's own post permalinks, newest-first as the vendor sends.
|
| 342 |
-
|
| 343 |
-
β THIS IS WHY TIKTOK POST CAPTURE COSTS NO EXTRA DISCOVERY. `top_videos` rides the PROFILE row
|
| 344 |
-
we have already bought, so the posts read is a scrape of links we hold, never a search for them.
|
| 345 |
-
The two TikTok DISCOVERY datasets (posts-by-profile, posts-by-keyword) are **404 for our key**,
|
| 346 |
-
so a design that reached for either would not merely be dearer, it would not work.
|
| 347 |
-
|
| 348 |
-
ββ CORRECTED 2026-08-12 β THIS DOCSTRING USED TO SAY *"the probe MEASURED `top_videos` as an
|
| 349 |
-
array of video permalinks, NO empties"*, AND THAT SENTENCE IS WHAT SHIPPED THE BUG. The probe
|
| 350 |
-
read `/datasets/{id}/metadata` β a DATASET description β and the phrase quoted was the field's
|
| 351 |
-
`ai_description`, not an observation of a row. The real row sends **dicts keyed `video_url`**
|
| 352 |
-
(measured below), so the reader built against the quoted sentence found nothing, forever, in
|
| 353 |
-
silence. β The transferable half: *"the probe measured X"* and *"the probe read a declaration
|
| 354 |
-
of X"* are different claims, and prose cannot be told apart by a reader downstream β which is
|
| 355 |
-
why the correction names the method, not just the value.
|
| 356 |
-
|
| 357 |
-
β `top_posts_data` is deliberately NOT read: the probe calls it *"a thin dup of `top_videos`"*,
|
| 358 |
-
and preferring whichever happened to be longer is how one creator's window silently differs
|
| 359 |
-
from another's.
|
| 360 |
-
β `limit <= 0` means "everything the row carried". The CAP IS THE CALLER'S β `config.maxPosts`,
|
| 361 |
-
validated 1..12 β and it is applied here rather than after the scrape so an unwanted post is
|
| 362 |
-
never bought. [[a-constant-two-features-share]]: the 12 is the vendor's measured profile window,
|
| 363 |
-
not a number this function may invent.
|
| 364 |
-
"""
|
| 365 |
-
raw = (node or {}).get("top_videos")
|
| 366 |
-
out = []
|
| 367 |
-
for item in raw if isinstance(raw, list) else []:
|
| 368 |
-
# ββ MEASURED ON A REAL ROW 2026-08-12, AND IT IS NOT WHAT THE SCHEMA SAID.
|
| 369 |
-
# `top_videos` is NOT an array of permalink strings. One paid TikTok Profiles scrape of a
|
| 370 |
-
# live handle returns **19 DICTS**, keyed
|
| 371 |
-
# `video_url Β· video_id Β· playcount Β· diggcount Β· commentcount Β· share_count Β·
|
| 372 |
-
# favorites_count Β· create_date Β· cover_image`.
|
| 373 |
-
# The docstring above cites the $0 probe as having "measured" permalinks β it had not, and
|
| 374 |
-
# could not: `/datasets/{id}/metadata` describes a DATASET, and the probe's own verdict says
|
| 375 |
-
# so in terms (*"what $0 cannot buy β¦ the real ROW shape β¦ that a declared field
|
| 376 |
-
# POPULATES"*, `wave29/proto/tiktok-schema.md`). This is the SECOND time this vendor's
|
| 377 |
-
# declaration has diverged from its delivery on this exact axis; BD's IG Reels `views` was
|
| 378 |
-
# the first. [[reachable-is-not-the-same-as-built]]
|
| 379 |
-
# β The consequence, live: `TT_DS_POSTS` was never reached, because this returned an EMPTY
|
| 380 |
-
# list on every real profile β post capture could not have worked for anybody, and the
|
| 381 |
-
# T10 gate stayed green because its canned fixture encoded the DECLARED shape. A fixture
|
| 382 |
-
# written from a schema tests the schema.
|
| 383 |
-
# β `video_url` is FIRST because it is the key the vendor actually sends; `url` is kept
|
| 384 |
-
# because it costs nothing and is what a future corpus revision would most likely use. The
|
| 385 |
-
# bare-string branch stays for the same reason β this widens what is ACCEPTED and invents
|
| 386 |
-
# nothing: a shape that yields no `httpβ¦` value still degrades to "no posts", exactly as
|
| 387 |
-
# before, rather than to a URL built out of a guess.
|
| 388 |
-
# β `top_posts_data` is STILL not read (it carries `post_url` and would work): preferring
|
| 389 |
-
# whichever array happened to be longer is how one creator's window silently differs from
|
| 390 |
-
# another's, and that reasoning is unchanged by this correction.
|
| 391 |
-
if isinstance(item, dict):
|
| 392 |
-
u = str(item.get("video_url") or item.get("url") or "").strip()
|
| 393 |
-
else:
|
| 394 |
-
u = str(item or "").strip()
|
| 395 |
-
if u.lower().startswith("http") and u not in out:
|
| 396 |
-
out.append(u)
|
| 397 |
-
return out[:limit] if limit and limit > 0 else out
|
| 398 |
-
|
| 399 |
-
|
| 400 |
-
def pull_posts_tt(post_urls, log=print, deferred=None):
|
| 401 |
-
"""The TikTok Posts dataset for a list of permalinks β `(rows, note)`, already normalised.
|
| 402 |
-
|
| 403 |
-
β ONE CALL FOR THE WHOLE WINDOW. `bd_scrape` has always taken a list, and the Instagram side
|
| 404 |
-
measured what happens when a caller forgets: 25 records, one billed snapshot each, a walk still
|
| 405 |
-
running at 67 minutes. Nothing here loops per URL.
|
| 406 |
-
"""
|
| 407 |
-
urls = [str(u) for u in (post_urls or []) if str(u or "").strip()]
|
| 408 |
-
if not urls:
|
| 409 |
-
return [], ""
|
| 410 |
-
rows, note = bd_scrape(TT_DS_POSTS, urls, deferred=deferred)
|
| 411 |
-
if note:
|
| 412 |
-
log(f"[aios-tt] posts: {note}")
|
| 413 |
-
return [], note
|
| 414 |
-
out = [r for r in (normalize_post(n) for n in rows) if r]
|
| 415 |
-
return out, ""
|
| 416 |
-
|
| 417 |
-
|
| 418 |
-
def pull_comments_tt(post_urls, log=print, deferred=None):
|
| 419 |
-
"""The TikTok Comments dataset for a list of POST permalinks β `(rows, note)`, normalised.
|
| 420 |
-
|
| 421 |
-
β THE MOST EXPENSIVE THING THIS PRODUCT BUYS, and the reason `commentMetrics` defaults OFF on
|
| 422 |
-
both networks: a comments scrape ingests identifiable third parties who never entered anybody's
|
| 423 |
-
list (D-24). The mapper already keeps every commenter field in `source_payload` and promotes
|
| 424 |
-
none of them to a column; this function adds no new exposure, it just has to be asked for.
|
| 425 |
-
"""
|
| 426 |
-
urls = [str(u) for u in (post_urls or []) if str(u or "").strip()]
|
| 427 |
-
if not urls:
|
| 428 |
-
return [], ""
|
| 429 |
-
rows, note = bd_scrape(TT_DS_COMMENTS, urls, deferred=deferred)
|
| 430 |
-
if note:
|
| 431 |
-
log(f"[aios-tt] comments: {note}")
|
| 432 |
-
return [], note
|
| 433 |
-
out = [r for r in (normalize_comment(n) for n in rows) if r]
|
| 434 |
-
return out, ""
|
| 435 |
-
|
| 436 |
-
|
| 437 |
-
#: ββ WAVE 30 Β· D-156 β THE MEDIA DATASETS, AS A SET, SO THE HAND-OFF CAN FILTER ON IDENTITY.
|
| 438 |
-
#: `_media_deferrals` uses this to lift ONLY posts/comments snapshots out of the local deferral
|
| 439 |
-
#: list. That is what makes it structurally impossible to file a PROFILE snapshot in the engine's
|
| 440 |
-
#: metric queue β the defect a draft of T10 shipped and A-39 booked as "the wrong fix is worse
|
| 441 |
-
#: than the gap". A membership test cannot be got wrong by a later edit the way `if` order can.
|
| 442 |
-
TT_MEDIA_DATASETS = (TT_DS_POSTS, TT_DS_COMMENTS)
|
| 443 |
-
|
| 444 |
-
|
| 445 |
-
def _media_deferrals(deferred):
|
| 446 |
-
"""The POSTS/COMMENTS entries of a `bd_scrape` deferral list β never the profile's.
|
| 447 |
-
|
| 448 |
-
β The engine, not this module, decides what a deferral MEANS: it stamps `kind` and the
|
| 449 |
-
handle and files it. TikTok needs no `_tag_metric_deferrals` twin because one dataset is one
|
| 450 |
-
kind here, so the id already carries everything a mapper choice depends on β and importing
|
| 451 |
-
Instagram's tagger is not available anyway (W30-T09 gates ZERO `from connectors_ig` lines).
|
| 452 |
-
"""
|
| 453 |
-
out = []
|
| 454 |
-
for d in deferred or []:
|
| 455 |
-
if isinstance(d, dict) and str(d.get("datasetId") or "") in TT_MEDIA_DATASETS:
|
| 456 |
-
out.append(dict(d))
|
| 457 |
-
return out
|
| 458 |
-
|
| 459 |
-
|
| 460 |
-
def pull_profile_tt(url, log=print, pending_profile=None, prefetch=None,
|
| 461 |
-
max_posts=0, post_metrics=False, comment_metrics=False):
|
| 462 |
-
"""ONE TikTok profile from the vendor. Same return contract as `pull_profile`.
|
| 463 |
-
|
| 464 |
-
`{state, profile, posts, comments, via, note}` with `state β ok | partial | blocked | error`,
|
| 465 |
-
so the engine's enrich branch treats every network identically and no caller learns a new
|
| 466 |
-
shape.
|
| 467 |
-
|
| 468 |
-
β WAVE 30 Β· T10 β POSTS AND COMMENTS ARE REAL NOW, AND BOTH DEFAULT OFF, exactly as Instagram's
|
| 469 |
-
do. `post_metrics` scrapes the profile row's own `top_videos` permalinks (see `tt_post_urls` β
|
| 470 |
-
no discovery call, because both TikTok discovery datasets 404 for our key); `comment_metrics`
|
| 471 |
-
then scrapes the comments of the posts that came back. β COMMENTS REQUIRE POSTS by construction
|
| 472 |
-
rather than by a rule: their input IS a post permalink, so asking for comments with post capture
|
| 473 |
-
off is a request with no subject, and it returns none instead of quietly buying posts nobody
|
| 474 |
-
asked for.
|
| 475 |
-
|
| 476 |
-
β `partial` IS THE SUCCESS STATE WHENEVER NO MEDIA WAS READ, and that is deliberate rather than
|
| 477 |
-
pessimistic. The Instagram contract reads `ok` only when identity AND media both landed
|
| 478 |
-
(`pull_profile_bd`: *"identity without media is still partial ... a run that wrote a follower
|
| 479 |
-
count and no posts must not paint green over a posts table that did not grow"*). So: posts not
|
| 480 |
-
ASKED for β `partial`, saying so; posts asked for and landed β `ok`; asked for and none came β
|
| 481 |
-
`partial` with the vendor's reason. The state answers "did this pull deliver what it went for",
|
| 482 |
-
never "did the function finish".
|
| 483 |
-
|
| 484 |
-
β **NO FREE RUNG, AND NO FALLBACK CHAIN.** Instagram's `pull_profile` drops to Apify when the
|
| 485 |
-
paid rung refuses; `providers.DEFAULT_CHAINS["tt_profile"]` is deliberately single-provider,
|
| 486 |
-
with its own note explaining that a multi-provider chain is a promise something walks it and
|
| 487 |
-
that nothing walks Instagram's second name today either. So a refusal here is final, and it
|
| 488 |
-
says so instead of implying a retry somewhere.
|
| 489 |
-
"""
|
| 490 |
-
handle = tt_handle(url)
|
| 491 |
-
if not handle:
|
| 492 |
-
return {"state": "error", "profile": {}, "posts": [], "comments": [], "via": "",
|
| 493 |
-
"note": f"{url!r} is not a TikTok profile URL or handle"}
|
| 494 |
-
|
| 495 |
-
# β THE BATCH FAST PATH, same shape as the Instagram side: `prefetch` is `{handle: node}` from
|
| 496 |
-
# one multi-URL scrape covering a whole selection. A hit is a vendor round trip that does not
|
| 497 |
-
# happen; a miss falls through to the single-URL call below.
|
| 498 |
-
cached = prefetch.get(handle) if isinstance(prefetch, dict) else None
|
| 499 |
-
_deferred = []
|
| 500 |
-
if isinstance(cached, dict) and cached:
|
| 501 |
-
rows, note = [cached], ""
|
| 502 |
-
else:
|
| 503 |
-
rows, note = bd_scrape(TT_DS_PROFILES, [tt_profile_url(handle)], deferred=_deferred)
|
| 504 |
-
|
| 505 |
-
node = rows[0] if rows else {}
|
| 506 |
-
profile = normalize_profile(node, handle) if node else {}
|
| 507 |
-
# β THE READABILITY TEST IS `followers`/`following`, NOT "did we get a dict". `normalize_profile`
|
| 508 |
-
# drops blanks, so an unreadable row still returns `{"platform": β¦, "handle": β¦}` β truthy, and
|
| 509 |
-
# carrying nothing anybody asked for. The Instagram rung tests exactly this pair for exactly
|
| 510 |
-
# this reason, and answering "0 followers" instead is the failure it exists to prevent.
|
| 511 |
-
unreadable = profile.get("followers") is None and profile.get("following") is None
|
| 512 |
-
if note or unreadable:
|
| 513 |
-
# β THE DEFERRAL IS HANDED OVER RATHER THAN DISCARDED. A snapshot the vendor is still
|
| 514 |
-
# building HAS ALREADY BEEN PAID FOR; dropping its id bills again on the next run for the
|
| 515 |
-
# same record. That was live on the Instagram profile path until 2026-08-09 β measured on
|
| 516 |
-
# nurilab as two runs, two fresh snapshots, both abandoned β and it is not being
|
| 517 |
-
# reintroduced here by omission.
|
| 518 |
-
if isinstance(pending_profile, list):
|
| 519 |
-
for d in _deferred:
|
| 520 |
-
pending_profile.append({**d, "kind": "profile", "influencer": handle})
|
| 521 |
-
why = note or ("the scrape answered, but no follower/following counts were readable in it "
|
| 522 |
-
"(the field names may have moved - see tiktok-capture.md)")
|
| 523 |
-
return {"state": "blocked", "profile": {}, "posts": [], "comments": [], "via": "brightdata",
|
| 524 |
-
"deferredProfile": [d.get("snapshotId") for d in _deferred],
|
| 525 |
-
"note": why}
|
| 526 |
-
|
| 527 |
-
# --- W30-T10: THE MEDIA, ONLY WHEN IT WAS ASKED FOR. ------------------------------------
|
| 528 |
-
if not post_metrics:
|
| 529 |
-
return {"state": "partial", "profile": profile, "posts": [], "comments": [],
|
| 530 |
-
"via": "brightdata",
|
| 531 |
-
"note": note or "profile read; post capture is off for this step"}
|
| 532 |
-
urls = tt_post_urls(node, limit=max_posts)
|
| 533 |
-
if not urls:
|
| 534 |
-
# β NOT AN ERROR AND NOT A RETRY. A creator with no `top_videos` has nothing to buy, and
|
| 535 |
-
# saying so is what stops the next run paying to be told the same thing.
|
| 536 |
-
return {"state": "partial", "profile": profile, "posts": [], "comments": [],
|
| 537 |
-
"via": "brightdata",
|
| 538 |
-
"note": note or "profile read; this account's row carried no post links"}
|
| 539 |
-
posts, p_note = pull_posts_tt(urls, log=log, deferred=_deferred)
|
| 540 |
-
comments, c_note = ([], "")
|
| 541 |
-
if comment_metrics and posts:
|
| 542 |
-
# The comments dataset is keyed on a POST permalink, so it reads the posts we just bought β
|
| 543 |
-
# `url` from the mapper, never the profile's raw array, so a post the posts scrape refused
|
| 544 |
-
# is not silently asked about again one rung later.
|
| 545 |
-
comments, c_note = pull_comments_tt([p.get("url") for p in posts if p.get("url")],
|
| 546 |
-
log=log, deferred=_deferred)
|
| 547 |
-
# ββ WAVE 30 Β· D-156 β THE MEDIA DEFERRALS ARE HANDED BACK, and the shape of the hand-off is
|
| 548 |
-
# the whole lesson. An earlier draft of T10 appended every `_deferred` entry to
|
| 549 |
-
# `pending_profile` tagged `kind: "profile"`. By the time control reaches here a PROFILE
|
| 550 |
-
# deferral is impossible β the profile branch above returns `blocked` on any note β so **every
|
| 551 |
-
# id fanned out that way was a POSTS or COMMENTS snapshot in the PROFILE queue**, whose
|
| 552 |
-
# collector writes preset profile cells onto somebody's record from post rows. The engine keeps
|
| 553 |
-
# the two queues apart deliberately (`_pending_profile_tasks` vs `_pending_metric_tasks`).
|
| 554 |
-
# β So this returns them under their OWN key, filtered by dataset identity
|
| 555 |
-
# (`_media_deferrals`), and the engine files them in the metric queue with the handle it
|
| 556 |
-
# already holds. Returning rather than appending also keeps the queue's vocabulary out of a
|
| 557 |
-
# connector: this module knows which CORPUS deferred, never what the engine calls it.
|
| 558 |
-
# β `deferredMedia` rides BOTH returns on purpose. The empty-posts case is the one that
|
| 559 |
-
# matters most β that is exactly the run where the vendor took too long, so a caller reading
|
| 560 |
-
# the ids only from the success path would lose every batch it actually paid for.
|
| 561 |
-
deferred_media = _media_deferrals(_deferred)
|
| 562 |
-
if not posts:
|
| 563 |
-
return {"state": "partial", "profile": profile, "posts": [], "comments": [],
|
| 564 |
-
"via": "brightdata", "deferredMedia": deferred_media,
|
| 565 |
-
"note": p_note or note or "profile read; the post source returned nothing"}
|
| 566 |
-
return {"state": "ok", "profile": profile, "posts": posts, "comments": comments,
|
| 567 |
-
"via": "brightdata", "deferredMedia": deferred_media,
|
| 568 |
-
"note": c_note or note or ""}
|
|
|
|
| 1 |
+
"""connectors_tt.py β the TIKTOK connector (wave 29 Β· item 7 Β· DEBT D-9 Β· rulings R1 + R2).
|
| 2 |
+
|
| 3 |
+
Everything in this file knows what a VENDOR's TikTok row looks like. Nothing in it knows what an
|
| 4 |
+
automation is. That split is `connectors_ig.py`'s (wave 27 item 23) and it is the reason this file
|
| 5 |
+
exists at all rather than another thousand lines inside the engine.
|
| 6 |
+
|
| 7 |
+
β **EVERY VENDOR FIELD NAME HERE WAS PROBED, NOT GUESSED.** The whole schema β 40 profile / 43
|
| 8 |
+
post / 17 comment fields, each with the vendor's own type, description and `pii` flag β was read
|
| 9 |
+
live from `GET /datasets/{id}/metadata` for **$0.00** and written down in
|
| 10 |
+
`.claude/wiki/waves/wave29/proto/tiktok-schema.md` (promoted to `tiktok-capture.md` at
|
| 11 |
+
close-out). That document is the AUTHORITY: do not re-probe it, and do not invent a key. Where a
|
| 12 |
+
name below reads through a candidate list it is because the vendor has two names for one fact
|
| 13 |
+
(`biography`/`signature`, `region`/`country`), never because the name is uncertain.
|
| 14 |
+
|
| 15 |
+
β **NOTHING HERE EVER AUTHENTICATES TO TIKTOK.** No login, no cookie, no account to get banned β
|
| 16 |
+
public data through a supplier, exactly the rail `connectors_ig.py` states for Instagram. The
|
| 17 |
+
vendor key is a key to a SUPPLIER.
|
| 18 |
+
|
| 19 |
+
β **THE TRANSPORT IS `connectors_bd.py` β SHARED, VENDOR-NAMED, AND NO LONGER BORROWED FROM THE
|
| 20 |
+
OTHER PLATFORM'S CONNECTOR** (WAVE 30 Β· T09, DEBT D-128). `bd_call`, `bd_scrape` and
|
| 21 |
+
`bd_filter_start` take the dataset id as a PARAMETER β they are Bright Data's wire, not
|
| 22 |
+
Instagram's β and re-implementing them here would be a second copy of the deferral handling, the
|
| 23 |
+
truncation guard, the SSRF rail and the snapshot-progress reader, i.e. five places for one bug.
|
| 24 |
+
Until wave 30 the code was right and the NAME was wrong: this file imported thirteen symbols from
|
| 25 |
+
`connectors_ig`, which read as a dependency on Instagram and was really a dependency on a supplier.
|
| 26 |
+
β **This file now imports ZERO names from `connectors_ig`, and a gate check asserts that**, because
|
| 27 |
+
the sentence above is the kind that quietly stops being true.
|
| 28 |
+
|
| 29 |
+
β **WHAT $0 COULD NOT BUY, so nobody reads this file as more measured than it is:**
|
| 30 |
+
1. the real ROW shape β `/metadata` describes a DATASET, and Instagram's rows carry undeclared
|
| 31 |
+
envelope keys (`timestamp`, `input`) that no metadata call mentions;
|
| 32 |
+
2. that a declared field POPULATES β Bright Data's Instagram Reels *declares* `views: number`
|
| 33 |
+
and delivers an account-grain wrong number (Β§4e). **Declared is not delivered**, and the one
|
| 34 |
+
TikTok claim that matters most (`play_count`) is exactly a declaration.
|
| 35 |
+
"""
|
| 36 |
+
from __future__ import annotations
|
| 37 |
+
|
| 38 |
+
import automation_engine as engine
|
| 39 |
+
# β T09 β FOUR NAMES CAME OFF THIS LIST AND NOTHING BROKE, which is the point of deriving an
|
| 40 |
+
# import block from the AST both ways. `bd_call`, `bd_filter_start`, `bd_key` and `bd_ready` were
|
| 41 |
+
# imported here under a comment claiming they were *"re-exported for the runners"*; no runner ever
|
| 42 |
+
# read them off this module (the engine imports them from the transport itself), so they were four
|
| 43 |
+
# lines of dependency nobody was paying for. `[[artifact-with-no-importer]]` in its smallest form.
|
| 44 |
+
from connectors_bd import (
|
| 45 |
+
_bd_first_url,
|
| 46 |
+
_bd_flag,
|
| 47 |
+
_bd_list,
|
| 48 |
+
_bd_source_payload,
|
| 49 |
+
_first,
|
| 50 |
+
_ig_int,
|
| 51 |
+
bd_scrape,
|
| 52 |
+
)
|
| 53 |
+
|
| 54 |
+
# ---------------------------------------------------------------------------------------------
|
| 55 |
+
# THE DATASETS
|
| 56 |
+
# ---------------------------------------------------------------------------------------------
|
| 57 |
+
# β CATALOGUE PRESENCE IS NOT ENTITLEMENT β the same finding Instagram produced. `GET
|
| 58 |
+
# /datasets/list` returned 1,735 rows of which 12 are TikTok; the three below answer 200 with a
|
| 59 |
+
# full field list, and the two DISCOVERY halves answer **404 for our key**:
|
| 60 |
+
# `gd_lj71gn6l68bz7y9hc` (posts by profile) and `gd_lilwhto81z415d9mdl` (posts by keyword).
|
| 61 |
+
# β TikTok discovery routes through the PROFILES dataset's corpus filter, exactly as Instagram's
|
| 62 |
+
# does. A ticket that reaches for a by-keyword endpoint is reaching for a 404.
|
| 63 |
+
TT_DS_PROFILES = "gd_l1villgoiiidt09ci" # TikTok - Profiles. 40 fields, 152,000,000 records
|
| 64 |
+
TT_DS_POSTS = "gd_lu702nij2f790tmv9h" # TikTok - Posts. 43 fields
|
| 65 |
+
TT_DS_COMMENTS = "gd_lkf2st302ap89utw5k" # TikTok - Comments. 17 fields
|
| 66 |
+
|
| 67 |
+
#: The vendor's two post-type tokens, verbatim from the dataset's own `ai_description`
|
| 68 |
+
#: (*"strictly these two"*, video = 99.5% of rows). Ours are `image`/`video`/`carousel`.
|
| 69 |
+
TT_POST_TYPE_VIDEO = "video"
|
| 70 |
+
TT_POST_TYPE_CONTENT = "content"
|
| 71 |
+
|
| 72 |
+
#: What a TikTok profile URL looks like, for the runner that has a handle and needs a URL. Kept
|
| 73 |
+
#: beside the dataset ids because it is the same class of vendor fact.
|
| 74 |
+
#: β ONE SPELLING. `platform/core/user_tables.profile_url(handle, 'tiktok')` builds the identical
|
| 75 |
+
#: string from `_PROFILE_RULES` (contract C2, E's half) β this exists for the connector's own
|
| 76 |
+
#: batch calls, and the gate asserts the two agree rather than trusting that they do.
|
| 77 |
+
TT_PROFILE_URL = "https://www.tiktok.com/@{handle}"
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
def tt_profile_url(handle):
|
| 81 |
+
"""`nurilab` β `https://www.tiktok.com/@nurilab`. `''` for a blank handle, never a bare `@`."""
|
| 82 |
+
h = str(handle or "").strip().lstrip("@")
|
| 83 |
+
return TT_PROFILE_URL.format(handle=h) if h else ""
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
# ---------------------------------------------------------------------------------------------
|
| 87 |
+
# THE FIELD MAPS
|
| 88 |
+
# ---------------------------------------------------------------------------------------------
|
| 89 |
+
# Each function turns ONE vendor row into the cell dict for one of the `ut_tt_*` schemas declared
|
| 90 |
+
# in `automation_engine`. The rules they all obey, stated once:
|
| 91 |
+
#
|
| 92 |
+
# * **BLANK MEANS NOT READ, NEVER "THEY HAVE NONE".** A key the vendor did not send is OMITTED,
|
| 93 |
+
# so a later, richer pull fills it instead of being overwritten by this one's silence. `_first`
|
| 94 |
+
# returns `None` (never 0) when nothing matches, which is what makes that possible.
|
| 95 |
+
# * **A zero from the vendor is a MEASUREMENT and survives.** (Instagram's `_ig_zero_is_blank`
|
| 96 |
+
# rule is scoped to a paid rung whose zeros were proven fictional; nothing here has earned it.)
|
| 97 |
+
# * **Every unpromoted vendor key stays whole in `source_payload`.** A schema addition on the
|
| 98 |
+
# vendor's side is preserved rather than silently discarded while our column model catches up.
|
| 99 |
+
# * **Nothing here writes a session token.** `tt_chain_token`, `secu_id` (~85% null), `short_id`
|
| 100 |
+
# (100% null in the sample), `ftc` (100% null) and `relation` are deliberately unmapped; they
|
| 101 |
+
# ride in `source_payload` where they make no claim.
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
def _tt_str(node, *names):
|
| 105 |
+
"""The first non-empty string among `names`, or None when the vendor sent nothing.
|
| 106 |
+
|
| 107 |
+
β `None`, NOT `""`. The callers below drop `None` keys, which is what keeps a blank honest β
|
| 108 |
+
an empty string written into a cell claims "we looked and it is empty".
|
| 109 |
+
"""
|
| 110 |
+
v = _first(node, *names)
|
| 111 |
+
if v is None:
|
| 112 |
+
return None
|
| 113 |
+
s = str(v).strip()
|
| 114 |
+
return s or None
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
def _tt_pct(node, *names):
|
| 118 |
+
"""A vendor 0β1 engagement fraction β our stored 0β100 percentage, or None.
|
| 119 |
+
|
| 120 |
+
β THE Γ100 IS NOT COSMETIC (wave 26, amendment C1-a). Our `pct` renderer appends the sign to
|
| 121 |
+
the STORED number, so writing the vendor's raw 0.0656 would print a 6.6% creator as `0.0%` β
|
| 122 |
+
measured on the Instagram side, and TikTok sends the same shape on all three of its rates.
|
| 123 |
+
"""
|
| 124 |
+
v = _first(node, *names)
|
| 125 |
+
if v is None:
|
| 126 |
+
return None
|
| 127 |
+
out = engine._pct100(v)
|
| 128 |
+
return out or None
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
def _tt_day(node, *names):
|
| 132 |
+
"""A vendor stamp β `YYYY-MM-DD`, or None. Our `date` columns store a day."""
|
| 133 |
+
v = _first(node, *names)
|
| 134 |
+
if v is None:
|
| 135 |
+
return None
|
| 136 |
+
return engine._day(v) or None
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
def _drop_blanks(row):
|
| 140 |
+
"""The one place a mapped row loses its `None`s β see the BLANK MEANS NOT READ rule above."""
|
| 141 |
+
return {k: v for k, v in row.items() if v is not None and v != ""}
|
| 142 |
+
|
| 143 |
+
|
| 144 |
+
def normalize_profile(node, handle=""):
|
| 145 |
+
"""A TikTok Profiles row β the `ut_tt_profile` / `ut_tt_snapshots` cell shape.
|
| 146 |
+
|
| 147 |
+
β TWO FIELDS READ THROUGH A CANDIDATE PAIR, and both pairs are the vendor's, not a guess:
|
| 148 |
+
* `biography` is PRIMARY and `signature` the FALLBACK β the probe measured `signature`
|
| 149 |
+
populated on 85% of rows and they carry the same text;
|
| 150 |
+
* `region` is PRIMARY and `country` the FALLBACK β `region` is the one with a documented
|
| 151 |
+
two-letter-ISO description, `country` has no description at all.
|
| 152 |
+
β `videos_count` is mapped to `posts_count` with a caveat recorded rather than hidden: its
|
| 153 |
+
`ai_description` ranges 1-89, so it may be a WINDOW rather than a lifetime total. It is the
|
| 154 |
+
only count of its kind the dataset offers.
|
| 155 |
+
"""
|
| 156 |
+
node = node if isinstance(node, dict) else {}
|
| 157 |
+
account = _tt_str(node, "account_id") or str(handle or "").strip().lstrip("@")
|
| 158 |
+
return _drop_blanks({
|
| 159 |
+
"platform": engine.PLATFORM_TIKTOK,
|
| 160 |
+
"handle": account,
|
| 161 |
+
"full_name": _tt_str(node, "nickname"),
|
| 162 |
+
"tt_id": _tt_str(node, "id"),
|
| 163 |
+
"profile_url": _tt_str(node, "url") or (tt_profile_url(account) or None),
|
| 164 |
+
"bio": _tt_str(node, "biography", "signature"),
|
| 165 |
+
"external_url": _bd_first_url(_first(node, "bio_link")),
|
| 166 |
+
"verified": _bd_flag(node, "is_verified"),
|
| 167 |
+
"is_private": _bd_flag(node, "is_private"),
|
| 168 |
+
# β APPROXIMATE, and the word is the vendor's: `is_commerce_user` has *"many null values"*
|
| 169 |
+
# on their own description. It is the closest thing TikTok has to Instagram's
|
| 170 |
+
# `is_business_account`, and `_bd_flag` writes nothing at all when the key is absent β so
|
| 171 |
+
# the approximation only ever fills a cell the vendor actually answered.
|
| 172 |
+
"is_business": _bd_flag(node, "is_commerce_user"),
|
| 173 |
+
"followers": _ig_int(_first(node, "followers")),
|
| 174 |
+
"following": _ig_int(_first(node, "following")),
|
| 175 |
+
"posts_count": _ig_int(_first(node, "videos_count")),
|
| 176 |
+
# β `likes` on a PROFILE row is likes RECEIVED across the account's videos (18-110,200, no
|
| 177 |
+
# nulls). It is not a post-level number and it is not our `likes` column, which is why it
|
| 178 |
+
# is stored under a different name.
|
| 179 |
+
"likes_received": _ig_int(_first(node, "likes")),
|
| 180 |
+
"avg_engagement": _tt_pct(node, "awg_engagement_rate"),
|
| 181 |
+
"like_engagement": _tt_pct(node, "like_engagement_rate"),
|
| 182 |
+
"comment_engagement": _tt_pct(node, "comment_engagement_rate"),
|
| 183 |
+
"country_code": _tt_str(node, "region", "country"),
|
| 184 |
+
"region": _tt_str(node, "region"),
|
| 185 |
+
"predicted_lang": _tt_str(node, "predicted_lang"),
|
| 186 |
+
# β ACCOUNT AGE, NOT A MEASUREMENT STAMP β `create_time` on a profile is when the ACCOUNT
|
| 187 |
+
# was made. TikTok stamps nothing with "when this number was true", exactly like Instagram,
|
| 188 |
+
# which is why the append law dates a snapshot by when WE read it.
|
| 189 |
+
"account_created_at": _tt_day(node, "create_time"),
|
| 190 |
+
"source_payload": _bd_source_payload(node),
|
| 191 |
+
})
|
| 192 |
+
|
| 193 |
+
|
| 194 |
+
def tt_post_type(node):
|
| 195 |
+
"""A TikTok post row β one of OUR three type options, or None.
|
| 196 |
+
|
| 197 |
+
β THE FIRST OF THE PROBE DOC'S TWO NAMED BLOCKERS. The vendor's vocabulary is `"video"` /
|
| 198 |
+
`"content"`; ours is `image` / `video` / `carousel` and has no `"content"`. Writing the
|
| 199 |
+
vendor's token would fail `_clean_field`'s option check on the way in and would put an
|
| 200 |
+
untranslated API word in front of a user on the way out.
|
| 201 |
+
|
| 202 |
+
So: `video` is `video`, and `content` β TikTok's photo-mode post β is `image`, EXCEPT when the
|
| 203 |
+
row carries more than one `carousel_images` entry, which is what a carousel IS on either
|
| 204 |
+
network. β The multi-image branch is decided from the IMAGES, never from the type token: the
|
| 205 |
+
token cannot express it, so inferring `carousel` from the word would be inventing a fact.
|
| 206 |
+
β An UNKNOWN token returns None rather than defaulting to `video` (99.5% of rows are video, and
|
| 207 |
+
that is exactly what would make the wrong default invisible).
|
| 208 |
+
"""
|
| 209 |
+
raw = str((node or {}).get("post_type") or "").strip().lower()
|
| 210 |
+
if raw == TT_POST_TYPE_VIDEO:
|
| 211 |
+
return "video"
|
| 212 |
+
if raw != TT_POST_TYPE_CONTENT:
|
| 213 |
+
return None
|
| 214 |
+
images = (node or {}).get("carousel_images")
|
| 215 |
+
return "carousel" if isinstance(images, list) and len(images) > 1 else "image"
|
| 216 |
+
|
| 217 |
+
|
| 218 |
+
def normalize_post(node):
|
| 219 |
+
"""A TikTok Posts row β the `ut_tt_posts` cell shape. None when it carries no identity.
|
| 220 |
+
|
| 221 |
+
β THE SECOND NAMED BLOCKER, RESOLVED HERE AND NOWHERE ELSE: `play_count` is ONE number and
|
| 222 |
+
Instagram's schema has TWO columns for it (`plays` and `views`). On TikTok they are the same
|
| 223 |
+
fact β `play_count` IS the count TikTok displays under a video β so it maps to `views` and
|
| 224 |
+
`ut_tt_posts` HAS NO `plays` COLUMN. Copying one vendor number into two of our columns would
|
| 225 |
+
manufacture a second measurement that a rollup could average or double-count, which is a worse
|
| 226 |
+
outcome than the missing column it would paper over.
|
| 227 |
+
|
| 228 |
+
β `shortcode` reads `shortcode` then `post_id`: both are 19-digit numerics on this dataset and
|
| 229 |
+
the probe measured them as the same shape. That equality is what lets the commentsβposts link
|
| 230 |
+
join with NO normaliser, which the Instagram side never had.
|
| 231 |
+
β `num_share_count` (a number) is preferred over `share_count` (typed TEXT by the vendor).
|
| 232 |
+
β `commerce_info` is a business/commerce LOCATION per its own description β cities and
|
| 233 |
+
countries. It is NOT a paid-partnership flag, and nothing in this dataset is: TikTok declares
|
| 234 |
+
no equivalent, so `paid_partnership`/`partner` have no column on this family at all.
|
| 235 |
+
"""
|
| 236 |
+
node = node if isinstance(node, dict) else {}
|
| 237 |
+
shortcode = _tt_str(node, "shortcode", "post_id")
|
| 238 |
+
if not shortcode:
|
| 239 |
+
return None
|
| 240 |
+
return _drop_blanks({
|
| 241 |
+
"platform": engine.PLATFORM_TIKTOK,
|
| 242 |
+
"shortcode": shortcode,
|
| 243 |
+
# ββ `account_id`, NOT `profile_username` β MEASURED on a real Posts row 2026-08-12.
|
| 244 |
+
# The vendor's `profile_username` is the DISPLAY NAME (`"Dina"`), while `account_id` is the
|
| 245 |
+
# @handle (`"d1na_th"`) β the same field `normalize_profile` already reads for `handle`, so
|
| 246 |
+
# one name means one thing across both corpora. Reading the display name silently broke the
|
| 247 |
+
# only join this table has: `ut_tt_posts.influencer_key` -> `ut_tt_profile.handle` matched
|
| 248 |
+
# NOTHING, so a person could not filter posts by creator and a rollup would count zero.
|
| 249 |
+
# β NO FALLBACK TO `profile_username`, deliberately. It is not a degraded handle, it is a
|
| 250 |
+
# different fact, and filling a join key with it is worse than leaving it blank β a blank
|
| 251 |
+
# is visibly missing, a display name looks like an answer [[one-question-two-normalizers]].
|
| 252 |
+
# The URL is the honest second source: it carries the handle by construction.
|
| 253 |
+
"influencer_key": (_tt_str(node, "account_id")
|
| 254 |
+
or tt_handle(_tt_str(node, "profile_url", "url") or "") or None),
|
| 255 |
+
"posted_at": _tt_day(node, "create_time"),
|
| 256 |
+
"type": tt_post_type(node),
|
| 257 |
+
"caption": _tt_str(node, "description"),
|
| 258 |
+
"url": _tt_str(node, "url"),
|
| 259 |
+
"hashtags": _bd_list(node, "hashtags"),
|
| 260 |
+
"tagged_location": _tt_str(node, "commerce_info"),
|
| 261 |
+
"views": _ig_int(_first(node, "play_count")),
|
| 262 |
+
"likes": _ig_int(_first(node, "digg_count")),
|
| 263 |
+
"comments": _ig_int(_first(node, "comment_count")),
|
| 264 |
+
"shares": _ig_int(_first(node, "num_share_count")),
|
| 265 |
+
"saves": _ig_int(_first(node, "collect_count")),
|
| 266 |
+
"video_duration": _ig_int(_first(node, "video_duration")),
|
| 267 |
+
"source_payload": _bd_source_payload(node),
|
| 268 |
+
})
|
| 269 |
+
|
| 270 |
+
|
| 271 |
+
def normalize_comment(node):
|
| 272 |
+
"""A TikTok Comments row β the `ut_tt_comments` cell shape. None without a comment id.
|
| 273 |
+
|
| 274 |
+
β THE NAME COLLISION, RESOLVED: TikTok's `replies` is an ARRAY of reply objects and OUR
|
| 275 |
+
`replies` column is an INT count. The count comes from `num_replies`; the array stays whole in
|
| 276 |
+
`source_payload`. Reading the array's length instead would be a second, disagreeing answer to
|
| 277 |
+
a question the vendor already answers β and it would disagree, because a page of replies is not
|
| 278 |
+
all of them.
|
| 279 |
+
β `date_created` is typed `date` by this vendor, unlike Instagram's `comment_date` which is
|
| 280 |
+
text and needs defensive parsing. It still goes through `_tt_day` β one date path, so a vendor
|
| 281 |
+
that changes its mind cannot change ours.
|
| 282 |
+
β The comment TEXT and every identifiable commenter field (`commenter_user_name` is flagged
|
| 283 |
+
PII) stay in `source_payload` and are promoted to no column, which is the same posture the
|
| 284 |
+
Instagram comment schema takes for the same D-24 reason.
|
| 285 |
+
"""
|
| 286 |
+
node = node if isinstance(node, dict) else {}
|
| 287 |
+
comment_key = _tt_str(node, "comment_id")
|
| 288 |
+
if not comment_key:
|
| 289 |
+
return None
|
| 290 |
+
return _drop_blanks({
|
| 291 |
+
"platform": engine.PLATFORM_TIKTOK,
|
| 292 |
+
"comment_key": comment_key,
|
| 293 |
+
"shortcode": _tt_str(node, "post_id"),
|
| 294 |
+
# ββ OWNER RULING 2026-08-12: the comment's CONTENT gets a column. `comment_text` is the
|
| 295 |
+
# vendor's own key and `comment_text_only` its stripped variant (the probe recorded both);
|
| 296 |
+
# primary first, so a row carrying the rich form is not silently served the plain one.
|
| 297 |
+
# β The commenter's identity is deliberately NOT promoted β see `TT_COMMENT_FIELDS`.
|
| 298 |
+
"text": _tt_str(node, "comment_text", "comment_text_only"),
|
| 299 |
+
"commented_at": _tt_day(node, "date_created"),
|
| 300 |
+
"likes": _ig_int(_first(node, "num_likes")),
|
| 301 |
+
"replies": _ig_int(_first(node, "num_replies")),
|
| 302 |
+
"source_payload": _bd_source_payload(node),
|
| 303 |
+
})
|
| 304 |
+
|
| 305 |
+
|
| 306 |
+
#: β The three maps, addressable by name β so a gate (and the runners in T04-T06) can walk them
|
| 307 |
+
#: rather than naming three functions, and so adding a fourth dataset is one entry.
|
| 308 |
+
TT_NORMALIZERS = {
|
| 309 |
+
"tt_profile": normalize_profile,
|
| 310 |
+
"tt_post_metrics": normalize_post,
|
| 311 |
+
"tt_comments": normalize_comment,
|
| 312 |
+
}
|
| 313 |
+
|
| 314 |
+
|
| 315 |
+
# ---------------------------------------------------------------------------------------------
|
| 316 |
+
# THE FETCH β wave 30 Β· W30-T08 (carrying wave-29's dropped T05)
|
| 317 |
+
# ---------------------------------------------------------------------------------------------
|
| 318 |
+
|
| 319 |
+
def tt_handle(url):
|
| 320 |
+
"""A TikTok profile URL **or** a bare handle β the handle. `''` when it is neither.
|
| 321 |
+
|
| 322 |
+
Deliberately permissive about the input and strict about the output, because the two callers
|
| 323 |
+
hand it different things: an automation stores whatever a person typed in the profile column
|
| 324 |
+
(`@nurilab`, `nurilab`, or the full URL), while the discovery runner already holds a clean
|
| 325 |
+
`account_id`. One normaliser, so a row found by discovery and a row typed by hand cannot
|
| 326 |
+
resolve to two different handles.
|
| 327 |
+
"""
|
| 328 |
+
s = str(url or "").strip()
|
| 329 |
+
if not s:
|
| 330 |
+
return ""
|
| 331 |
+
if "tiktok.com" in s.lower():
|
| 332 |
+
# Everything after the first `@`, up to the next path segment or query.
|
| 333 |
+
tail = s.split("@", 1)[1] if "@" in s else ""
|
| 334 |
+
s = tail.split("/")[0].split("?")[0].split("#")[0]
|
| 335 |
+
s = s.strip().lstrip("@").strip()
|
| 336 |
+
# A handle is the vendor's `account_id` shape: alphanumerics, dots and underscores.
|
| 337 |
+
return s if s and all(c.isalnum() or c in "._" for c in s) else ""
|
| 338 |
+
|
| 339 |
+
|
| 340 |
+
def tt_post_urls(node, limit=0):
|
| 341 |
+
"""β WAVE 30 Β· T10 β the profile row's own post permalinks, newest-first as the vendor sends.
|
| 342 |
+
|
| 343 |
+
β THIS IS WHY TIKTOK POST CAPTURE COSTS NO EXTRA DISCOVERY. `top_videos` rides the PROFILE row
|
| 344 |
+
we have already bought, so the posts read is a scrape of links we hold, never a search for them.
|
| 345 |
+
The two TikTok DISCOVERY datasets (posts-by-profile, posts-by-keyword) are **404 for our key**,
|
| 346 |
+
so a design that reached for either would not merely be dearer, it would not work.
|
| 347 |
+
|
| 348 |
+
ββ CORRECTED 2026-08-12 β THIS DOCSTRING USED TO SAY *"the probe MEASURED `top_videos` as an
|
| 349 |
+
array of video permalinks, NO empties"*, AND THAT SENTENCE IS WHAT SHIPPED THE BUG. The probe
|
| 350 |
+
read `/datasets/{id}/metadata` β a DATASET description β and the phrase quoted was the field's
|
| 351 |
+
`ai_description`, not an observation of a row. The real row sends **dicts keyed `video_url`**
|
| 352 |
+
(measured below), so the reader built against the quoted sentence found nothing, forever, in
|
| 353 |
+
silence. β The transferable half: *"the probe measured X"* and *"the probe read a declaration
|
| 354 |
+
of X"* are different claims, and prose cannot be told apart by a reader downstream β which is
|
| 355 |
+
why the correction names the method, not just the value.
|
| 356 |
+
|
| 357 |
+
β `top_posts_data` is deliberately NOT read: the probe calls it *"a thin dup of `top_videos`"*,
|
| 358 |
+
and preferring whichever happened to be longer is how one creator's window silently differs
|
| 359 |
+
from another's.
|
| 360 |
+
β `limit <= 0` means "everything the row carried". The CAP IS THE CALLER'S β `config.maxPosts`,
|
| 361 |
+
validated 1..12 β and it is applied here rather than after the scrape so an unwanted post is
|
| 362 |
+
never bought. [[a-constant-two-features-share]]: the 12 is the vendor's measured profile window,
|
| 363 |
+
not a number this function may invent.
|
| 364 |
+
"""
|
| 365 |
+
raw = (node or {}).get("top_videos")
|
| 366 |
+
out = []
|
| 367 |
+
for item in raw if isinstance(raw, list) else []:
|
| 368 |
+
# ββ MEASURED ON A REAL ROW 2026-08-12, AND IT IS NOT WHAT THE SCHEMA SAID.
|
| 369 |
+
# `top_videos` is NOT an array of permalink strings. One paid TikTok Profiles scrape of a
|
| 370 |
+
# live handle returns **19 DICTS**, keyed
|
| 371 |
+
# `video_url Β· video_id Β· playcount Β· diggcount Β· commentcount Β· share_count Β·
|
| 372 |
+
# favorites_count Β· create_date Β· cover_image`.
|
| 373 |
+
# The docstring above cites the $0 probe as having "measured" permalinks β it had not, and
|
| 374 |
+
# could not: `/datasets/{id}/metadata` describes a DATASET, and the probe's own verdict says
|
| 375 |
+
# so in terms (*"what $0 cannot buy β¦ the real ROW shape β¦ that a declared field
|
| 376 |
+
# POPULATES"*, `wave29/proto/tiktok-schema.md`). This is the SECOND time this vendor's
|
| 377 |
+
# declaration has diverged from its delivery on this exact axis; BD's IG Reels `views` was
|
| 378 |
+
# the first. [[reachable-is-not-the-same-as-built]]
|
| 379 |
+
# β The consequence, live: `TT_DS_POSTS` was never reached, because this returned an EMPTY
|
| 380 |
+
# list on every real profile β post capture could not have worked for anybody, and the
|
| 381 |
+
# T10 gate stayed green because its canned fixture encoded the DECLARED shape. A fixture
|
| 382 |
+
# written from a schema tests the schema.
|
| 383 |
+
# β `video_url` is FIRST because it is the key the vendor actually sends; `url` is kept
|
| 384 |
+
# because it costs nothing and is what a future corpus revision would most likely use. The
|
| 385 |
+
# bare-string branch stays for the same reason β this widens what is ACCEPTED and invents
|
| 386 |
+
# nothing: a shape that yields no `httpβ¦` value still degrades to "no posts", exactly as
|
| 387 |
+
# before, rather than to a URL built out of a guess.
|
| 388 |
+
# β `top_posts_data` is STILL not read (it carries `post_url` and would work): preferring
|
| 389 |
+
# whichever array happened to be longer is how one creator's window silently differs from
|
| 390 |
+
# another's, and that reasoning is unchanged by this correction.
|
| 391 |
+
if isinstance(item, dict):
|
| 392 |
+
u = str(item.get("video_url") or item.get("url") or "").strip()
|
| 393 |
+
else:
|
| 394 |
+
u = str(item or "").strip()
|
| 395 |
+
if u.lower().startswith("http") and u not in out:
|
| 396 |
+
out.append(u)
|
| 397 |
+
return out[:limit] if limit and limit > 0 else out
|
| 398 |
+
|
| 399 |
+
|
| 400 |
+
def pull_posts_tt(post_urls, log=print, deferred=None):
|
| 401 |
+
"""The TikTok Posts dataset for a list of permalinks β `(rows, note)`, already normalised.
|
| 402 |
+
|
| 403 |
+
β ONE CALL FOR THE WHOLE WINDOW. `bd_scrape` has always taken a list, and the Instagram side
|
| 404 |
+
measured what happens when a caller forgets: 25 records, one billed snapshot each, a walk still
|
| 405 |
+
running at 67 minutes. Nothing here loops per URL.
|
| 406 |
+
"""
|
| 407 |
+
urls = [str(u) for u in (post_urls or []) if str(u or "").strip()]
|
| 408 |
+
if not urls:
|
| 409 |
+
return [], ""
|
| 410 |
+
rows, note = bd_scrape(TT_DS_POSTS, urls, deferred=deferred)
|
| 411 |
+
if note:
|
| 412 |
+
log(f"[aios-tt] posts: {note}")
|
| 413 |
+
return [], note
|
| 414 |
+
out = [r for r in (normalize_post(n) for n in rows) if r]
|
| 415 |
+
return out, ""
|
| 416 |
+
|
| 417 |
+
|
| 418 |
+
def pull_comments_tt(post_urls, log=print, deferred=None):
|
| 419 |
+
"""The TikTok Comments dataset for a list of POST permalinks β `(rows, note)`, normalised.
|
| 420 |
+
|
| 421 |
+
β THE MOST EXPENSIVE THING THIS PRODUCT BUYS, and the reason `commentMetrics` defaults OFF on
|
| 422 |
+
both networks: a comments scrape ingests identifiable third parties who never entered anybody's
|
| 423 |
+
list (D-24). The mapper already keeps every commenter field in `source_payload` and promotes
|
| 424 |
+
none of them to a column; this function adds no new exposure, it just has to be asked for.
|
| 425 |
+
"""
|
| 426 |
+
urls = [str(u) for u in (post_urls or []) if str(u or "").strip()]
|
| 427 |
+
if not urls:
|
| 428 |
+
return [], ""
|
| 429 |
+
rows, note = bd_scrape(TT_DS_COMMENTS, urls, deferred=deferred)
|
| 430 |
+
if note:
|
| 431 |
+
log(f"[aios-tt] comments: {note}")
|
| 432 |
+
return [], note
|
| 433 |
+
out = [r for r in (normalize_comment(n) for n in rows) if r]
|
| 434 |
+
return out, ""
|
| 435 |
+
|
| 436 |
+
|
| 437 |
+
#: ββ WAVE 30 Β· D-156 β THE MEDIA DATASETS, AS A SET, SO THE HAND-OFF CAN FILTER ON IDENTITY.
|
| 438 |
+
#: `_media_deferrals` uses this to lift ONLY posts/comments snapshots out of the local deferral
|
| 439 |
+
#: list. That is what makes it structurally impossible to file a PROFILE snapshot in the engine's
|
| 440 |
+
#: metric queue β the defect a draft of T10 shipped and A-39 booked as "the wrong fix is worse
|
| 441 |
+
#: than the gap". A membership test cannot be got wrong by a later edit the way `if` order can.
|
| 442 |
+
TT_MEDIA_DATASETS = (TT_DS_POSTS, TT_DS_COMMENTS)
|
| 443 |
+
|
| 444 |
+
|
| 445 |
+
def _media_deferrals(deferred):
|
| 446 |
+
"""The POSTS/COMMENTS entries of a `bd_scrape` deferral list β never the profile's.
|
| 447 |
+
|
| 448 |
+
β The engine, not this module, decides what a deferral MEANS: it stamps `kind` and the
|
| 449 |
+
handle and files it. TikTok needs no `_tag_metric_deferrals` twin because one dataset is one
|
| 450 |
+
kind here, so the id already carries everything a mapper choice depends on β and importing
|
| 451 |
+
Instagram's tagger is not available anyway (W30-T09 gates ZERO `from connectors_ig` lines).
|
| 452 |
+
"""
|
| 453 |
+
out = []
|
| 454 |
+
for d in deferred or []:
|
| 455 |
+
if isinstance(d, dict) and str(d.get("datasetId") or "") in TT_MEDIA_DATASETS:
|
| 456 |
+
out.append(dict(d))
|
| 457 |
+
return out
|
| 458 |
+
|
| 459 |
+
|
| 460 |
+
def pull_profile_tt(url, log=print, pending_profile=None, prefetch=None,
|
| 461 |
+
max_posts=0, post_metrics=False, comment_metrics=False):
|
| 462 |
+
"""ONE TikTok profile from the vendor. Same return contract as `pull_profile`.
|
| 463 |
+
|
| 464 |
+
`{state, profile, posts, comments, via, note}` with `state β ok | partial | blocked | error`,
|
| 465 |
+
so the engine's enrich branch treats every network identically and no caller learns a new
|
| 466 |
+
shape.
|
| 467 |
+
|
| 468 |
+
β WAVE 30 Β· T10 β POSTS AND COMMENTS ARE REAL NOW, AND BOTH DEFAULT OFF, exactly as Instagram's
|
| 469 |
+
do. `post_metrics` scrapes the profile row's own `top_videos` permalinks (see `tt_post_urls` β
|
| 470 |
+
no discovery call, because both TikTok discovery datasets 404 for our key); `comment_metrics`
|
| 471 |
+
then scrapes the comments of the posts that came back. β COMMENTS REQUIRE POSTS by construction
|
| 472 |
+
rather than by a rule: their input IS a post permalink, so asking for comments with post capture
|
| 473 |
+
off is a request with no subject, and it returns none instead of quietly buying posts nobody
|
| 474 |
+
asked for.
|
| 475 |
+
|
| 476 |
+
β `partial` IS THE SUCCESS STATE WHENEVER NO MEDIA WAS READ, and that is deliberate rather than
|
| 477 |
+
pessimistic. The Instagram contract reads `ok` only when identity AND media both landed
|
| 478 |
+
(`pull_profile_bd`: *"identity without media is still partial ... a run that wrote a follower
|
| 479 |
+
count and no posts must not paint green over a posts table that did not grow"*). So: posts not
|
| 480 |
+
ASKED for β `partial`, saying so; posts asked for and landed β `ok`; asked for and none came β
|
| 481 |
+
`partial` with the vendor's reason. The state answers "did this pull deliver what it went for",
|
| 482 |
+
never "did the function finish".
|
| 483 |
+
|
| 484 |
+
β **NO FREE RUNG, AND NO FALLBACK CHAIN.** Instagram's `pull_profile` drops to Apify when the
|
| 485 |
+
paid rung refuses; `providers.DEFAULT_CHAINS["tt_profile"]` is deliberately single-provider,
|
| 486 |
+
with its own note explaining that a multi-provider chain is a promise something walks it and
|
| 487 |
+
that nothing walks Instagram's second name today either. So a refusal here is final, and it
|
| 488 |
+
says so instead of implying a retry somewhere.
|
| 489 |
+
"""
|
| 490 |
+
handle = tt_handle(url)
|
| 491 |
+
if not handle:
|
| 492 |
+
return {"state": "error", "profile": {}, "posts": [], "comments": [], "via": "",
|
| 493 |
+
"note": f"{url!r} is not a TikTok profile URL or handle"}
|
| 494 |
+
|
| 495 |
+
# β THE BATCH FAST PATH, same shape as the Instagram side: `prefetch` is `{handle: node}` from
|
| 496 |
+
# one multi-URL scrape covering a whole selection. A hit is a vendor round trip that does not
|
| 497 |
+
# happen; a miss falls through to the single-URL call below.
|
| 498 |
+
cached = prefetch.get(handle) if isinstance(prefetch, dict) else None
|
| 499 |
+
_deferred = []
|
| 500 |
+
if isinstance(cached, dict) and cached:
|
| 501 |
+
rows, note = [cached], ""
|
| 502 |
+
else:
|
| 503 |
+
rows, note = bd_scrape(TT_DS_PROFILES, [tt_profile_url(handle)], deferred=_deferred)
|
| 504 |
+
|
| 505 |
+
node = rows[0] if rows else {}
|
| 506 |
+
profile = normalize_profile(node, handle) if node else {}
|
| 507 |
+
# β THE READABILITY TEST IS `followers`/`following`, NOT "did we get a dict". `normalize_profile`
|
| 508 |
+
# drops blanks, so an unreadable row still returns `{"platform": β¦, "handle": β¦}` β truthy, and
|
| 509 |
+
# carrying nothing anybody asked for. The Instagram rung tests exactly this pair for exactly
|
| 510 |
+
# this reason, and answering "0 followers" instead is the failure it exists to prevent.
|
| 511 |
+
unreadable = profile.get("followers") is None and profile.get("following") is None
|
| 512 |
+
if note or unreadable:
|
| 513 |
+
# β THE DEFERRAL IS HANDED OVER RATHER THAN DISCARDED. A snapshot the vendor is still
|
| 514 |
+
# building HAS ALREADY BEEN PAID FOR; dropping its id bills again on the next run for the
|
| 515 |
+
# same record. That was live on the Instagram profile path until 2026-08-09 β measured on
|
| 516 |
+
# nurilab as two runs, two fresh snapshots, both abandoned β and it is not being
|
| 517 |
+
# reintroduced here by omission.
|
| 518 |
+
if isinstance(pending_profile, list):
|
| 519 |
+
for d in _deferred:
|
| 520 |
+
pending_profile.append({**d, "kind": "profile", "influencer": handle})
|
| 521 |
+
why = note or ("the scrape answered, but no follower/following counts were readable in it "
|
| 522 |
+
"(the field names may have moved - see tiktok-capture.md)")
|
| 523 |
+
return {"state": "blocked", "profile": {}, "posts": [], "comments": [], "via": "brightdata",
|
| 524 |
+
"deferredProfile": [d.get("snapshotId") for d in _deferred],
|
| 525 |
+
"note": why}
|
| 526 |
+
|
| 527 |
+
# --- W30-T10: THE MEDIA, ONLY WHEN IT WAS ASKED FOR. ------------------------------------
|
| 528 |
+
if not post_metrics:
|
| 529 |
+
return {"state": "partial", "profile": profile, "posts": [], "comments": [],
|
| 530 |
+
"via": "brightdata",
|
| 531 |
+
"note": note or "profile read; post capture is off for this step"}
|
| 532 |
+
urls = tt_post_urls(node, limit=max_posts)
|
| 533 |
+
if not urls:
|
| 534 |
+
# β NOT AN ERROR AND NOT A RETRY. A creator with no `top_videos` has nothing to buy, and
|
| 535 |
+
# saying so is what stops the next run paying to be told the same thing.
|
| 536 |
+
return {"state": "partial", "profile": profile, "posts": [], "comments": [],
|
| 537 |
+
"via": "brightdata",
|
| 538 |
+
"note": note or "profile read; this account's row carried no post links"}
|
| 539 |
+
posts, p_note = pull_posts_tt(urls, log=log, deferred=_deferred)
|
| 540 |
+
comments, c_note = ([], "")
|
| 541 |
+
if comment_metrics and posts:
|
| 542 |
+
# The comments dataset is keyed on a POST permalink, so it reads the posts we just bought β
|
| 543 |
+
# `url` from the mapper, never the profile's raw array, so a post the posts scrape refused
|
| 544 |
+
# is not silently asked about again one rung later.
|
| 545 |
+
comments, c_note = pull_comments_tt([p.get("url") for p in posts if p.get("url")],
|
| 546 |
+
log=log, deferred=_deferred)
|
| 547 |
+
# ββ WAVE 30 Β· D-156 β THE MEDIA DEFERRALS ARE HANDED BACK, and the shape of the hand-off is
|
| 548 |
+
# the whole lesson. An earlier draft of T10 appended every `_deferred` entry to
|
| 549 |
+
# `pending_profile` tagged `kind: "profile"`. By the time control reaches here a PROFILE
|
| 550 |
+
# deferral is impossible β the profile branch above returns `blocked` on any note β so **every
|
| 551 |
+
# id fanned out that way was a POSTS or COMMENTS snapshot in the PROFILE queue**, whose
|
| 552 |
+
# collector writes preset profile cells onto somebody's record from post rows. The engine keeps
|
| 553 |
+
# the two queues apart deliberately (`_pending_profile_tasks` vs `_pending_metric_tasks`).
|
| 554 |
+
# β So this returns them under their OWN key, filtered by dataset identity
|
| 555 |
+
# (`_media_deferrals`), and the engine files them in the metric queue with the handle it
|
| 556 |
+
# already holds. Returning rather than appending also keeps the queue's vocabulary out of a
|
| 557 |
+
# connector: this module knows which CORPUS deferred, never what the engine calls it.
|
| 558 |
+
# β `deferredMedia` rides BOTH returns on purpose. The empty-posts case is the one that
|
| 559 |
+
# matters most β that is exactly the run where the vendor took too long, so a caller reading
|
| 560 |
+
# the ids only from the success path would lose every batch it actually paid for.
|
| 561 |
+
deferred_media = _media_deferrals(_deferred)
|
| 562 |
+
if not posts:
|
| 563 |
+
return {"state": "partial", "profile": profile, "posts": [], "comments": [],
|
| 564 |
+
"via": "brightdata", "deferredMedia": deferred_media,
|
| 565 |
+
"note": p_note or note or "profile read; the post source returned nothing"}
|
| 566 |
+
return {"state": "ok", "profile": profile, "posts": posts, "comments": comments,
|
| 567 |
+
"via": "brightdata", "deferredMedia": deferred_media,
|
| 568 |
+
"note": c_note or note or ""}
|
api/main.py
CHANGED
|
@@ -85,6 +85,8 @@ import routes_slack # noqa: E402 (wave 33 R4/C2 β Manage agent + the Slack d
|
|
| 85 |
import routes_starred # noqa: E402 (wave 35 R4/C2/C3/C5 β the star; E's router, E's line)
|
| 86 |
import routes_usage # noqa: E402 (wave 35 R9/C7 β the AI usage meter; E's router, E's line)
|
| 87 |
import routes_feedback # noqa: E402 (wave 35 R8/C6 β feedback to the operator plane; E's router)
|
|
|
|
|
|
|
| 88 |
from core import grid_events # noqa: E402
|
| 89 |
# D-315 / D-305 (2026-08-18) β the two store REFUSALS get their own app-level handlers below.
|
| 90 |
# β Neither is a `StoreUnavailable` subclass, on purpose: routes that degrade a store outage into a
|
|
@@ -376,6 +378,23 @@ app.include_router(routes_usage.router) # R9 / C7 β GET /usage, the one
|
|
| 376 |
# POST is any authenticated session's own act, the GET is `is_platform_admin` only. Mounting it does
|
| 377 |
# not widen anything a tenant admin can reach β `verify_api` proves that by having one try.
|
| 378 |
app.include_router(routes_feedback.router) # R8 / C6 β feedback to the operator plane
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 379 |
|
| 380 |
|
| 381 |
# --- DEPRECATED ALIASES (removed when S2's shell flips; kept so the current bundle keeps working)
|
|
|
|
| 85 |
import routes_starred # noqa: E402 (wave 35 R4/C2/C3/C5 β the star; E's router, E's line)
|
| 86 |
import routes_usage # noqa: E402 (wave 35 R9/C7 β the AI usage meter; E's router, E's line)
|
| 87 |
import routes_feedback # noqa: E402 (wave 35 R8/C6 β feedback to the operator plane; E's router)
|
| 88 |
+
import routes_agent_harness # noqa: E402 (wave 36 R8/C4 β the agent harness store; E's router)
|
| 89 |
+
import routes_script_views # noqa: E402 (wave 36 R3/R10/C3 β script Views; E's router)
|
| 90 |
from core import grid_events # noqa: E402
|
| 91 |
# D-315 / D-305 (2026-08-18) β the two store REFUSALS get their own app-level handlers below.
|
| 92 |
# β Neither is a `StoreUnavailable` subclass, on purpose: routes that degrade a store outage into a
|
|
|
|
| 378 |
# POST is any authenticated session's own act, the GET is `is_platform_admin` only. Mounting it does
|
| 379 |
# not widen anything a tenant admin can reach β `verify_api` proves that by having one try.
|
| 380 |
app.include_router(routes_feedback.router) # R8 / C6 β feedback to the operator plane
|
| 381 |
+
# ββ WAVE 36 (R8 / C4) β THE AGENT HARNESS FILE STORE, mounted in the SAME change that created
|
| 382 |
+
# `routes_agent_harness.py`. Eighth consecutive wave in which this block is the artefact the
|
| 383 |
+
# protocol nearly loses; `verify_web_agent::section_w36_harness` asserts this path in
|
| 384 |
+
# `app.openapi()["paths"]` and CALLS the route, because mounted is not callable (D-107).
|
| 385 |
+
# β Placement, as for every line above: ABOVE `app.mount("/", _AppStatic(...), html=True)` at the
|
| 386 |
+
# end of this file, or a GET answers 404 and a PUT answers 405 while every gate stays green.
|
| 387 |
+
# β ITS PATHS SIT UNDER `/agents/{id}/...`, WHICH `routes_slack` ALSO SERVES β and that is safe
|
| 388 |
+
# rather than lucky: a FastAPI path parameter never spans a `/`, so `/agents/{agent_id}` cannot
|
| 389 |
+
# match `/agents/x/harness`. The two routers share a prefix and no route.
|
| 390 |
+
app.include_router(routes_agent_harness.router) # R8 / C4 β versioned agent harness files
|
| 391 |
+
# ββ WAVE 36 (R3 / R10 / C3) β THE SCRIPT VIEW, owner item 6. Mounted in the SAME change that
|
| 392 |
+
# created `routes_script_views.py`; `verify_script_views.py` asserts both of its paths in
|
| 393 |
+
# `app.openapi()["paths"]` AND calls them, and its NC comments this line out.
|
| 394 |
+
# β ITS RUN DOOR SPAWNS A SUBPROCESS AND IS A PLAIN `def`, so FastAPI runs it in the threadpool.
|
| 395 |
+
# Mounting it does not put a ten-second wait anywhere near the event loop; the router's own header
|
| 396 |
+
# says why that is not a style choice.
|
| 397 |
+
app.include_router(routes_script_views.router) # R3 / R10 / C3 β code-script database Views
|
| 398 |
|
| 399 |
|
| 400 |
# --- DEPRECATED ALIASES (removed when S2's shell flips; kept so the current bundle keeps working)
|
api/odoo_relational.py
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
api/providers.py
CHANGED
|
@@ -124,7 +124,7 @@ PROVIDERS: dict[str, Provider] = {
|
|
| 124 |
"likes + comments + shares + saves, and play_count inline (DECLARED "
|
| 125 |
"no-empties-or-zeros, UNPROVEN until one live pull)"),
|
| 126 |
"tt_comments": Capability(True, _COST_BRIGHTDATA,
|
| 127 |
-
"separate paid dataset, opt-in
|
| 128 |
}),
|
| 129 |
"apify": Provider(
|
| 130 |
key="apify", label="Apify", key_env="AIOS_APIFY_KEY",
|
|
@@ -132,7 +132,7 @@ PROVIDERS: dict[str, Provider] = {
|
|
| 132 |
# β MEASURED 2026-08-08 against the public Reels grid: `videoPlayCount` 137,684 and
|
| 133 |
# 299,493 vs a browser-read ground truth of 134K-137K and 299K. Exact.
|
| 134 |
"ig_post_views": Capability(True, _COST_APIFY,
|
| 135 |
-
"videoPlayCount
|
| 136 |
"ig_post_metrics": Capability(True, _COST_APIFY, "likes + comments (fallback)"),
|
| 137 |
"ig_profile": Capability(True, _COST_APIFY, "profile fields (fallback)"),
|
| 138 |
}),
|
|
@@ -240,12 +240,12 @@ def run(capability, work, satisfied=None, log=None):
|
|
| 240 |
secs = round(time.time() - started, 2)
|
| 241 |
if note:
|
| 242 |
attempts.append(Attempt(provider.key, False, note, 0, secs))
|
| 243 |
-
log(f" {provider.label}: {note}
|
| 244 |
continue
|
| 245 |
if not ok(result):
|
| 246 |
attempts.append(Attempt(provider.key, False, "answered without the field asked for",
|
| 247 |
0, secs))
|
| 248 |
-
log(f" {provider.label}: answered, but not with what was asked for
|
| 249 |
last = result if last is None else last
|
| 250 |
continue
|
| 251 |
attempts.append(Attempt(provider.key, True, "", _count(result), secs))
|
|
@@ -282,6 +282,323 @@ def wire():
|
|
| 282 |
}
|
| 283 |
|
| 284 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 285 |
# =============================================================================================
|
| 286 |
# THE CANONICAL SCHEMA β owner ruling 2026-08-08:
|
| 287 |
# *"standardize the schema between Bright Data and APIfy so we keep using the same pre-set
|
|
|
|
| 124 |
"likes + comments + shares + saves, and play_count inline (DECLARED "
|
| 125 |
"no-empties-or-zeros, UNPROVEN until one live pull)"),
|
| 126 |
"tt_comments": Capability(True, _COST_BRIGHTDATA,
|
| 127 |
+
"separate paid dataset, opt-in: 17 fields"),
|
| 128 |
}),
|
| 129 |
"apify": Provider(
|
| 130 |
key="apify", label="Apify", key_env="AIOS_APIFY_KEY",
|
|
|
|
| 132 |
# β MEASURED 2026-08-08 against the public Reels grid: `videoPlayCount` 137,684 and
|
| 133 |
# 299,493 vs a browser-read ground truth of 134K-137K and 299K. Exact.
|
| 134 |
"ig_post_views": Capability(True, _COST_APIFY,
|
| 135 |
+
"videoPlayCount, matches Instagram's displayed views"),
|
| 136 |
"ig_post_metrics": Capability(True, _COST_APIFY, "likes + comments (fallback)"),
|
| 137 |
"ig_profile": Capability(True, _COST_APIFY, "profile fields (fallback)"),
|
| 138 |
}),
|
|
|
|
| 240 |
secs = round(time.time() - started, 2)
|
| 241 |
if note:
|
| 242 |
attempts.append(Attempt(provider.key, False, note, 0, secs))
|
| 243 |
+
log(f" {provider.label}: {note}, falling through")
|
| 244 |
continue
|
| 245 |
if not ok(result):
|
| 246 |
attempts.append(Attempt(provider.key, False, "answered without the field asked for",
|
| 247 |
0, secs))
|
| 248 |
+
log(f" {provider.label}: answered, but not with what was asked for, falling through")
|
| 249 |
last = result if last is None else last
|
| 250 |
continue
|
| 251 |
attempts.append(Attempt(provider.key, True, "", _count(result), secs))
|
|
|
|
| 282 |
}
|
| 283 |
|
| 284 |
|
| 285 |
+
|
| 286 |
+
|
| 287 |
+
# βββββββββββββββββββ WAVE 36 Β· W36-T35 (ruling R4, contract C5) β THE LLM LADDER ββββββββββββββββ
|
| 288 |
+
#
|
| 289 |
+
# Owner item 4, verbatim (2026-08-18): *"I'm also getting errors everywhere when I want to use the
|
| 290 |
+
# assisntant: 'the assistant could not be reached just now (openrouter: HTTP 402)' and 'cerebras:
|
| 291 |
+
# HTTP 402; groq: HTTP 404; openrouter: HTTP 402; anthropic: tool calls are not wired for this
|
| 292 |
+
# shape'."*
|
| 293 |
+
#
|
| 294 |
+
# ββ R4 IS THREE CLAUSES AND THEY LAND IN THREE DIFFERENT PLACES. (1) Anthropic becomes the
|
| 295 |
+
# tool-calling path that always works β a WIRE, in `routes_query`. (2) A provider with no credit is
|
| 296 |
+
# SKIPPED rather than tried β a memo, `mark_no_credit` below. (3) No raw `HTTP 402` ever reaches a
|
| 297 |
+
# screen β a SENTENCE, `refusal_sentence` below. The declaration here is what the first two read.
|
| 298 |
+
#
|
| 299 |
+
# β WHY A SECOND REGISTRY RATHER THAN ROWS IN `PROVIDERS` ABOVE. A scraping provider is
|
| 300 |
+
# `(key_env, caps)` and is billed per RECORD; an LLM provider is `(env, url, model, wire)` and is
|
| 301 |
+
# billed per TOKEN. Folding them into one dict would mean four fields that are meaningless for half
|
| 302 |
+
# the rows and an `estimate()` that answers $0.00 for anything LLM-shaped. What they SHARE is the
|
| 303 |
+
# thing worth sharing: `Capability`, so "MEASURED INCAPABLE" means exactly the same thing on both
|
| 304 |
+
# sides, and `llm_chain()` refuses an incapable row exactly as `chain()` does.
|
| 305 |
+
#
|
| 306 |
+
# β THE DECLARATION IS THE POINT (staged item 3). `_FAILED_GEN` in `routes_query` is a REGEX that
|
| 307 |
+
# recovers a tool call out of a provider's 400 β the evidence that guessing at capability failed.
|
| 308 |
+
# A row that says `llm_tool_calling: Capability(False, β¦)` is never offered for a tool-calling
|
| 309 |
+
# turn at all, so the guess never has to be made.
|
| 310 |
+
|
| 311 |
+
#: How long a provider stays skipped after it tells us it is out of credit. β A MEMO, NOT A FACT:
|
| 312 |
+
#: the balance can be topped up at any moment, so this expires rather than latching. Fifteen
|
| 313 |
+
#: minutes is long enough that a chat session does not re-pay the timeout on every turn, and short
|
| 314 |
+
#: enough that a top-up is picked up without a restart.
|
| 315 |
+
CREDIT_COOLDOWN_S = float(os.environ.get("AIOS_CREDIT_COOLDOWN_S") or 900)
|
| 316 |
+
|
| 317 |
+
#: `{provider name: unix ts when the memo expires}`. β PROCESS-LOCAL AND DELIBERATELY SO β it is a
|
| 318 |
+
#: latency optimisation, not a billing record. A second container learns the same thing from its
|
| 319 |
+
#: own first 402, and neither one can be wrong for longer than the cooldown.
|
| 320 |
+
_NO_CREDIT: dict[str, float] = {}
|
| 321 |
+
|
| 322 |
+
|
| 323 |
+
@dataclass(frozen=True)
|
| 324 |
+
class LlmProvider:
|
| 325 |
+
"""One chat-completions endpoint, and what it is DECLARED able to do."""
|
| 326 |
+
name: str
|
| 327 |
+
label: str
|
| 328 |
+
env: str
|
| 329 |
+
url: str
|
| 330 |
+
model: str
|
| 331 |
+
#: `openai` = the OpenAI-compatible `/chat/completions` shape. `anthropic` = the Messages API,
|
| 332 |
+
#: which is a different body, a different auth header and a different result shape.
|
| 333 |
+
wire: str
|
| 334 |
+
caps: dict = field(default_factory=dict)
|
| 335 |
+
|
| 336 |
+
def configured(self) -> bool:
|
| 337 |
+
return bool((os.environ.get(self.env) or "").strip())
|
| 338 |
+
|
| 339 |
+
def can(self, capability: str) -> bool:
|
| 340 |
+
cap = self.caps.get(capability)
|
| 341 |
+
return bool(cap and cap.capable and self.configured())
|
| 342 |
+
|
| 343 |
+
|
| 344 |
+
#: β ORDER IS THE LADDER, AND ANTHROPIC IS FIRST BECAUSE OF R4. `routes_query`'s old comment put
|
| 345 |
+
#: cerebras first *"because this path needs tool calling and cerebras carries this account's
|
| 346 |
+
#: tool-capable model"* β R4 replaces that premise: Anthropic is the tool-calling path that always
|
| 347 |
+
#: works, and the others are the cheap seats it falls through to.
|
| 348 |
+
LLM_PROVIDERS: dict[str, LlmProvider] = {
|
| 349 |
+
"anthropic": LlmProvider(
|
| 350 |
+
name="anthropic", label="Anthropic", env="ANTHROPIC_API_KEY",
|
| 351 |
+
url="https://api.anthropic.com/v1/messages",
|
| 352 |
+
# β Claude Opus 5, $5 / $25 per million tokens (2026-06 list). Override per deployment with
|
| 353 |
+
# `AIOS_ANTHROPIC_MODEL` β the id is read at call time, so a cheaper tier
|
| 354 |
+
# (`claude-sonnet-5`, $3 / $15) is an environment change, not a release.
|
| 355 |
+
model=os.environ.get("AIOS_ANTHROPIC_MODEL") or "claude-opus-5",
|
| 356 |
+
wire="anthropic",
|
| 357 |
+
caps={
|
| 358 |
+
# β MEASURED, and it is the whole of R4's first clause: the Messages API answers with a
|
| 359 |
+
# typed `tool_use` content block carrying parsed `input`. There is nothing to recover
|
| 360 |
+
# out of a 400 and no regex in the path β which is exactly what `_FAILED_GEN` exists to
|
| 361 |
+
# apologise for on the other wire.
|
| 362 |
+
"llm_tool_calling": Capability(True, 0.0,
|
| 363 |
+
"typed tool_use content block; no text recovery path"),
|
| 364 |
+
"llm_chat": Capability(True, 0.0, "Messages API"),
|
| 365 |
+
"llm_json_mode": Capability(True, 0.0, "output_config.format, schema-constrained"),
|
| 366 |
+
}),
|
| 367 |
+
"cerebras": LlmProvider(
|
| 368 |
+
name="cerebras", label="Cerebras", env="CEREBRAS_API_KEY",
|
| 369 |
+
url="https://api.cerebras.ai/v1/chat/completions",
|
| 370 |
+
model="gpt-oss-120b", wire="openai",
|
| 371 |
+
caps={
|
| 372 |
+
"llm_tool_calling": Capability(True, 0.0,
|
| 373 |
+
"DECLARED, not measured: this account's tool-capable "
|
| 374 |
+
"model per the wave-32 ladder note"),
|
| 375 |
+
"llm_chat": Capability(True, 0.0, "OpenAI-compatible"),
|
| 376 |
+
"llm_json_mode": Capability(True, 0.0, "response_format json_object"),
|
| 377 |
+
}),
|
| 378 |
+
"groq": LlmProvider(
|
| 379 |
+
name="groq", label="Groq", env="GROQ_API_KEY",
|
| 380 |
+
url="https://api.groq.com/openai/v1/chat/completions",
|
| 381 |
+
model="llama-3.3-70b-versatile", wire="openai",
|
| 382 |
+
caps={
|
| 383 |
+
"llm_tool_calling": Capability(True, 0.0,
|
| 384 |
+
"DECLARED, not measured. β `routes_query._FAILED_GEN` "
|
| 385 |
+
"exists because SOME provider on this wire answers 400 "
|
| 386 |
+
"with the call in `failed_generation`; the repo never "
|
| 387 |
+
"recorded which. Flip this to False the day it is"),
|
| 388 |
+
"llm_chat": Capability(True, 0.0, "OpenAI-compatible"),
|
| 389 |
+
"llm_json_mode": Capability(True, 0.0, "response_format json_object"),
|
| 390 |
+
}),
|
| 391 |
+
"openrouter": LlmProvider(
|
| 392 |
+
name="openrouter", label="OpenRouter", env="OPENROUTER_API_KEY",
|
| 393 |
+
url="https://openrouter.ai/api/v1/chat/completions",
|
| 394 |
+
model="openai/gpt-4o-mini", wire="openai",
|
| 395 |
+
caps={
|
| 396 |
+
"llm_tool_calling": Capability(True, 0.0, "DECLARED, not measured"),
|
| 397 |
+
"llm_chat": Capability(True, 0.0, "OpenAI-compatible"),
|
| 398 |
+
"llm_json_mode": Capability(True, 0.0, "response_format json_object"),
|
| 399 |
+
}),
|
| 400 |
+
}
|
| 401 |
+
|
| 402 |
+
LLM_DEFAULT_ORDER = ("anthropic", "cerebras", "groq", "openrouter")
|
| 403 |
+
|
| 404 |
+
|
| 405 |
+
def mark_no_credit(name, seconds=None):
|
| 406 |
+
"""Remember that `name` said it is out of credit, so the next turn SKIPS it (R4).
|
| 407 |
+
|
| 408 |
+
β THE SECOND CLAUSE OF R4 IS "SKIPPED, NOT TRIED", and without a memo there is nowhere for
|
| 409 |
+
that to live: a stateless ladder re-tries the empty account on every single turn, pays its
|
| 410 |
+
round trip, and shows the reader a longer error each time. This is that memo.
|
| 411 |
+
"""
|
| 412 |
+
_NO_CREDIT[str(name)] = time.time() + float(
|
| 413 |
+
CREDIT_COOLDOWN_S if seconds is None else seconds)
|
| 414 |
+
return _NO_CREDIT[str(name)]
|
| 415 |
+
|
| 416 |
+
|
| 417 |
+
def no_credit(name):
|
| 418 |
+
"""Is this provider inside its out-of-credit cooldown? Expiry is checked, never assumed."""
|
| 419 |
+
until = _NO_CREDIT.get(str(name))
|
| 420 |
+
if not until:
|
| 421 |
+
return False
|
| 422 |
+
if time.time() >= until:
|
| 423 |
+
_NO_CREDIT.pop(str(name), None)
|
| 424 |
+
return False
|
| 425 |
+
return True
|
| 426 |
+
|
| 427 |
+
|
| 428 |
+
def clear_credit_memo(name=None):
|
| 429 |
+
"""Forget one memo, or all of them. For a gate, and for an operator after a top-up."""
|
| 430 |
+
if name is None:
|
| 431 |
+
_NO_CREDIT.clear()
|
| 432 |
+
else:
|
| 433 |
+
_NO_CREDIT.pop(str(name), None)
|
| 434 |
+
|
| 435 |
+
|
| 436 |
+
def llm_chain(capability="llm_tool_calling"):
|
| 437 |
+
"""The provider order for `capability` β declaration first, credit memo second.
|
| 438 |
+
|
| 439 |
+
Three filters, in this order, and each removes a DIFFERENT kind of row:
|
| 440 |
+
1. `can()` β declared capable AND configured. An incapable row is never offered, so a
|
| 441 |
+
turn cannot be spent discovering it (the `chain()` rule, one layer up).
|
| 442 |
+
2. `no_credit()` β R4's skip. A provider that told us its balance is empty is passed over
|
| 443 |
+
until the memo expires.
|
| 444 |
+
3. the ORDER itself, overridable with `AIOS_LLM_ORDER` (`anthropic,groq`) so a deployment
|
| 445 |
+
can be moved off a vendor without a release β the same clause `AIOS_PROVIDER_ORDER`
|
| 446 |
+
carries for the scraping side.
|
| 447 |
+
"""
|
| 448 |
+
raw = (os.environ.get("AIOS_LLM_ORDER") or "").strip()
|
| 449 |
+
names = [x.strip() for x in raw.split(",") if x.strip()] or list(LLM_DEFAULT_ORDER)
|
| 450 |
+
return [LLM_PROVIDERS[n] for n in names
|
| 451 |
+
if n in LLM_PROVIDERS and LLM_PROVIDERS[n].can(capability) and not no_credit(n)]
|
| 452 |
+
|
| 453 |
+
|
| 454 |
+
#: What an HTTP status MEANS, in words a person can act on. ββ R4's THIRD CLAUSE LIVES HERE AND
|
| 455 |
+
#: IT IS NOT COSMETIC: `HTTP 402` on a screen tells a reader nothing they can do, and the owner
|
| 456 |
+
#: quoted it back at us twice. Every sentence names the VENDOR and the ACTION.
|
| 457 |
+
_STATUS_WORDS = {
|
| 458 |
+
401: "{label} would not accept our key",
|
| 459 |
+
403: "{label} would not accept our key",
|
| 460 |
+
402: "{label} is out of credit",
|
| 461 |
+
404: "{label} does not offer the model we asked it for",
|
| 462 |
+
408: "{label} took too long",
|
| 463 |
+
413: "the question was too long for {label}",
|
| 464 |
+
429: "{label} is rate limiting us right now",
|
| 465 |
+
}
|
| 466 |
+
|
| 467 |
+
#: Substrings that mean "no money" on a wire that does not use 402. β Anthropic answers a spent
|
| 468 |
+
#: balance with a 400 or 403 carrying a message, not with a status code of its own, so this is the
|
| 469 |
+
#: one place a body has to be read. It is a LOWERCASE substring test on the vendor's own words and
|
| 470 |
+
#: it only ever decides whether to SKIP a provider, never whether to trust one.
|
| 471 |
+
_CREDIT_WORDS = ("credit balance", "insufficient credit", "insufficient_quota", "out of credit",
|
| 472 |
+
"quota exceeded", "billing", "payment required", "add credits")
|
| 473 |
+
|
| 474 |
+
|
| 475 |
+
def is_credit_failure(status, body=""):
|
| 476 |
+
"""Did this response mean "the account is empty"? Status first, then the vendor's own words."""
|
| 477 |
+
if int(status or 0) == 402:
|
| 478 |
+
return True
|
| 479 |
+
if int(status or 0) not in (400, 403, 429):
|
| 480 |
+
return False
|
| 481 |
+
return any(word in str(body or "").lower() for word in _CREDIT_WORDS)
|
| 482 |
+
|
| 483 |
+
|
| 484 |
+
def refusal_sentence(name, status, body=""):
|
| 485 |
+
"""One provider's failure, as a SENTENCE. Never a bare status code, never a vendor stack trace.
|
| 486 |
+
|
| 487 |
+
β THE BODY IS READ AND NEVER QUOTED. A provider's error body can carry an account id, a key
|
| 488 |
+
prefix or an internal trace; the only thing taken out of it is the yes/no answer to "is this a
|
| 489 |
+
credit problem", and what reaches the caller is this module's own wording.
|
| 490 |
+
"""
|
| 491 |
+
label = (LLM_PROVIDERS.get(str(name)) or LlmProvider(name, str(name), "", "", "", "")).label
|
| 492 |
+
if is_credit_failure(status, body):
|
| 493 |
+
return f"{label} is out of credit"
|
| 494 |
+
code = int(status or 0)
|
| 495 |
+
if code in _STATUS_WORDS:
|
| 496 |
+
return _STATUS_WORDS[code].format(label=label)
|
| 497 |
+
if 500 <= code <= 599:
|
| 498 |
+
return f"{label} is having trouble at their end"
|
| 499 |
+
return f"{label} did not answer"
|
| 500 |
+
|
| 501 |
+
|
| 502 |
+
# βββββββββββ THE ANTHROPIC WIRE, ONCE (W36-T35 / ASK D-18, ruling R4) βββββββββββββββββββββββββββ
|
| 503 |
+
#
|
| 504 |
+
# ββ TWO DOORS IN THIS PRODUCT CALL ANTHROPIC AND THEY MUST NOT EACH LEARN THE MESSAGES API.
|
| 505 |
+
# `routes_query._call_model` (the Assistant and Query) and `ai_review.draft_flow` (the automation
|
| 506 |
+
# drafter) both need it, and the owner quoted an error from EACH of them in one breath:
|
| 507 |
+
# *"the assistant could not be reached just now (openrouter: HTTP 402)"* and *"anthropic: tool
|
| 508 |
+
# calls are not wired for this shape"*. Two implementations of one wire is
|
| 509 |
+
# [[one-question-two-normalizers]] before a line is written, so the wire lives here, beside the
|
| 510 |
+
# ladder that declares the rung.
|
| 511 |
+
#
|
| 512 |
+
# FOUR THINGS THE OPENAI-COMPATIBLE SHAPE GETS WRONG, each a 400 on its own:
|
| 513 |
+
# 1. the system prompt is a TOP-LEVEL field, not a `{"role": "system"}` message
|
| 514 |
+
# 2. a tool is `{name, description, input_schema}` FLAT, not nested under `function`
|
| 515 |
+
# 3. `temperature` and friends are REMOVED on the current model family
|
| 516 |
+
# 4. `tool_choice` is an OBJECT (`{"type": "auto"}` / `{"type": "any"}`), not a string
|
| 517 |
+
#
|
| 518 |
+
# β AND ONE THING THAT IS NOT A SHAPE: `effort` is model-gated. `output_config.effort` errors on
|
| 519 |
+
# Haiku 4.5, so it is a PARAMETER here and the caller decides β the drafter runs on haiku and omits
|
| 520 |
+
# it, the assistant runs on the Opus tier and sends it.
|
| 521 |
+
|
| 522 |
+
#: The Messages API version header. A DATE that pins the WIRE FORMAT, never a model.
|
| 523 |
+
ANTHROPIC_VERSION = "2023-06-01"
|
| 524 |
+
|
| 525 |
+
|
| 526 |
+
def anthropic_request(*, model, key, system, messages, tools, max_tokens,
|
| 527 |
+
tool_choice="auto", effort=None):
|
| 528 |
+
"""`{url, headers, json}` for one Messages API call. Pure: reads no environment, sends nothing.
|
| 529 |
+
|
| 530 |
+
`messages` is the OpenAI-shaped list this product already builds; the `system` turns are lifted
|
| 531 |
+
out of it, because that is where this API wants them. `tools` is the OpenAI-shaped tool list,
|
| 532 |
+
re-addressed rather than re-derived, so a schema change happens in one place.
|
| 533 |
+
"""
|
| 534 |
+
system_text = "\n\n".join(str(m.get("content") or "") for m in messages
|
| 535 |
+
if m.get("role") == "system")
|
| 536 |
+
if system:
|
| 537 |
+
system_text = (system_text + "\n\n" + str(system)).strip() if system_text else str(system)
|
| 538 |
+
turns = [{"role": ("assistant" if m.get("role") == "assistant" else "user"),
|
| 539 |
+
"content": str(m.get("content") or "")}
|
| 540 |
+
for m in messages if m.get("role") != "system" and str(m.get("content") or "").strip()]
|
| 541 |
+
body = {
|
| 542 |
+
"model": str(model),
|
| 543 |
+
"max_tokens": int(max_tokens),
|
| 544 |
+
"system": system_text,
|
| 545 |
+
"messages": turns,
|
| 546 |
+
"tools": [{"name": t["function"]["name"],
|
| 547 |
+
"description": t["function"]["description"],
|
| 548 |
+
"input_schema": t["function"]["parameters"]} for t in (tools or [])],
|
| 549 |
+
"tool_choice": {"type": "any" if tool_choice in ("required", "any") else "auto"},
|
| 550 |
+
}
|
| 551 |
+
if effort:
|
| 552 |
+
body["output_config"] = {"effort": str(effort)}
|
| 553 |
+
return {"url": LLM_PROVIDERS["anthropic"].url,
|
| 554 |
+
"headers": {"x-api-key": str(key),
|
| 555 |
+
"anthropic-version": ANTHROPIC_VERSION,
|
| 556 |
+
"content-type": "application/json"},
|
| 557 |
+
"json": body}
|
| 558 |
+
|
| 559 |
+
|
| 560 |
+
def anthropic_read(body):
|
| 561 |
+
"""`(text, tool_input, refusal)` out of a Messages API answer.
|
| 562 |
+
|
| 563 |
+
β `stop_reason` IS CHECKED BEFORE `content` IS READ. A safety decline answers **HTTP 200** with
|
| 564 |
+
`stop_reason: "refusal"` and an empty or partial `content`, so code that indexes `content[0]`
|
| 565 |
+
unconditionally breaks on exactly the turn a person most needs explained.
|
| 566 |
+
β AND THE TOOL CALL ARRIVES PARSED. `tool_use.input` is already a dict β no `json.loads`, and
|
| 567 |
+
no regex recovering a call out of a 400, which is what the other wire needs.
|
| 568 |
+
"""
|
| 569 |
+
body = body if isinstance(body, dict) else {}
|
| 570 |
+
if str(body.get("stop_reason") or "") == "refusal":
|
| 571 |
+
return "", None, "the assistant declined to answer that one"
|
| 572 |
+
blocks = [b for b in (body.get("content") or []) if isinstance(b, dict)]
|
| 573 |
+
text = " ".join(str(b.get("text") or "") for b in blocks if b.get("type") == "text").strip()
|
| 574 |
+
calls = [b for b in blocks if b.get("type") == "tool_use"]
|
| 575 |
+
got = calls[0].get("input") if calls else None
|
| 576 |
+
return text, (got if isinstance(got, dict) else None), None
|
| 577 |
+
|
| 578 |
+
|
| 579 |
+
def llm_status(capability="llm_tool_calling"):
|
| 580 |
+
"""Per provider: configured, declared-capable, in cooldown, and WHY β contract C5's payload.
|
| 581 |
+
|
| 582 |
+
β ONE LIST, ONE DOOR. The Assistant's model picker and the Agent chat's toggle (W36-T34) read
|
| 583 |
+
THIS, so a model offered in one place cannot be missing from the other, and neither can offer a
|
| 584 |
+
provider the ladder would refuse to call [[permitted-is-not-answerable]].
|
| 585 |
+
"""
|
| 586 |
+
rows = []
|
| 587 |
+
for name in LLM_DEFAULT_ORDER:
|
| 588 |
+
p = LLM_PROVIDERS[name]
|
| 589 |
+
cap = p.caps.get(capability)
|
| 590 |
+
rows.append({
|
| 591 |
+
"provider": p.name, "label": p.label, "model": p.model, "wire": p.wire,
|
| 592 |
+
"configured": p.configured(),
|
| 593 |
+
"toolCalling": bool((p.caps.get("llm_tool_calling") or Capability(False)).capable),
|
| 594 |
+
"jsonMode": bool((p.caps.get("llm_json_mode") or Capability(False)).capable),
|
| 595 |
+
"capable": bool(cap and cap.capable),
|
| 596 |
+
"outOfCredit": no_credit(name),
|
| 597 |
+
"note": (cap.note if cap else ""),
|
| 598 |
+
})
|
| 599 |
+
return rows
|
| 600 |
+
|
| 601 |
+
|
| 602 |
# =============================================================================================
|
| 603 |
# THE CANONICAL SCHEMA β owner ruling 2026-08-08:
|
| 604 |
# *"standardize the schema between Bright Data and APIfy so we keep using the same pre-set
|
api/routes_admin.py
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
api/routes_agent_harness.py
ADDED
|
@@ -0,0 +1,455 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""routes_agent_harness.py β CONTRACT C4: the agent's HARNESS, kept as versioned files.
|
| 2 |
+
|
| 3 |
+
Owner item 3, verbatim (2026-08-18): *"This new module under agent is supposed to host any file
|
| 4 |
+
pertaining to the agent skills, router etc. So we build a user can build a custom harness for each
|
| 5 |
+
agent through the chat interface."*
|
| 6 |
+
|
| 7 |
+
GET /api/v1/agents/{id}/harness the file LIST (no bodies)
|
| 8 |
+
GET /api/v1/agents/{id}/harness?path=β¦ one file: current body + its versions
|
| 9 |
+
GET /api/v1/agents/{id}/harness?path=β¦&version=N one older body, verbatim
|
| 10 |
+
PUT /api/v1/agents/{id}/harness write a NEW version of one file
|
| 11 |
+
DELETE /api/v1/agents/{id}/harness?path=β¦ drop a file and its history
|
| 12 |
+
|
| 13 |
+
ββ **R8 IS THE WHOLE SHAPE: BOTH PRINCIPALS WRITE, AND NOTHING IS EVER OVERWRITTEN.** A write
|
| 14 |
+
appends a version; a roll-back is a write of an older body (`restoredFrom`), never a delete. So the
|
| 15 |
+
history is a record of what happened rather than of what somebody last wanted it to look like.
|
| 16 |
+
|
| 17 |
+
β **R8 IS DELIBERATELY NOT R9.** An agent-authored ACTION's configuration is agent-only (item 8,
|
| 18 |
+
`routes_automation`); a harness file is not. Do not copy this file's posture over there or that
|
| 19 |
+
one's over here β the two rulings differ on purpose and they sit one screen apart in the product.
|
| 20 |
+
|
| 21 |
+
β **`author` IS THE SESSION, `authorKind` IS THE PROVENANCE, AND THEY ARE DIFFERENT FACTS.** Every
|
| 22 |
+
write through this router is made BY a signed-in administrator, so `author` is stamped from the
|
| 23 |
+
session and can never be supplied by the caller. `authorKind` says whether the BODY was drafted by
|
| 24 |
+
a person or by the agent in the chat β a claim the client is entitled to make, because both are
|
| 25 |
+
permitted (R8) and so nothing is bought by forging it. `record_version()` below is the server-side
|
| 26 |
+
door the automation engine uses when the agent writes with no session at all; that one stamps the
|
| 27 |
+
agent's own id as the author, which is the only case where `author` is not a username.
|
| 28 |
+
|
| 29 |
+
β **THE VERSION LIST IS CAPPED AND THE CAP IS REPORTED, NEVER SILENT.** `MAX_VERSIONS` versions of
|
| 30 |
+
one file are kept; past that the OLDEST are dropped and the count of what was dropped rides in
|
| 31 |
+
`trimmed` on every payload that mentions the file, so a reader can see that the history is partial
|
| 32 |
+
rather than infer that the file was only ever saved twice. This is the tenant document, which is
|
| 33 |
+
already 28.6 MB on tenant #0 and is deep-copied on every read: an unbounded per-agent history is
|
| 34 |
+
a store-sized leak with a UI in front of it.
|
| 35 |
+
"""
|
| 36 |
+
from datetime import datetime, timezone
|
| 37 |
+
|
| 38 |
+
from fastapi import APIRouter, Body, Depends
|
| 39 |
+
|
| 40 |
+
from deps import Session, err, require_session
|
| 41 |
+
|
| 42 |
+
router = APIRouter(prefix="/api/v1")
|
| 43 |
+
|
| 44 |
+
#: The tenant's harness files: `{agent_id: {path: file_record}}`. A per-tenant bucket, so it rides
|
| 45 |
+
#: `runtime.store_key`'s prefix and never lands in tenant #0's namespace β the same rule
|
| 46 |
+
#: `routes_slack.AGENTS_KEY` follows for the agent records these hang off.
|
| 47 |
+
HARNESS_KEY = "agent_harness"
|
| 48 |
+
|
| 49 |
+
#: One body. Generous for a skill or a router file and far under the point where a single write
|
| 50 |
+
#: would move the tenant document measurably. A larger body is a REFUSAL at the door, not a
|
| 51 |
+
#: truncation: a silently truncated skill file is a harness that does not do what its text says.
|
| 52 |
+
MAX_BODY_BYTES = 128 * 1024
|
| 53 |
+
|
| 54 |
+
#: Files per agent. A refusal, not a trim β creating the 65th file is a different act from saving
|
| 55 |
+
#: the 101st version of one, and only the second can be a routine consequence of ordinary editing.
|
| 56 |
+
MAX_FILES = 64
|
| 57 |
+
|
| 58 |
+
#: Versions kept per file. Past this the oldest go and `trimmed` counts them (see the header).
|
| 59 |
+
MAX_VERSIONS = 100
|
| 60 |
+
|
| 61 |
+
MAX_PATH = 200
|
| 62 |
+
|
| 63 |
+
#: What a path may contain. β THIS IS NOT A FILESYSTEM PATH AND NOTHING HERE EVER TOUCHES A DISK β
|
| 64 |
+
#: the "files" are keys in a store bucket. The character rule exists so the key is displayable, is
|
| 65 |
+
#: safe to put in a URL, and cannot carry a traversal sequence that would look meaningful to a
|
| 66 |
+
#: future reader who assumes it IS a filesystem path. Fail-closed on the character set, not on a
|
| 67 |
+
#: list of forbidden sequences: an allow-list cannot be walked around by a spelling.
|
| 68 |
+
_PATH_OK = set("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789._-/")
|
| 69 |
+
|
| 70 |
+
AUTHOR_KINDS = ("user", "agent")
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
def _now():
|
| 74 |
+
return datetime.now(timezone.utc).isoformat(timespec="seconds")
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
def _all(rt):
|
| 78 |
+
"""`{agent_id: {path: record}}` for one tenant. `{}` on any failure β an unreadable bucket must
|
| 79 |
+
degrade to "this agent has no harness files", never to a 500 on the pane that lists them."""
|
| 80 |
+
try:
|
| 81 |
+
found = rt.get(HARNESS_KEY) or {}
|
| 82 |
+
except Exception: # noqa: BLE001
|
| 83 |
+
return {}
|
| 84 |
+
return found if isinstance(found, dict) else {}
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
def _files(rt, agent_id):
|
| 88 |
+
found = _all(rt).get(str(agent_id))
|
| 89 |
+
return found if isinstance(found, dict) else {}
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
#: What kind of principal an id names. `None` = this tenant has no agent with that id.
|
| 93 |
+
#:
|
| 94 |
+
#: ββ THERE ARE TWO AGENT REGISTRIES IN THIS PRODUCT AND THE FIRST DRAFT OF THIS FILE KNEW ONLY
|
| 95 |
+
#: ONE (ASK D-5, 2026-08-18). `routes_slack._agents` is the per-Slack-channel permission wall under
|
| 96 |
+
#: Manage users. The **Agents module** the owner's item 3 is about is the AUTOMATION surface β
|
| 97 |
+
#: `Shell.tsx` mounts `<Lazily surface="Agents"><AutomationSurface /></Lazily>`, and
|
| 98 |
+
#: `AutomationDetail`'s `panelTabs` is literally the `Properties | Run history` strip R7 adds
|
| 99 |
+
#: "Harness" to. Keyed on the Slack bucket alone, every `GET/PUT /agents/{id}/harness` from that
|
| 100 |
+
#: tab would have answered **404 no_agent**: whole, gate-green and dead on arrival
|
| 101 |
+
#: [[reachable-is-not-the-same-as-built]]. Verified independently before acting, not taken on
|
| 102 |
+
#: report: `surface="Agents"` mounts `AutomationSurface`, and `ManageAgentPane` contains ZERO
|
| 103 |
+
#: occurrences of Canvas, Properties, Run history or panelTabs.
|
| 104 |
+
#:
|
| 105 |
+
#: β ONE STORE, BOTH PRINCIPALS β never a second bucket keyed by surface. The owner's "agent
|
| 106 |
+
#: skills, router" files in two places is the parallel code path item 13 exists to refuse.
|
| 107 |
+
AGENT_SLACK = "slack"
|
| 108 |
+
AGENT_AUTOMATION = "automation"
|
| 109 |
+
|
| 110 |
+
|
| 111 |
+
def agent_kind(rt, agent_id):
|
| 112 |
+
"""Which registry holds this id β `AGENT_SLACK`, `AGENT_AUTOMATION`, or `None`.
|
| 113 |
+
|
| 114 |
+
β A SYNTHETIC ROW IS NOT AN AGENT FOR THIS PURPOSE. `field:` and `system:` ids are DERIVED at
|
| 115 |
+
read time from a column definition or a connector schedule; they have no stored home, so a
|
| 116 |
+
harness file hung off one is an orphan the moment the column changes. `patch_automation`
|
| 117 |
+
already refuses those ids for the neighbouring reason, and this refuses them by simply not
|
| 118 |
+
finding them β `all_definitions` holds stored automations only.
|
| 119 |
+
"""
|
| 120 |
+
import routes_slack
|
| 121 |
+
aid = str(agent_id or "")
|
| 122 |
+
if isinstance(routes_slack._agents(rt).get(aid), dict):
|
| 123 |
+
return AGENT_SLACK
|
| 124 |
+
import automation_engine as engine
|
| 125 |
+
try:
|
| 126 |
+
known = engine.all_definitions(rt) or {}
|
| 127 |
+
except Exception: # noqa: BLE001
|
| 128 |
+
return None
|
| 129 |
+
return AGENT_AUTOMATION if aid in known else None
|
| 130 |
+
|
| 131 |
+
|
| 132 |
+
def _known_agent(rt, agent_id):
|
| 133 |
+
"""Does this tenant have an agent with this id, in EITHER registry?"""
|
| 134 |
+
return agent_kind(rt, agent_id) is not None
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
def agent_wall(session, agent_id):
|
| 138 |
+
"""404 for an unknown id, else apply the wall THAT PRINCIPAL'S OWN SURFACE applies.
|
| 139 |
+
|
| 140 |
+
ββ ONE DOOR, TWO WALLS, AND THAT IS NOT A SECOND CODE PATH β it is the refusal to invent a
|
| 141 |
+
THIRD wall. A Slack channel agent is administered under Manage users and every `/agents/*` door
|
| 142 |
+
in `routes_slack` is `admin_gate`; an automation lives in the Agents module and every door in
|
| 143 |
+
`routes_automation` is `module_gate("automation")`. A harness file is configuration OF the
|
| 144 |
+
agent it hangs off, so it is reached by whoever may already configure that agent. Picking one
|
| 145 |
+
of the two walls for both would either lock the Agents module's own users out of a tab the
|
| 146 |
+
owner asked for, or hand the Slack permission wall to anyone with an automation grant.
|
| 147 |
+
"""
|
| 148 |
+
kind = agent_kind(session.runtime, agent_id)
|
| 149 |
+
if kind is None:
|
| 150 |
+
raise err(404, "no_agent", "there is no agent with that id in this workspace")
|
| 151 |
+
if kind == AGENT_SLACK:
|
| 152 |
+
import core.perms as perms
|
| 153 |
+
if not perms.is_admin(session.user):
|
| 154 |
+
raise err(403, "forbidden", "administrators only")
|
| 155 |
+
else:
|
| 156 |
+
session.require("automation")
|
| 157 |
+
return kind
|
| 158 |
+
|
| 159 |
+
|
| 160 |
+
def normalize_path(raw):
|
| 161 |
+
"""THE path rule, in ONE place. Returns the cleaned path, or `None` if it is not acceptable.
|
| 162 |
+
|
| 163 |
+
ββ ONE RULE, TWO DOORS, AND THAT IS WHY THIS IS A FUNCTION RATHER THAN TWO IF-BLOCKS. There
|
| 164 |
+
are two ways into this store β the HTTP route (which must answer 400) and `record_version()`
|
| 165 |
+
(which must raise `ValueError`, having no response to put a status into). Written twice, the
|
| 166 |
+
two copies are [[one-question-two-normalizers]] waiting to happen: the first weakening of one
|
| 167 |
+
copy is invisible because the other still refuses, so nothing goes red and the wall is now
|
| 168 |
+
half there. Written once, a change to the rule is felt at both doors and by the gate.
|
| 169 |
+
"""
|
| 170 |
+
path = str(raw or "").strip().strip("/")
|
| 171 |
+
if not path or len(path) > MAX_PATH:
|
| 172 |
+
return None
|
| 173 |
+
if set(path) - _PATH_OK or ".." in path or "//" in path:
|
| 174 |
+
return None
|
| 175 |
+
return path
|
| 176 |
+
|
| 177 |
+
|
| 178 |
+
def _clean_path(raw):
|
| 179 |
+
"""`normalize_path` at the HTTP door, where a refusal is a 400 with a reason."""
|
| 180 |
+
path = normalize_path(raw)
|
| 181 |
+
if path is None:
|
| 182 |
+
if not str(raw or "").strip().strip("/"):
|
| 183 |
+
raise err(400, "no_path", "a harness file needs a path, for example skills/router.md")
|
| 184 |
+
if len(str(raw)) > MAX_PATH:
|
| 185 |
+
raise err(400, "path_too_long", f"a harness path is at most {MAX_PATH} characters")
|
| 186 |
+
raise err(400, "bad_path",
|
| 187 |
+
"a harness path may use letters, digits, dot, dash, underscore and / only")
|
| 188 |
+
return path
|
| 189 |
+
|
| 190 |
+
|
| 191 |
+
def _blank(path, author, author_kind):
|
| 192 |
+
return {"path": path, "versions": [], "trimmed": 0,
|
| 193 |
+
"created": _now(), "createdBy": author, "createdKind": author_kind}
|
| 194 |
+
|
| 195 |
+
|
| 196 |
+
def _append(record, body, author, author_kind, restored_from=None):
|
| 197 |
+
"""Append ONE version to a file record, in place, and report what was trimmed.
|
| 198 |
+
|
| 199 |
+
The version NUMBER is monotonic and survives trimming β it counts writes, not stored entries.
|
| 200 |
+
A version list whose numbers restart at 1 after a trim would make two different bodies share a
|
| 201 |
+
name, and `restoredFrom` would then point at whichever one happened to be in the window.
|
| 202 |
+
"""
|
| 203 |
+
# β A NEW LIST, NEVER `versions.append(...)` ON THE STORED ONE. `update()` hands the callback
|
| 204 |
+
# the live document and may run it more than once; appending in place would then stack two
|
| 205 |
+
# copies of the same version into the history on a retry.
|
| 206 |
+
prior = record.get("versions") if isinstance(record.get("versions"), list) else []
|
| 207 |
+
last = max((int(v.get("version") or 0) for v in prior if isinstance(v, dict)), default=0)
|
| 208 |
+
entry = {"version": last + 1, "body": body, "author": author, "authorKind": author_kind,
|
| 209 |
+
"created": _now(), "bytes": len(body.encode("utf-8"))}
|
| 210 |
+
if restored_from:
|
| 211 |
+
entry["restoredFrom"] = int(restored_from)
|
| 212 |
+
versions = [*prior, entry]
|
| 213 |
+
dropped = max(0, len(versions) - MAX_VERSIONS)
|
| 214 |
+
if dropped:
|
| 215 |
+
versions = versions[dropped:]
|
| 216 |
+
record["versions"] = versions
|
| 217 |
+
record["trimmed"] = int(record.get("trimmed") or 0) + dropped
|
| 218 |
+
return entry
|
| 219 |
+
|
| 220 |
+
|
| 221 |
+
def _head(record):
|
| 222 |
+
"""The newest version of a file record, or `None` for a record with no versions at all."""
|
| 223 |
+
versions = record.get("versions") if isinstance(record.get("versions"), list) else []
|
| 224 |
+
return versions[-1] if versions else None
|
| 225 |
+
|
| 226 |
+
|
| 227 |
+
def _row(record):
|
| 228 |
+
"""One file, as the LIST door reports it: everything except the bodies.
|
| 229 |
+
|
| 230 |
+
β NO BODY, AND THAT IS THE POINT. A list door that carried every version of every file would
|
| 231 |
+
ship the whole harness on every pane render; the Harness tab lists first and opens one file
|
| 232 |
+
second, which is exactly the shape this answers.
|
| 233 |
+
"""
|
| 234 |
+
head = _head(record) or {}
|
| 235 |
+
versions = record.get("versions") if isinstance(record.get("versions"), list) else []
|
| 236 |
+
return {"path": record.get("path") or "",
|
| 237 |
+
"version": int(head.get("version") or 0),
|
| 238 |
+
"bytes": int(head.get("bytes") or 0),
|
| 239 |
+
"author": head.get("author") or record.get("createdBy") or "",
|
| 240 |
+
"authorKind": head.get("authorKind") or record.get("createdKind") or "user",
|
| 241 |
+
"updated": head.get("created") or record.get("created") or "",
|
| 242 |
+
"created": record.get("created") or "",
|
| 243 |
+
"versions": len(versions),
|
| 244 |
+
"trimmed": int(record.get("trimmed") or 0)}
|
| 245 |
+
|
| 246 |
+
|
| 247 |
+
def _version_rows(record):
|
| 248 |
+
"""The history of one file, newest first, WITHOUT the bodies.
|
| 249 |
+
|
| 250 |
+
A body per version is what makes a diff possible, and it is also what makes this payload big:
|
| 251 |
+
100 versions of a 128 KB file is 12 MB. The client asks for the two bodies it is diffing
|
| 252 |
+
(`?path=β¦&version=N`), which is two round trips for a diff and none for a history list.
|
| 253 |
+
"""
|
| 254 |
+
versions = record.get("versions") if isinstance(record.get("versions"), list) else []
|
| 255 |
+
out = []
|
| 256 |
+
for entry in reversed(versions):
|
| 257 |
+
if not isinstance(entry, dict):
|
| 258 |
+
continue
|
| 259 |
+
row = {"version": int(entry.get("version") or 0),
|
| 260 |
+
"author": entry.get("author") or "",
|
| 261 |
+
"authorKind": entry.get("authorKind") or "user",
|
| 262 |
+
"created": entry.get("created") or "",
|
| 263 |
+
"bytes": int(entry.get("bytes") or 0)}
|
| 264 |
+
if entry.get("restoredFrom"):
|
| 265 |
+
row["restoredFrom"] = int(entry["restoredFrom"])
|
| 266 |
+
out.append(row)
|
| 267 |
+
return out
|
| 268 |
+
|
| 269 |
+
|
| 270 |
+
def _limits():
|
| 271 |
+
"""The caps, IN the payload, so a client can say "this file is full" before a write fails.
|
| 272 |
+
|
| 273 |
+
β A limit the client cannot see is a limit the user meets as an error. `trimmed` reports the
|
| 274 |
+
one cap that acts without refusing; these report the three that refuse.
|
| 275 |
+
"""
|
| 276 |
+
return {"maxBodyBytes": MAX_BODY_BYTES, "maxFiles": MAX_FILES,
|
| 277 |
+
"maxVersions": MAX_VERSIONS, "maxPath": MAX_PATH}
|
| 278 |
+
|
| 279 |
+
|
| 280 |
+
# ββ the write door the SERVER uses (no session) ββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 281 |
+
def record_version(runtime, agent_id, path, body, author="", author_kind="agent",
|
| 282 |
+
restored_from=None):
|
| 283 |
+
"""Write one version from INSIDE the server β the agent's own half of R8.
|
| 284 |
+
|
| 285 |
+
β THIS IS THE FUNCTION, NOT THE ROUTE, THAT MAKES "editable by BOTH the user and the agent"
|
| 286 |
+
true. An agent acting inside an automation run holds no session and no cookie; if its only way
|
| 287 |
+
to write were the HTTP door it would have to borrow a person's identity, and the authorship
|
| 288 |
+
column would then be a record of who was logged in rather than of what wrote the file.
|
| 289 |
+
|
| 290 |
+
Returns the appended entry. Raises nothing the caller cannot handle: an unknown agent is a
|
| 291 |
+
`ValueError`, because a server-side caller has no HTTP response to put a 404 into.
|
| 292 |
+
"""
|
| 293 |
+
agent_id = str(agent_id or "")
|
| 294 |
+
if not _known_agent(runtime, agent_id):
|
| 295 |
+
raise ValueError(f"no agent {agent_id!r} in this workspace")
|
| 296 |
+
path = normalize_path(path)
|
| 297 |
+
if path is None:
|
| 298 |
+
raise ValueError("that is not an acceptable harness path")
|
| 299 |
+
body = str(body or "")
|
| 300 |
+
if len(body.encode("utf-8")) > MAX_BODY_BYTES:
|
| 301 |
+
raise ValueError("harness body is over the size limit")
|
| 302 |
+
kind = author_kind if author_kind in AUTHOR_KINDS else "agent"
|
| 303 |
+
author = str(author or agent_id)
|
| 304 |
+
# β THE FILE-COUNT CEILING IS CHECKED HERE, NOT INSIDE `_set`. An exception raised inside a
|
| 305 |
+
# store `update` callback propagates out of a half-run read-modify-write, and the one thing a
|
| 306 |
+
# refusal must never do is leave the caller unsure whether the write happened.
|
| 307 |
+
existing = _files(runtime, agent_id)
|
| 308 |
+
if path not in existing and len(existing) >= MAX_FILES:
|
| 309 |
+
raise ValueError(f"this agent already has {MAX_FILES} harness files")
|
| 310 |
+
appended = {}
|
| 311 |
+
|
| 312 |
+
def _set(cur):
|
| 313 |
+
cur = dict(cur or {})
|
| 314 |
+
files = dict(cur.get(agent_id) or {}) if isinstance(cur.get(agent_id), dict) else {}
|
| 315 |
+
record = dict(files[path]) if isinstance(files.get(path), dict) else _blank(path, author, kind)
|
| 316 |
+
appended.clear()
|
| 317 |
+
appended.update(_append(record, body, author, kind, restored_from))
|
| 318 |
+
files[path] = record
|
| 319 |
+
cur[agent_id] = files
|
| 320 |
+
return cur
|
| 321 |
+
|
| 322 |
+
runtime.update(HARNESS_KEY, _set, flush="sync")
|
| 323 |
+
return appended
|
| 324 |
+
|
| 325 |
+
|
| 326 |
+
# ββ the routes ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 327 |
+
@router.get("/agents/{agent_id}/harness")
|
| 328 |
+
def get_harness(agent_id: str, path: str = "", version: int = 0,
|
| 329 |
+
session: Session = Depends(require_session)):
|
| 330 |
+
"""The list, one file, or one older body β decided by the query string (C4).
|
| 331 |
+
|
| 332 |
+
β ADMIN-GATED LIKE EVERY OTHER AGENT DOOR (`routes_slack`), and for a stronger reason than
|
| 333 |
+
consistency: a harness file is what the agent is INSTRUCTED to do, so writing one is closer to
|
| 334 |
+
editing a permission than to editing a document.
|
| 335 |
+
"""
|
| 336 |
+
agent_id = str(agent_id or "")
|
| 337 |
+
agent_wall(session, agent_id)
|
| 338 |
+
files = _files(session.runtime, agent_id)
|
| 339 |
+
if not path:
|
| 340 |
+
rows = [_row(rec) for _p, rec in sorted(files.items()) if isinstance(rec, dict)]
|
| 341 |
+
return {"agent": agent_id, "files": rows, "limits": _limits()}
|
| 342 |
+
|
| 343 |
+
wanted = _clean_path(path)
|
| 344 |
+
record = files.get(wanted)
|
| 345 |
+
if not isinstance(record, dict):
|
| 346 |
+
raise err(404, "no_file", f"this agent has no harness file at {wanted}")
|
| 347 |
+
versions = record.get("versions") if isinstance(record.get("versions"), list) else []
|
| 348 |
+
if version:
|
| 349 |
+
for entry in versions:
|
| 350 |
+
if isinstance(entry, dict) and int(entry.get("version") or 0) == int(version):
|
| 351 |
+
return {"agent": agent_id, **_row(record), "body": entry.get("body") or "",
|
| 352 |
+
"atVersion": int(version), "history": _version_rows(record),
|
| 353 |
+
"limits": _limits()}
|
| 354 |
+
# β A TRIMMED VERSION IS A NAMED REFUSAL, NOT A 404 SHAPED LIKE A TYPO. The client asked
|
| 355 |
+
# for something that existed and no longer does, and telling it apart from a bad number is
|
| 356 |
+
# the difference between "roll back to v3" failing loudly and failing as if v3 never was.
|
| 357 |
+
trimmed = int(record.get("trimmed") or 0)
|
| 358 |
+
if trimmed and int(version) <= trimmed:
|
| 359 |
+
raise err(410, "version_trimmed",
|
| 360 |
+
f"version {int(version)} is older than the {MAX_VERSIONS} versions kept "
|
| 361 |
+
f"for this file, and its body is gone")
|
| 362 |
+
raise err(404, "no_version", f"this file has no version {int(version)}")
|
| 363 |
+
head = _head(record) or {}
|
| 364 |
+
return {"agent": agent_id, **_row(record), "body": head.get("body") or "",
|
| 365 |
+
"atVersion": int(head.get("version") or 0), "history": _version_rows(record),
|
| 366 |
+
"limits": _limits()}
|
| 367 |
+
|
| 368 |
+
|
| 369 |
+
@router.put("/agents/{agent_id}/harness")
|
| 370 |
+
def put_harness(agent_id: str, body: dict = Body(default=None),
|
| 371 |
+
session: Session = Depends(require_session)):
|
| 372 |
+
"""Write a NEW version of one harness file. `{path, body, authorKind?, restoredFrom?}` (C4).
|
| 373 |
+
|
| 374 |
+
β THERE IS NO OVERWRITE HERE AND THERE IS NO EDIT-IN-PLACE. R8's "every version kept" is not a
|
| 375 |
+
UI affordance; it is this function refusing to have a code path that replaces a body. A
|
| 376 |
+
roll-back arrives as an ordinary write carrying `restoredFrom`, so the history records that
|
| 377 |
+
somebody went back rather than pretending the intervening versions never happened.
|
| 378 |
+
"""
|
| 379 |
+
agent_id = str(agent_id or "")
|
| 380 |
+
agent_wall(session, agent_id)
|
| 381 |
+
body = body if isinstance(body, dict) else {}
|
| 382 |
+
path = _clean_path(body.get("path"))
|
| 383 |
+
text = body.get("body")
|
| 384 |
+
if not isinstance(text, str):
|
| 385 |
+
raise err(400, "no_body", "a harness file needs a body, even an empty one")
|
| 386 |
+
if len(text.encode("utf-8")) > MAX_BODY_BYTES:
|
| 387 |
+
raise err(413, "body_too_long",
|
| 388 |
+
f"a harness file is at most {MAX_BODY_BYTES // 1024} KB; this one is larger")
|
| 389 |
+
|
| 390 |
+
# β THE CALLER DECLARES THE PROVENANCE AND THE SERVER STAMPS THE IDENTITY. `authorKind` is a
|
| 391 |
+
# claim about who WROTE the text (the person, or the agent in the chat panel); `author` is the
|
| 392 |
+
# session and is never read off the request. Nothing is bought by forging the first β both
|
| 393 |
+
# principals may write (R8) β and everything would be bought by forging the second.
|
| 394 |
+
kind = str(body.get("authorKind") or "user").strip().lower()
|
| 395 |
+
if kind not in AUTHOR_KINDS:
|
| 396 |
+
raise err(400, "bad_author_kind", "authorKind is either user or agent")
|
| 397 |
+
restored = body.get("restoredFrom")
|
| 398 |
+
try:
|
| 399 |
+
restored = int(restored) if restored else None
|
| 400 |
+
except (TypeError, ValueError):
|
| 401 |
+
raise err(400, "bad_version", "restoredFrom must be a version number")
|
| 402 |
+
|
| 403 |
+
files = _files(session.runtime, agent_id)
|
| 404 |
+
if path not in files and len(files) >= MAX_FILES:
|
| 405 |
+
raise err(409, "too_many_files",
|
| 406 |
+
f"this agent already has {MAX_FILES} harness files; delete one to add another")
|
| 407 |
+
try:
|
| 408 |
+
record_version(session.runtime, agent_id, path, text,
|
| 409 |
+
author=session.uname, author_kind=kind, restored_from=restored)
|
| 410 |
+
except ValueError as exc:
|
| 411 |
+
raise err(400, "refused", str(exc))
|
| 412 |
+
|
| 413 |
+
fresh = _files(session.runtime, agent_id).get(path)
|
| 414 |
+
if not isinstance(fresh, dict) or not _head(fresh):
|
| 415 |
+
# The store took the write and did not record it. A 200 here would tell an administrator
|
| 416 |
+
# their skill file was saved when it was not β the shape `routes_slack` refuses too.
|
| 417 |
+
raise err(503, "store_unavailable", "the harness file was NOT saved")
|
| 418 |
+
head = _head(fresh)
|
| 419 |
+
return {"agent": agent_id, **_row(fresh), "body": head.get("body") or "",
|
| 420 |
+
"atVersion": int(head.get("version") or 0), "history": _version_rows(fresh),
|
| 421 |
+
"limits": _limits()}
|
| 422 |
+
|
| 423 |
+
|
| 424 |
+
@router.delete("/agents/{agent_id}/harness")
|
| 425 |
+
def delete_harness(agent_id: str, path: str = "", session: Session = Depends(require_session)):
|
| 426 |
+
"""Drop one harness file AND its history.
|
| 427 |
+
|
| 428 |
+
β THIS IS NOT THE THING R8 FORBIDS. R8 forbids a ROLL-BACK implemented as a delete β losing
|
| 429 |
+
versions as a side effect of an edit. Deleting a file is a person deciding the file should not
|
| 430 |
+
exist, which is a different act with a different button, and a store with no way to remove a
|
| 431 |
+
file is one where a typo'd path is permanent.
|
| 432 |
+
"""
|
| 433 |
+
agent_id = str(agent_id or "")
|
| 434 |
+
agent_wall(session, agent_id)
|
| 435 |
+
wanted = _clean_path(path)
|
| 436 |
+
if wanted not in _files(session.runtime, agent_id):
|
| 437 |
+
raise err(404, "no_file", f"this agent has no harness file at {wanted}")
|
| 438 |
+
|
| 439 |
+
def _set(cur):
|
| 440 |
+
cur = dict(cur or {})
|
| 441 |
+
files = dict(cur.get(agent_id) or {}) if isinstance(cur.get(agent_id), dict) else {}
|
| 442 |
+
files.pop(wanted, None)
|
| 443 |
+
# An agent with no harness files leaves NO key behind. An empty dict per agent id is how a
|
| 444 |
+
# bucket accumulates a row for every agent anybody ever opened the tab on.
|
| 445 |
+
if files:
|
| 446 |
+
cur[agent_id] = files
|
| 447 |
+
else:
|
| 448 |
+
cur.pop(agent_id, None)
|
| 449 |
+
return cur
|
| 450 |
+
|
| 451 |
+
session.runtime.update(HARNESS_KEY, _set, flush="sync")
|
| 452 |
+
return {"agent": agent_id, "deleted": wanted,
|
| 453 |
+
"files": [_row(rec) for _p, rec in sorted(_files(session.runtime, agent_id).items())
|
| 454 |
+
if isinstance(rec, dict)],
|
| 455 |
+
"limits": _limits()}
|
api/routes_alerts.py
CHANGED
|
@@ -1,668 +1,668 @@
|
|
| 1 |
-
"""routes_alerts.py β the Alerts module (wave 20, owner item 25, contract C-ALERT).
|
| 2 |
-
|
| 3 |
-
GET /api/v1/alerts -> {alerts:[...]}
|
| 4 |
-
POST /api/v1/alerts <- {viewId, topic, label?}
|
| 5 |
-
DELETE /api/v1/alerts/{alert_id}
|
| 6 |
-
POST /api/v1/alerts/{alert_id}/run -> evaluate now (the pane's manual refresh)
|
| 7 |
-
GET /api/v1/notifications -> {unread, items:[...]}
|
| 8 |
-
POST /api/v1/notifications/read <- {ids:[...]|null, read?:bool}
|
| 9 |
-
|
| 10 |
-
The semantics β an alert is a view plus a remembered matched set, a notification is a NEW
|
| 11 |
-
ENTRANT, and the first evaluation seeds silently β live in `core.alerts` with the reasoning.
|
| 12 |
-
This file owns the two things a route must: WHO may do it, and HOW the view gets evaluated.
|
| 13 |
-
|
| 14 |
-
β **THE EVALUATION RUNS AS THE ALERT'S OWNER, NOT AS THE CALLER.** `_run_alert` builds the pool
|
| 15 |
-
for `rec['owner']`, never for whoever tripped the write hook. Any other choice leaks: a
|
| 16 |
-
full-access admin editing a cell would otherwise evaluate a BU-scoped user's alert over the whole
|
| 17 |
-
book, and the notification would name customers that user may not see β a permission leak wearing
|
| 18 |
-
a notification's clothes. The owner's own scope is the only correct basis for their alert.
|
| 19 |
-
|
| 20 |
-
β **AN ALERT IS NOT A SECOND READ PATH.** It resolves rows through the same
|
| 21 |
-
`routes_customers.grid_assembly` / `routes_tables.ut_assembly` the grid uses, so a row that an
|
| 22 |
-
alert can see is by construction a row its owner could open. Re-implementing the filter here
|
| 23 |
-
would be a second definition of "matches", and those two would drift.
|
| 24 |
-
"""
|
| 25 |
-
import re
|
| 26 |
-
|
| 27 |
-
from fastapi import APIRouter, Body, Depends
|
| 28 |
-
|
| 29 |
-
import core.alerts as alerts
|
| 30 |
-
from deps import Session, err, require_session
|
| 31 |
-
|
| 32 |
-
router = APIRouter(prefix="/api/v1")
|
| 33 |
-
|
| 34 |
-
#: The alert-bearing surfaces. `ut_` tables are admitted by prefix, like everywhere else.
|
| 35 |
-
_TOPICS = ("customer", "product")
|
| 36 |
-
|
| 37 |
-
# ββ ββ WAVE 32 Β· T20 Β· CONTRACT C3 β THE INBOX SHAPE, DERIVED ON READ ββββββββββββββββββββββββ
|
| 38 |
-
#
|
| 39 |
-
# `GET /notifications` gains `subject`, `kind` and `target` per item (`read` was always there).
|
| 40 |
-
#
|
| 41 |
-
# β DERIVED, NEVER STORED, AND THAT IS THE WHOLE OF WHY THIS WAVE EXISTS. Stamping the three
|
| 42 |
-
# keys onto the record at write time would give them to notifications minted AFTER the deploy and
|
| 43 |
-
# to nothing else β every notification already sitting in every tenant's inbox would open nothing,
|
| 44 |
-
# and the feature would be correct in the source and absent from the product
|
| 45 |
-
# ([[a-migration-that-runs-on-the-next-write]], D-201). A read-side derivation reaches a
|
| 46 |
-
# notification queued last month. It also keeps the store shape out of `core/alerts.py`, which is
|
| 47 |
-
# another lane's file this wave β but that is the convenience, not the reason.
|
| 48 |
-
#
|
| 49 |
-
# β TWO PRODUCERS WRITE TWO SHAPES into one inbox, and the vocabulary below is what tells them
|
| 50 |
-
# apart. `_queue` (a record ENTERED a watched view) sets `topic`+`viewId`. `notify()` sets
|
| 51 |
-
# `topic='automation'` and puts the producer's key in `alertId`, leaving `viewId` empty. Deciding
|
| 52 |
-
# here means the client branches on ONE field instead of re-deriving the same split.
|
| 53 |
-
#
|
| 54 |
-
# β **D-101 IS CLOSED HERE, BY SUBTRACTION.** There was a THIRD shape β `kind='automation_review'`
|
| 55 |
-
# + `autoId`, a card arriving at a review stage β and its producer `notify_review` was deleted by
|
| 56 |
-
# W27/R3 with the review lanes. `automation_engine.py`'s own tombstone (search `notify_review`)
|
| 57 |
-
# records the 2026-08-12 sweep: **no `.py` file anywhere produces one**, while the client branch,
|
| 58 |
-
# its route and three gate legs stayed fully alive. D-101's exit condition is *"the client review
|
| 59 |
-
# branch is deleted in the same change as any remaining residue, OR `notify_review` gains its real
|
| 60 |
-
# caller"* β the residue is zero, so the branch goes. It is not carried into the Inbox: a stored
|
| 61 |
-
# review notification (if any survives in a tenant from the wave-23 era) derives as an ordinary
|
| 62 |
-
# `alert` with no target, i.e. an honest unclickable row, which is correct β the board it pointed
|
| 63 |
-
# at was deleted two waves ago.
|
| 64 |
-
|
| 65 |
-
#: C3's `kind` vocabulary. Plain strings on the wire β the client must never union over them
|
| 66 |
-
#: (alertsModel's wave-9 law: a client union turns "the server grew a kind" into a dropped row).
|
| 67 |
-
NOTIF_KIND_ALERT = "alert"
|
| 68 |
-
NOTIF_KIND_AUTOMATION = "automation"
|
| 69 |
-
NOTIF_KIND_SHARE = "share"
|
| 70 |
-
|
| 71 |
-
#: C3's `target.module` vocabulary, and the automation sub-selection.
|
| 72 |
-
TARGET_MODULE_DATABASE = "database"
|
| 73 |
-
TARGET_MODULE_AUTOMATION = "automation"
|
| 74 |
-
TARGET_TAB_RUNS = "runs"
|
| 75 |
-
|
| 76 |
-
#: The topic `notify()` carries for a SHARE (W32-T28 writes it; nothing does yet, and a kind with
|
| 77 |
-
#: no producer is a string that reads as a feature β the reason this constant is named here and
|
| 78 |
-
#: cited from `routes_shares` rather than typed twice).
|
| 79 |
-
SHARE_TOPIC = "share"
|
| 80 |
-
|
| 81 |
-
#: `core.alerts.notify`'s default topic for a run outcome. Mirrors `inboxModel.AUTOMATION_TOPIC`.
|
| 82 |
-
AUTOMATION_TOPIC = "automation"
|
| 83 |
-
|
| 84 |
-
_UT_TOPIC = re.compile(r"ut_[A-Za-z0-9_]+\Z")
|
| 85 |
-
|
| 86 |
-
|
| 87 |
-
def route_for_topic(topic):
|
| 88 |
-
"""A grid SCOPE key -> the registry route that renders it, or None.
|
| 89 |
-
|
| 90 |
-
β THE SAME TABLE AS `alertsModel.routeForTopic`, and the parity is GATED
|
| 91 |
-
(`verify_alerts.py`'s vocabulary scan) rather than trusted. The two built-ins are the only
|
| 92 |
-
pair that differ β the registry names the surface (`customer_data`) while the grid names the
|
| 93 |
-
scope (`customer`) β so a topic passed through as a route sends every click to a page that
|
| 94 |
-
does not exist. `None` for anything else: a target this product cannot resolve must be ABSENT
|
| 95 |
-
rather than plausible, because an absent target renders as a row that does not pretend to be
|
| 96 |
-
clickable, and a wrong one renders as a click that silently goes nowhere.
|
| 97 |
-
"""
|
| 98 |
-
t = str(topic or "").strip()
|
| 99 |
-
if t == "customer":
|
| 100 |
-
return "customer_data"
|
| 101 |
-
if t == "product":
|
| 102 |
-
return "product_data"
|
| 103 |
-
if _UT_TOPIC.match(t):
|
| 104 |
-
return t
|
| 105 |
-
return None
|
| 106 |
-
|
| 107 |
-
|
| 108 |
-
def _refusal_code(exc):
|
| 109 |
-
"""The `error.code` an `HTTPException` raised by `deps.err()` carries, or `""`.
|
| 110 |
-
|
| 111 |
-
β W32-T22. Four refusals travel up the assembly chain β `unknown_table` (404), `forbidden`
|
| 112 |
-
(403), `window_required` (409) and `store_not_ready` (503) β and each already names its own
|
| 113 |
-
cause. Anything that reduces all four to one word is throwing away the only information the
|
| 114 |
-
reader could have acted on. Returns `""` for a plain exception, so a caller can tell
|
| 115 |
-
"refused, and here is why" apart from "broke, and we do not know why".
|
| 116 |
-
"""
|
| 117 |
-
detail = getattr(exc, "detail", None)
|
| 118 |
-
if isinstance(detail, dict):
|
| 119 |
-
inner = detail.get("error")
|
| 120 |
-
if isinstance(inner, dict):
|
| 121 |
-
return str(inner.get("code") or "")
|
| 122 |
-
return ""
|
| 123 |
-
|
| 124 |
-
|
| 125 |
-
def notification_view(item):
|
| 126 |
-
"""One STORED notification -> the shape the Inbox renders. PURE, and total.
|
| 127 |
-
|
| 128 |
-
Never raises and never drops a row: an item it cannot classify comes back as an `alert` with
|
| 129 |
-
no `target`, which the client renders as an unclickable row rather than hiding. An inbox that
|
| 130 |
-
silently omits what it does not understand is the one failure a reader cannot detect.
|
| 131 |
-
"""
|
| 132 |
-
if not isinstance(item, dict):
|
| 133 |
-
return item
|
| 134 |
-
topic = str(item.get("topic") or "").strip()
|
| 135 |
-
alert_id = str(item.get("alertId") or "").strip()
|
| 136 |
-
|
| 137 |
-
# β THE ID TEST IS HALF OF EVERY BRANCH, and it is the load-bearing half. A row whose topic
|
| 138 |
-
# says `automation` but whose producer key never arrived (a truncated payload, a server
|
| 139 |
-
# mid-deploy) would otherwise be handed a target naming NOTHING β a click that appears to work
|
| 140 |
-
# and silently does not, which is this repo's most-repeated failure shape. Failing the test
|
| 141 |
-
# drops it to the `alert` branch, where `route_for_topic` refuses out loud by answering None.
|
| 142 |
-
if topic == AUTOMATION_TOPIC and alert_id:
|
| 143 |
-
kind = NOTIF_KIND_AUTOMATION
|
| 144 |
-
target = {"module": TARGET_MODULE_AUTOMATION, "id": alert_id, "tab": TARGET_TAB_RUNS}
|
| 145 |
-
elif topic == SHARE_TOPIC and alert_id:
|
| 146 |
-
# β W32-T28: the sharer writes `key=<the ROUTE to open>` and, for a shared VIEW,
|
| 147 |
-
# `row_id=<the view to select>`.
|
| 148 |
-
#
|
| 149 |
-
# β `key` IS ALREADY A ROUTE, NOT A RAW OBJECT ID, and the first version of this got it
|
| 150 |
-
# wrong in a way worth recording: a shared VIEW put the VIEW's id in `alertId`, so the
|
| 151 |
-
# target read `{module: "database", id: "view_42"}` β an instruction to open a database
|
| 152 |
-
# called `view_42`. It looked right in the payload and would have opened nothing. The
|
| 153 |
-
# producer resolves the object to its topic and hands over the route; this branch only
|
| 154 |
-
# shapes what it is given.
|
| 155 |
-
kind = NOTIF_KIND_SHARE
|
| 156 |
-
row_id = str(item.get("rowId") or "").strip()
|
| 157 |
-
target = {"module": TARGET_MODULE_DATABASE, "id": alert_id,
|
| 158 |
-
**({"tab": row_id} if row_id else {})}
|
| 159 |
-
else:
|
| 160 |
-
kind = NOTIF_KIND_ALERT
|
| 161 |
-
route = route_for_topic(topic)
|
| 162 |
-
view_id = str(item.get("viewId") or "").strip()
|
| 163 |
-
target = None if route is None else (
|
| 164 |
-
{"module": TARGET_MODULE_DATABASE, "id": route,
|
| 165 |
-
**({"tab": view_id} if view_id else {})})
|
| 166 |
-
|
| 167 |
-
# The email split: `subject` is the HEADER (what this is about β the alert, the automation,
|
| 168 |
-
# the database), `label` stays the BODY (what happened β the record that entered, the run
|
| 169 |
-
# summary). They were one field, which is why a notification read as a sentence with no
|
| 170 |
-
# sender and the pane could not be laid out like mail.
|
| 171 |
-
subject = str(item.get("alertLabel") or "").strip() or str(item.get("label") or "").strip()
|
| 172 |
-
# β `kind` is OVERWRITTEN, not merged. There was one stored value (`automation_review`) and it
|
| 173 |
-
# is D-101's dead one; leaving it through would give the client two vocabularies for one
|
| 174 |
-
# question, which is the defect this wave's item 6 is about in a different file.
|
| 175 |
-
# ββ W33-T28 (owner: "the Inbox reads like email") β THE SENDER, WHICH DID NOT EXIST.
|
| 176 |
-
#
|
| 177 |
-
# β A `verifier` reading the finished wave-32 surface found that the row's sender POSITION was
|
| 178 |
-
# occupied by `kindLabel(n.kind)` β the literals "Alert" / "Automation" / "Shared with you" β
|
| 179 |
-
# i.e. a CATEGORY standing where a who belongs, and no sender field anywhere on the wire, in
|
| 180 |
-
# the model or in the markup. Mail has a from. This is it.
|
| 181 |
-
#
|
| 182 |
-
# β IT IS DERIVED HERE, NOT STORED, FOR EVERY KIND BUT ONE β and the exception is the point.
|
| 183 |
-
# An alert firing and an automation landing rows have no person behind them; their honest
|
| 184 |
-
# sender is the machine that did it, named as the thing the reader recognises. A SHARE has a
|
| 185 |
-
# real person, and only the producer knows who: `routes_shares.py` writes it as `actor` and
|
| 186 |
-
# this reads it back. β It is NOT parsed out of the body prose ("<name> shared this with
|
| 187 |
-
# you") β a sender recovered by regexing a sentence breaks the first time the sentence is
|
| 188 |
-
# reworded, and it would break silently, in the header.
|
| 189 |
-
#
|
| 190 |
-
# β FALLS BACK, NEVER BLANK. A share queued BEFORE `actor` existed has none, and a row with an
|
| 191 |
-
# empty from column reads as a broken inbox rather than as an old notification.
|
| 192 |
-
actor = str(item.get("actor") or "").strip()
|
| 193 |
-
if kind == NOTIF_KIND_SHARE:
|
| 194 |
-
sender = actor or "A teammate"
|
| 195 |
-
elif kind == NOTIF_KIND_AUTOMATION:
|
| 196 |
-
# β "Agents", not "Automation" (W34-T40, corrected at QA 2026-08-17). The client's
|
| 197 |
-
# `senderOf` already falls back to `AGENTS_MODULE_LABEL` β but `if (sent) return sent`
|
| 198 |
-
# runs FIRST, so this server literal won and every actor-less automation notification
|
| 199 |
-
# showed the retired module name in the inbox's From column.
|
| 200 |
-
sender = actor or "Agents"
|
| 201 |
-
else:
|
| 202 |
-
sender = actor or "Alerts"
|
| 203 |
-
out = {**item, "read": bool(item.get("read")), "kind": kind,
|
| 204 |
-
"subject": subject or "Notification", "sender": sender}
|
| 205 |
-
if target is not None:
|
| 206 |
-
out["target"] = target
|
| 207 |
-
return out
|
| 208 |
-
|
| 209 |
-
|
| 210 |
-
def inbox_view(box):
|
| 211 |
-
"""`core.alerts.inbox()`'s answer, with every item put through {@link notification_view}.
|
| 212 |
-
|
| 213 |
-
β `unread` IS NOT RECOUNTED. It is the ACCOUNT's number and `items` is one page of it; a
|
| 214 |
-
recount here would make the badge a function of whatever this page happened to include, which
|
| 215 |
-
is the exact defect `alertsModel.parseInbox`'s own header records from the other side.
|
| 216 |
-
"""
|
| 217 |
-
if not isinstance(box, dict):
|
| 218 |
-
return box
|
| 219 |
-
items = box.get("items")
|
| 220 |
-
if not isinstance(items, list):
|
| 221 |
-
return box
|
| 222 |
-
# ββ W33-T28 / D-208 β THE SERVER'S CLOCK RIDES WITH THE PAGE, and it is what lets the client
|
| 223 |
-
# render a mail-shaped stamp ("09:41" today, "Aug 12" beyond) instead of `2026-08-13 09:41`.
|
| 224 |
-
#
|
| 225 |
-
# β THE CLIENT MUST NOT READ ITS OWN CLOCK, which is D-208's exit condition word for word and
|
| 226 |
-
# is why this key exists rather than a `new Date()` in the browser. `at` is sent as UTC WITH
|
| 227 |
-
# its offset (D-18) precisely so every reader sees the same instant; deciding "is this today?"
|
| 228 |
-
# against a browser clock would re-introduce the drift the offset exists to remove β a reader a
|
| 229 |
-
# day ahead being told an event happened tomorrow [[date-window-vocabulary]]. Both operands
|
| 230 |
-
# now come from the same machine.
|
| 231 |
-
# β Same funnel as the enrichment, so the read door and the mark-read door cannot disagree β
|
| 232 |
-
# the note two lines up records what happened last time only one of them was enriched.
|
| 233 |
-
return {**box, "now": alerts._now_iso(),
|
| 234 |
-
"items": [notification_view(n) for n in items]}
|
| 235 |
-
|
| 236 |
-
|
| 237 |
-
def _view_by_id(g, view_id):
|
| 238 |
-
"""One saved view out of an assembly, by id. `None` when there is no such view.
|
| 239 |
-
|
| 240 |
-
ββ W33-T29 (owner: *"Alert me about new records"* answering "Something went wrong") β THIS
|
| 241 |
-
FUNCTION EXISTS BECAUSE TWO CALL SITES BOTH WROTE `(g.get("views") or {}).get(view_id)`, AND
|
| 242 |
-
`g["views"] IS A LIST`. `aios_grid.views_from_defs` returns `[{...}]`, `workspace_wire` passes
|
| 243 |
-
it straight out and both `ut_assembly` and `grid_assembly` return it unchanged β so `.get` on
|
| 244 |
-
it raises `AttributeError`, and `views_from_defs` always returns at least one element, so the
|
| 245 |
-
`or {}` never fires. **It raised on EVERY call, on every topic, since wave 20.**
|
| 246 |
-
|
| 247 |
-
β AND THE TWO SITES FAILED DIFFERENTLY, WHICH IS WHY ONLY ONE WAS EVER REPORTED. In
|
| 248 |
-
`_require_filtered_view` the raise lands ABOVE the handler's own `try`, so it leaves as a bare
|
| 249 |
-
FastAPI 500 and the client's `errorMessage` turns any 5xx into *"Something went wrong on our
|
| 250 |
-
side"* β the exact sentence the owner reported (D-107's shape, again: an attribute error above
|
| 251 |
-
the guard arrives as plain text rather than as our envelope). In `_evaluate` the identical
|
| 252 |
-
line is swallowed by `/notifications`' `except Exception: continue`, so **every stored
|
| 253 |
-
view-alert was silently dropped from the Inbox** and nobody had anything to report at all.
|
| 254 |
-
One expression, one loud symptom and one silent one.
|
| 255 |
-
|
| 256 |
-
β SO IT IS A FUNCTION, NOT TWO FIXED LINES. Two copies of "find the view" is what let one site
|
| 257 |
-
be discussed for three waves while its twin went unnoticed [[one-question-two-normalizers]].
|
| 258 |
-
|
| 259 |
-
β It accepts a dict too, and that is not defensive noise: `verify_alerts`' door fixture was
|
| 260 |
-
keyed `{id: view}` β which is precisely why the gate was green while production raised on
|
| 261 |
-
every call. The fixture is moving to the production shape in this same change, and tolerating
|
| 262 |
-
both here means a caller that legitimately holds one cannot resurrect the bug.
|
| 263 |
-
"""
|
| 264 |
-
want = str(view_id or "")
|
| 265 |
-
if not want:
|
| 266 |
-
return None
|
| 267 |
-
views = (g or {}).get("views")
|
| 268 |
-
if isinstance(views, dict):
|
| 269 |
-
found = views.get(want)
|
| 270 |
-
return found if isinstance(found, dict) else None
|
| 271 |
-
if not isinstance(views, list):
|
| 272 |
-
return None
|
| 273 |
-
for v in views:
|
| 274 |
-
if isinstance(v, dict) and str(v.get("id") or "") == want:
|
| 275 |
-
return v
|
| 276 |
-
return None
|
| 277 |
-
|
| 278 |
-
|
| 279 |
-
def _topic_or_400(raw):
|
| 280 |
-
topic = str(raw or "").strip().lower()
|
| 281 |
-
if topic.startswith("ut_") or topic in _TOPICS:
|
| 282 |
-
return topic
|
| 283 |
-
raise err(400, "bad_topic", f"topic must be one of {', '.join(_TOPICS)} or a ut_ table")
|
| 284 |
-
|
| 285 |
-
|
| 286 |
-
def _owner_session(session: Session, owner: str):
|
| 287 |
-
"""A `Session` for the alert's OWNER (see the module note on why the owner, not the caller).
|
| 288 |
-
|
| 289 |
-
β `Session` exposes `uname`/`admin` as PROPERTIES derived from `user`, not as fields β so an
|
| 290 |
-
owner session is built by swapping the `user` RECORD and letting both derive themselves. An
|
| 291 |
-
earlier version passed `uname=`/`admin=` to the constructor, which would have raised on the
|
| 292 |
-
first write hook of the wave; the properties are the single definition of who a session is,
|
| 293 |
-
and going around them is how a session with an admin flag and a non-admin record exists.
|
| 294 |
-
|
| 295 |
-
Returns None when the owner is gone or deactivated β their alerts then stop evaluating rather
|
| 296 |
-
than evaluating as somebody else, which is the fail-closed direction.
|
| 297 |
-
"""
|
| 298 |
-
import core.users as users
|
| 299 |
-
|
| 300 |
-
if str(owner) == str(session.uname):
|
| 301 |
-
return session
|
| 302 |
-
rec = (users.registry() or {}).get(str(owner))
|
| 303 |
-
if not isinstance(rec, dict) or not rec.get("active", True):
|
| 304 |
-
return None
|
| 305 |
-
# `_public` is THE definition of what a session may know about its own account (never a hash
|
| 306 |
-
# or a salt) β the same one `routes_auth` uses. Building the dict by hand here would be a
|
| 307 |
-
# second definition, and the one that leaks is always the copy.
|
| 308 |
-
return Session(tenant=session.tenant, user=users._public(str(owner), rec),
|
| 309 |
-
claims=session.claims, runtime=session.runtime)
|
| 310 |
-
|
| 311 |
-
|
| 312 |
-
def _evaluate(session: Session, rec: dict, assemblies=None):
|
| 313 |
-
"""Resolve `rec`'s view over its topic AS THE ALERT'S OWNER, then fold the result in.
|
| 314 |
-
|
| 315 |
-
ββ W31-T24 β `assemblies` IS A PER-REQUEST MEMO, KEYED `(topic, owner)`, and it is the whole
|
| 316 |
-
of this ticket's server half. `/notifications` re-evaluates EVERY alert inline on read and each
|
| 317 |
-
one built a FULL assembly β the pool, the workspace, `rows_from_pool` over every row. Two
|
| 318 |
-
alerts on one view built that table twice; ten built it ten times. Nothing dedupes them,
|
| 319 |
-
because each `_evaluate` was a closed call.
|
| 320 |
-
β `(topic, owner)` and not `topic`: the assembly is built as the alert's OWNER (see the module
|
| 321 |
-
note β evaluating a BU-scoped user's alert on a full-access admin's pool is a permission leak
|
| 322 |
-
wearing a notification's clothes), so two owners on one topic are two DIFFERENT tables and
|
| 323 |
-
must never share an entry. Getting that key wrong is the one way this optimisation could leak.
|
| 324 |
-
β Passing nothing keeps the old behaviour exactly, which is what the create/run doors want:
|
| 325 |
-
they evaluate ONE alert and a memo for a single call is pure overhead.
|
| 326 |
-
"""
|
| 327 |
-
import aios_grid
|
| 328 |
-
from harness import filter_eval
|
| 329 |
-
|
| 330 |
-
owner_sess = _owner_session(session, rec.get("owner"))
|
| 331 |
-
if owner_sess is None:
|
| 332 |
-
return {"skipped": "owner_unavailable"}
|
| 333 |
-
topic = str(rec.get("topic") or "")
|
| 334 |
-
memo_key = (topic, str(owner_sess.uname))
|
| 335 |
-
g = assemblies.get(memo_key) if isinstance(assemblies, dict) else None
|
| 336 |
-
if g is None:
|
| 337 |
-
try:
|
| 338 |
-
if topic.startswith("ut_"):
|
| 339 |
-
from routes_tables import ut_assembly
|
| 340 |
-
# β `consume_corrections=False`, and the default was a REAL BUG, not a tidy-up.
|
| 341 |
-
# `ut_assembly` defaults it True, so every `/notifications` read CONSUMED the
|
| 342 |
-
# one-shot field-name correction acks for every `ut_` topic that has an alert β
|
| 343 |
-
# taking them from the `/workspace` refresh that exists to show them to the person
|
| 344 |
-
# who made the edit. The customer branch below has always passed False; this one
|
| 345 |
-
# inherited a default nobody re-read. An inbox poll must never consume a one-shot.
|
| 346 |
-
g = ut_assembly(owner_sess, topic,
|
| 347 |
-
storage_key=f"{owner_sess.tenant}:{topic}:{owner_sess.uname}",
|
| 348 |
-
consume_corrections=False)
|
| 349 |
-
else:
|
| 350 |
-
from routes_customers import grid_assembly
|
| 351 |
-
g = grid_assembly(owner_sess, scope=topic, consume_corrections=False)
|
| 352 |
-
except Exception as e: # noqa: BLE001
|
| 353 |
-
# β W32-T22 β SKIPPING IS FINE HERE; SKIPPING ANONYMOUSLY IS NOT. This one must not
|
| 354 |
-
# raise (one bad alert cannot empty an inbox), so unlike `_require_filtered_view` it
|
| 355 |
-
# keeps a blanket catch β but it now reports the refusal's OWN code where there is
|
| 356 |
-
# one. `type(e).__name__` said `HTTPException` for four different causes, and
|
| 357 |
-
# `lastError` is the only place a user ever learns why an alert stopped firing.
|
| 358 |
-
#
|
| 359 |
-
# β `with_rows=True` STAYS on this path, deliberately: unlike the create door, an
|
| 360 |
-
# evaluation genuinely needs the rows to run the filter over. So an alert on a
|
| 361 |
-
# read-through grid is created (T22) and then skips at evaluation with
|
| 362 |
-
# `window_required` naming why β which is D-184's remaining half, and it is a
|
| 363 |
-
# SENTENCE now rather than silence.
|
| 364 |
-
return {"skipped": _refusal_code(e) or "unavailable", "detail": type(e).__name__}
|
| 365 |
-
if isinstance(assemblies, dict):
|
| 366 |
-
assemblies[memo_key] = g
|
| 367 |
-
|
| 368 |
-
view = _view_by_id(g, rec.get("viewId"))
|
| 369 |
-
if not isinstance(view, dict):
|
| 370 |
-
# Deleted, or un-shared out from under the alert. Say so on the RECORD rather than
|
| 371 |
-
# deleting the alert: an alert that silently vanishes is indistinguishable from one that
|
| 372 |
-
# never fires, and the user cannot debug what is not there.
|
| 373 |
-
return {"skipped": "view_missing"}
|
| 374 |
-
|
| 375 |
-
# The SAME row build the grid and `/customers` use β `rows_from_pool` is what puts derived
|
| 376 |
-
# and overlay values on a row. Evaluating a filter against raw pool dicts would silently
|
| 377 |
-
# never match any condition on a user-created or measure column.
|
| 378 |
-
#
|
| 379 |
-
# ββ AND "THE SAME ROW BUILD" WAS NOT TRUE, WHICH MADE EVERY ALERT ON A `ut_*` DATABASE BLIND
|
| 380 |
-
# TO IMPORTED DATA. Found by a verifier driving one real assembly through both paths.
|
| 381 |
-
#
|
| 382 |
-
# `routes_tables.table_rows` β the grid the person is looking at β merges the DEFINITION rows
|
| 383 |
-
# underneath the overlay ("base first, overlay wins"; that merge is itself the fix for owner
|
| 384 |
-
# item 3, *"it all got reseted"*). `_evaluate` is a second copy of that read and never got it:
|
| 385 |
-
# it handed `ws['overlays']` to `rows_from_pool` raw, so for a `ut_*` table every base cell
|
| 386 |
-
# evaluated as BLANK. Measured on one assembly, same view, same rows:
|
| 387 |
-
# rows_src state='unpaid' / 'paid'
|
| 388 |
-
# _evaluate saw state='' / '' β every base cell blank
|
| 389 |
-
# the GRID saw state='unpaid' / 'paid'
|
| 390 |
-
# so `state eq unpaid` matched NOTHING while the view showed one row, and `state isEmpty`
|
| 391 |
-
# matched EVERYTHING while the view showed none. **The alert did not merely miss rows β it
|
| 392 |
-
# inverted.** End to end: a row whose value arrived by import, automation, paste or the create
|
| 393 |
-
# door never fired; only a value typed as a hand EDIT did.
|
| 394 |
-
#
|
| 395 |
-
# β Scope, so nobody widens the fix past its cause: materialised `ut_*` tables are hit;
|
| 396 |
-
# `customer`/`product` are not (their fields are `source: "odoo"` and read off `rows_src`);
|
| 397 |
-
# `ut_odoo_*` never reaches here (`with_rows=True` refuses first and returns
|
| 398 |
-
# `skipped: window_required`).
|
| 399 |
-
# β ORDER IS LOAD-BEARING AND IS THE GRID'S: base underneath, overlay ON TOP. Inverting it
|
| 400 |
-
# would let a stale definition value shadow an edit the user has just made β the same defect
|
| 401 |
-
# `table_rows`' own note records, arriving from the other side.
|
| 402 |
-
_ov = (g.get("ws") or {}).get("overlays") or {}
|
| 403 |
-
_merged = {}
|
| 404 |
-
for _r in g["rows_src"]:
|
| 405 |
-
_pid = str(_r.get("pid"))
|
| 406 |
-
_cells = {k: v for k, v in _r.items() if k != "pid"}
|
| 407 |
-
_o = _ov.get(_pid)
|
| 408 |
-
if isinstance(_o, dict):
|
| 409 |
-
_cells.update(_o)
|
| 410 |
-
_merged[_pid] = _cells
|
| 411 |
-
rows = aios_grid.rows_from_pool(g["rows_src"], g["fields"], _merged,
|
| 412 |
-
derived=g.get("derived"))
|
| 413 |
-
config = view.get("config") or view
|
| 414 |
-
ctx = filter_eval.EvalCtx(
|
| 415 |
-
cohort_sets={str(k): {str(p) for p in (v.get("memberPids") or ())}
|
| 416 |
-
for k, v in (g.get("lists") or {}).items() if isinstance(v, dict)},
|
| 417 |
-
measure_sets=g.get("measure_sets") or {},
|
| 418 |
-
today=g.get("today"))
|
| 419 |
-
pids = filter_eval.visible_pids(config.get("filters") or [], rows, g["fields"], ctx,
|
| 420 |
-
member_pids=config.get("memberPids"))
|
| 421 |
-
labels = {str(r.get("pid")): str(r.get("name") or r.get("pid")) for r in rows}
|
| 422 |
-
return alerts.evaluate(rec.get("id"), [str(p) for p in pids],
|
| 423 |
-
labels=labels, partial=False, st=session.runtime)
|
| 424 |
-
|
| 425 |
-
|
| 426 |
-
@router.get("/alerts")
|
| 427 |
-
def list_alerts(session: Session = Depends(require_session)):
|
| 428 |
-
return {"alerts": alerts.list_alerts(user=session.uname, is_admin=session.admin,
|
| 429 |
-
st=session.runtime)}
|
| 430 |
-
|
| 431 |
-
|
| 432 |
-
@router.post("/alerts")
|
| 433 |
-
def create_alert(body: dict = Body(default=None), session: Session = Depends(require_session)):
|
| 434 |
-
body = body or {}
|
| 435 |
-
view_id = str(body.get("viewId") or "").strip()
|
| 436 |
-
if not view_id:
|
| 437 |
-
raise err(400, "bad_view", "an alert needs the id of the view it watches")
|
| 438 |
-
topic = _topic_or_400(body.get("topic"))
|
| 439 |
-
_require_filtered_view(session, topic, view_id)
|
| 440 |
-
import uuid
|
| 441 |
-
aid = f"al_{uuid.uuid4().hex[:12]}"
|
| 442 |
-
rec = alerts.create(aid, view_id=view_id, topic=topic, owner=session.uname,
|
| 443 |
-
label=body.get("label") or "", st=session.runtime)
|
| 444 |
-
# SEED IMMEDIATELY, so the alert starts from "everything currently matching is old news".
|
| 445 |
-
# Deferring this to the first write hook would mean the next edit announces the whole view.
|
| 446 |
-
outcome = _evaluate(session, rec)
|
| 447 |
-
return {"alert": {**rec, "seeded": True}, "first": outcome}
|
| 448 |
-
|
| 449 |
-
|
| 450 |
-
def _require_filtered_view(session: Session, topic: str, view_id: str):
|
| 451 |
-
"""400 unless `view_id` exists on `topic` AND actually narrows something.
|
| 452 |
-
|
| 453 |
-
β AN ALERT ON AN UNFILTERED VIEW IS SILENTLY INCAPABLE OF ALERTING, which is worse than one
|
| 454 |
-
that is refused. `filter_eval` treats an inactive tree as "no narrowing, every row shows"
|
| 455 |
-
(`visible_pids`'s own rule), so such an alert seeds with the entire table and can never see an
|
| 456 |
-
entrant again β there is nothing left to enter. The owner's words are *"when a Record gets
|
| 457 |
-
into that Filter's criteria"*: no criteria, no alert, and said at creation rather than
|
| 458 |
-
discovered by never being notified.
|
| 459 |
-
|
| 460 |
-
`is_rule_active` is the SAME activeness predicate the engine and the column tints use β a
|
| 461 |
-
half-typed rule is not a filter, and this must agree with what actually narrows or it would
|
| 462 |
-
accept a view whose one rule the engine then ignores.
|
| 463 |
-
|
| 464 |
-
ββ WAVE 32 Β· T22 (owner item 17) β THIS FUNCTION WAS THE ERROR. Two defects, stacked, and
|
| 465 |
-
the second one hid the first.
|
| 466 |
-
|
| 467 |
-
(1) **IT ASKED FOR EVERY ROW OF A TABLE IT NEVER LOOKS AT.** The only thing read below is
|
| 468 |
-
`g["views"]`. `ut_assembly` defaults `with_rows=True`, so creating an alert on a
|
| 469 |
-
read-through grid built the whole pool β and `scoped_pool` refuses that with
|
| 470 |
-
`409 window_required` over 963,783 rows, exactly as it is supposed to. `with_rows=False`
|
| 471 |
-
(W31-T20's flag, built for precisely this) answers the same question with `scoped_pids`,
|
| 472 |
-
runs the SAME `_defn_or_refuse` wall, and does not refuse. **That is D-184's create half,
|
| 473 |
-
closed** β an alert on a read-through grid can now be made at all.
|
| 474 |
-
(2) **A BLANKET `except Exception` TURNED EVERY NAMED REFUSAL INTO A 503.** `HTTPException`
|
| 475 |
-
is an `Exception`, so `404 unknown_table`, `403 forbidden`, `409 window_required` and
|
| 476 |
-
`503 store_not_ready` β four refusals that each say what is wrong β were all replaced by
|
| 477 |
-
*"the table is unavailable β try again in a moment"*. β AND THAT SENTENCE NEVER REACHED
|
| 478 |
-
A USER EITHER: `alertsApi.errorMessage` discards the text of any status β₯ 500 by design
|
| 479 |
-
(a 5xx body is the server's internals), substituting *"Something went wrong on our
|
| 480 |
-
side."* β which is the owner's screenshot, word for word. A knowable cause returned as a
|
| 481 |
-
5xx is invisible by construction, so re-wording the 503 could never have fixed this.
|
| 482 |
-
β The except is narrowed, not deleted: an UNEXPECTED failure is still a 503, because that is
|
| 483 |
-
honest. What it may no longer do is catch a refusal that already knows its own name.
|
| 484 |
-
"""
|
| 485 |
-
from fastapi import HTTPException
|
| 486 |
-
|
| 487 |
-
from harness import filter_eval
|
| 488 |
-
|
| 489 |
-
try:
|
| 490 |
-
if topic.startswith("ut_"):
|
| 491 |
-
from routes_tables import ut_assembly
|
| 492 |
-
# β `consume_corrections=False` β the customer branch has always passed it and this
|
| 493 |
-
# one inherited a default nobody re-read. Creating an alert must not eat the one-shot
|
| 494 |
-
# field-name correction acks belonging to the `/workspace` refresh that exists to show
|
| 495 |
-
# them to the person who made the edit. Same defect `_evaluate`'s header records.
|
| 496 |
-
g = ut_assembly(session, topic,
|
| 497 |
-
storage_key=f"{session.tenant}:{topic}:{session.uname}",
|
| 498 |
-
consume_corrections=False, with_rows=False)
|
| 499 |
-
else:
|
| 500 |
-
from routes_customers import grid_assembly
|
| 501 |
-
g = grid_assembly(session, scope=topic, consume_corrections=False)
|
| 502 |
-
except HTTPException:
|
| 503 |
-
raise # it already names its own cause
|
| 504 |
-
except Exception as e: # noqa: BLE001
|
| 505 |
-
# Genuinely unexpected. Still a 503, and now it carries the exception TYPE β without it,
|
| 506 |
-
# the one path that reaches this branch is also the one path with nothing to debug from.
|
| 507 |
-
raise err(503, "unavailable",
|
| 508 |
-
f"the table could not be read ({type(e).__name__}) β try again in a moment")
|
| 509 |
-
view = _view_by_id(g, view_id)
|
| 510 |
-
if not isinstance(view, dict):
|
| 511 |
-
raise err(404, "no_view", "that view does not exist on this table")
|
| 512 |
-
nodes, _conj = filter_eval.tree_parts((view.get("config") or view).get("filters") or [])
|
| 513 |
-
|
| 514 |
-
# ββ WAVE 33 Β· T29 β **CORRECTION: THE BLOCK BELOW IS TRUE ABOUT THE CODE AND FALSE ABOUT
|
| 515 |
-
# PRODUCTION, AND IT MUST BE READ SECOND.** It claims the missing-argument `TypeError` "IS
|
| 516 |
-
# owner item 17" β the owner's *"Something went wrong"*. It was not, and it could not have
|
| 517 |
-
# been: at `cbcf005`, the build the owner was using, the dict-read on `views` sat ~10 lines
|
| 518 |
-
# ABOVE this call and raised `AttributeError` on EVERY request, so the walk never reached the
|
| 519 |
-
# leaf and the arity bug was unreachable. `_view_by_id`'s own header records that fix.
|
| 520 |
-
#
|
| 521 |
-
# β WHY THE STALE PARAGRAPH STAYS RATHER THAN GETTING DELETED: the arity bug was real, the
|
| 522 |
-
# fix was right, and the three reasons it hid are the most transferable thing in this file.
|
| 523 |
-
# What was wrong is only its CLAIM TO BE THE CAUSE. Two comment blocks in one function each
|
| 524 |
-
# naming themselves as the origin of the same screenshot are mutually exclusive, and the next
|
| 525 |
-
# reader believes whichever they meet first β which is why this correction sits above rather
|
| 526 |
-
# than below. Caught by a verifier that read the SHIPPED file at the deployed commit instead
|
| 527 |
-
# of the working tree. [[grep-output-is-not-source]]
|
| 528 |
-
#
|
| 529 |
-
# ββ WAVE 32 Β· T22 β **THE CALL BELOW WAS MISSING AN ARGUMENT** (and wave 32 believed, wrongly,
|
| 530 |
-
# that this was owner item 17 β see the correction directly above).
|
| 531 |
-
#
|
| 532 |
-
# `is_rule_active(rule, columns)` takes TWO parameters (`harness/filter_sql.py`; every other
|
| 533 |
-
# caller in the repo passes both). This one passed ONE, so the moment the walk reached a LEAF
|
| 534 |
-
# rule it raised `TypeError: is_rule_active() missing 1 required positional argument`.
|
| 535 |
-
#
|
| 536 |
-
# β READ WHAT THAT MEANS BEFORE FIXING ANYTHING ELSE: the walk only reaches a leaf when the
|
| 537 |
-
# view HAS a condition β and a view with a condition is the only kind an alert is allowed on.
|
| 538 |
-
# A view with no filters yields an empty `nodes`, so `_any_active` returns False without ever
|
| 539 |
-
# calling this, and the reader gets the honest 400 `no_filter`. **So the only path that
|
| 540 |
-
# worked was the refusal path: "Alert me about new records" had never once created an alert
|
| 541 |
-
# on a filtered view.** β And the raise lands OUTSIDE the `try` above, so it was not even the
|
| 542 |
-
# 503 β it was a bare FastAPI 500, which `alertsApi.errorMessage` renders as *"Something went
|
| 543 |
-
# wrong on our side. Try again in a moment."*, the owner's screenshot word for word.
|
| 544 |
-
#
|
| 545 |
-
# β THREE THINGS HID IT, and they are worth more than the fix. (1) Python does not check
|
| 546 |
-
# arity until the line RUNS, and this line runs only on the success path of a feature whose
|
| 547 |
-
# every test exercised its refusals. (2) The `no_filter` 400 above it is a real, correct,
|
| 548 |
-
# well-tested refusal, so the door looked alive. (3) `verify_alerts.py` asserts the refusal
|
| 549 |
-
# (`no_filter` reaches the user) and the transport β never a creation. A gate can be green,
|
| 550 |
-
# thorough and honest about everything except the one path the feature exists for.
|
| 551 |
-
#
|
| 552 |
-
# `_columns_map` is the DEFINITION of fields -> the membership set `is_rule_active` looks a
|
| 553 |
-
# column up in; building a second dict here would be a second answer to one question, which
|
| 554 |
-
# is this wave's other headline defect in a different file. Its leading underscore is a real
|
| 555 |
-
# smell and is BOOKED (PENDING, mailbox/C.md) rather than worked around.
|
| 556 |
-
columns = filter_eval._columns_map(g.get("fields") or [])
|
| 557 |
-
|
| 558 |
-
def _any_active(ns):
|
| 559 |
-
for n in ns or ():
|
| 560 |
-
if isinstance(n, dict) and isinstance(n.get("children"), list):
|
| 561 |
-
if _any_active(n["children"]):
|
| 562 |
-
return True
|
| 563 |
-
elif filter_eval.is_rule_active(n, columns):
|
| 564 |
-
return True
|
| 565 |
-
return False
|
| 566 |
-
|
| 567 |
-
if not _any_active(nodes):
|
| 568 |
-
raise err(400, "no_filter",
|
| 569 |
-
"this view has no active filter, so no record can ever ENTER it β add a "
|
| 570 |
-
"condition to the view first, then create the alert")
|
| 571 |
-
|
| 572 |
-
|
| 573 |
-
@router.delete("/alerts/{alert_id}")
|
| 574 |
-
def delete_alert(alert_id: str, session: Session = Depends(require_session)):
|
| 575 |
-
rec = next((r for r in alerts.list_alerts(st=session.runtime)
|
| 576 |
-
if str(r.get("id")) == str(alert_id)), None)
|
| 577 |
-
if rec is None:
|
| 578 |
-
raise err(404, "no_alert", "that alert does not exist")
|
| 579 |
-
if str(rec.get("owner")) != str(session.uname) and not session.admin:
|
| 580 |
-
raise err(403, "not_yours", "only the alert's owner (or an administrator) can delete it")
|
| 581 |
-
alerts.delete(alert_id, st=session.runtime)
|
| 582 |
-
return {"ok": True}
|
| 583 |
-
|
| 584 |
-
|
| 585 |
-
@router.post("/alerts/{alert_id}/run")
|
| 586 |
-
def run_alert(alert_id: str, session: Session = Depends(require_session)):
|
| 587 |
-
rec = next((r for r in alerts.list_alerts(user=session.uname, is_admin=session.admin,
|
| 588 |
-
st=session.runtime)
|
| 589 |
-
if str(r.get("id")) == str(alert_id)), None)
|
| 590 |
-
if rec is None:
|
| 591 |
-
raise err(404, "no_alert", "that alert does not exist")
|
| 592 |
-
return _evaluate(session, rec)
|
| 593 |
-
|
| 594 |
-
|
| 595 |
-
@router.get("/notifications")
|
| 596 |
-
def notifications(session: Session = Depends(require_session)):
|
| 597 |
-
"""The inbox β RE-EVALUATED on read, which is a deliberate design choice.
|
| 598 |
-
|
| 599 |
-
β A-S1-2 RESOLVED THE OTHER WAY, and the reason is structural rather than a shortcut. The
|
| 600 |
-
plan was a push hook: the automation engine calls `after_write` when it lands rows. But
|
| 601 |
-
`run_async` runs on a BACKGROUND THREAD with no `Session` in scope, and an alert must be
|
| 602 |
-
evaluated as its OWNER (see `_evaluate`) β so a push hook would have to mint a session inside
|
| 603 |
-
a worker thread from a tenant runtime, which is exactly the kind of ad-hoc identity
|
| 604 |
-
construction that leaks scope.
|
| 605 |
-
|
| 606 |
-
Pulling on read has none of that: the caller IS a session, the assemblies are already
|
| 607 |
-
scope-cached, and the user cannot observe the difference β an inbox is only ever read by
|
| 608 |
-
someone opening it. The cost is that a notification is minted when you LOOK rather than when
|
| 609 |
-
the row landed, so the `at` stamp is detection time, not arrival time.
|
| 610 |
-
|
| 611 |
-
`after_write` stays exported for the day the engine can hand over a real identity.
|
| 612 |
-
|
| 613 |
-
ββ W31-T24 β ONE ASSEMBLY PER (TOPIC, OWNER), NOT ONE PER ALERT.
|
| 614 |
-
β MEASURED FIRST, AND THE MEASUREMENT CORRECTS AN EARLIER READING OF IT. This route is
|
| 615 |
-
**20 ms in-process and 3,280 ms live** on tenant #0 β but tenant #0 has **ZERO alerts**
|
| 616 |
-
(censused 2026-08-12), so the 20 ms is an EMPTY LOOP and says nothing at all about what the
|
| 617 |
-
re-evaluation costs. The live 3,280 ms is the two store reads either side of that loop. So the
|
| 618 |
-
body below is not slow today; it is UNEXERCISED, and every alert a tenant creates adds a whole
|
| 619 |
-
grid assembly to an inbox poll. The memo turns O(alerts) into O(distinct topic Γ owner), which
|
| 620 |
-
is the difference between "fine" and "three seconds per alert" the day somebody uses the
|
| 621 |
-
feature. β Making the read cheap by evaluating LESS is the obvious wrong fix and is not what
|
| 622 |
-
this does: every alert is still evaluated, against the same rows, in the same order.
|
| 623 |
-
"""
|
| 624 |
-
assemblies = {}
|
| 625 |
-
for rec in alerts.list_alerts(user=session.uname, is_admin=False, st=session.runtime):
|
| 626 |
-
try:
|
| 627 |
-
_evaluate(session, rec, assemblies=assemblies)
|
| 628 |
-
except Exception: # noqa: BLE001
|
| 629 |
-
continue # one bad alert must not empty the pane
|
| 630 |
-
# β W32-T20 (C3): every item leaves through `inbox_view`, so a notification queued before
|
| 631 |
-
# this wave carries a `target` too. See `notification_view`'s header for why it is derived.
|
| 632 |
-
return inbox_view(alerts.inbox(session.uname, st=session.runtime))
|
| 633 |
-
|
| 634 |
-
|
| 635 |
-
@router.post("/notifications/read")
|
| 636 |
-
def read_notifications(body: dict = Body(default=None),
|
| 637 |
-
session: Session = Depends(require_session)):
|
| 638 |
-
body = body or {}
|
| 639 |
-
ids = body.get("ids")
|
| 640 |
-
if ids is not None and not isinstance(ids, list):
|
| 641 |
-
raise err(400, "bad_ids", "ids must be a list, or null to mark every notification")
|
| 642 |
-
# β THE SAME ENRICHMENT ON BOTH DOORS. `mark_read` returns a fresh inbox, and the Inbox
|
| 643 |
-
# module re-renders from it β an un-enriched answer here would strip `target` off every row
|
| 644 |
-
# the moment somebody marked one read, i.e. the feature would work until first use.
|
| 645 |
-
return inbox_view(alerts.mark_read(session.uname, ids, read=bool(body.get("read", True)),
|
| 646 |
-
st=session.runtime))
|
| 647 |
-
|
| 648 |
-
|
| 649 |
-
def after_write(session: Session, topic_key: str):
|
| 650 |
-
"""THE WRITE HOOK β call after a write that could change what a view matches.
|
| 651 |
-
|
| 652 |
-
Exported as a plain function (not a route) so `core.grid_events`' callers and S2's automation
|
| 653 |
-
upserts reach it the same way. It never raises: an alert evaluation failing must not fail the
|
| 654 |
-
edit that triggered it.
|
| 655 |
-
|
| 656 |
-
β W31-T24 β it shares `/notifications`' memo shape for the same reason: a write that changes
|
| 657 |
-
one view can trip several alerts on the SAME topic, and each would otherwise rebuild the table.
|
| 658 |
-
β STILL ZERO PRODUCTION CALLERS (W31-T24 confirmed it; the route docstring above says why the
|
| 659 |
-
push hook was resolved the other way). Booked rather than wired: minting a session inside the
|
| 660 |
-
engine's worker thread is the ad-hoc identity construction this file exists to avoid.
|
| 661 |
-
"""
|
| 662 |
-
try:
|
| 663 |
-
assemblies = {}
|
| 664 |
-
return alerts.after_write(topic_key, st=session.runtime,
|
| 665 |
-
runner=lambda rec: _evaluate(session, rec,
|
| 666 |
-
assemblies=assemblies))
|
| 667 |
-
except Exception: # noqa: BLE001
|
| 668 |
-
return {"evaluated": 0}
|
|
|
|
| 1 |
+
"""routes_alerts.py β the Alerts module (wave 20, owner item 25, contract C-ALERT).
|
| 2 |
+
|
| 3 |
+
GET /api/v1/alerts -> {alerts:[...]}
|
| 4 |
+
POST /api/v1/alerts <- {viewId, topic, label?}
|
| 5 |
+
DELETE /api/v1/alerts/{alert_id}
|
| 6 |
+
POST /api/v1/alerts/{alert_id}/run -> evaluate now (the pane's manual refresh)
|
| 7 |
+
GET /api/v1/notifications -> {unread, items:[...]}
|
| 8 |
+
POST /api/v1/notifications/read <- {ids:[...]|null, read?:bool}
|
| 9 |
+
|
| 10 |
+
The semantics β an alert is a view plus a remembered matched set, a notification is a NEW
|
| 11 |
+
ENTRANT, and the first evaluation seeds silently β live in `core.alerts` with the reasoning.
|
| 12 |
+
This file owns the two things a route must: WHO may do it, and HOW the view gets evaluated.
|
| 13 |
+
|
| 14 |
+
β **THE EVALUATION RUNS AS THE ALERT'S OWNER, NOT AS THE CALLER.** `_run_alert` builds the pool
|
| 15 |
+
for `rec['owner']`, never for whoever tripped the write hook. Any other choice leaks: a
|
| 16 |
+
full-access admin editing a cell would otherwise evaluate a BU-scoped user's alert over the whole
|
| 17 |
+
book, and the notification would name customers that user may not see β a permission leak wearing
|
| 18 |
+
a notification's clothes. The owner's own scope is the only correct basis for their alert.
|
| 19 |
+
|
| 20 |
+
β **AN ALERT IS NOT A SECOND READ PATH.** It resolves rows through the same
|
| 21 |
+
`routes_customers.grid_assembly` / `routes_tables.ut_assembly` the grid uses, so a row that an
|
| 22 |
+
alert can see is by construction a row its owner could open. Re-implementing the filter here
|
| 23 |
+
would be a second definition of "matches", and those two would drift.
|
| 24 |
+
"""
|
| 25 |
+
import re
|
| 26 |
+
|
| 27 |
+
from fastapi import APIRouter, Body, Depends
|
| 28 |
+
|
| 29 |
+
import core.alerts as alerts
|
| 30 |
+
from deps import Session, err, require_session
|
| 31 |
+
|
| 32 |
+
router = APIRouter(prefix="/api/v1")
|
| 33 |
+
|
| 34 |
+
#: The alert-bearing surfaces. `ut_` tables are admitted by prefix, like everywhere else.
|
| 35 |
+
_TOPICS = ("customer", "product")
|
| 36 |
+
|
| 37 |
+
# ββ ββ WAVE 32 Β· T20 Β· CONTRACT C3 β THE INBOX SHAPE, DERIVED ON READ ββββββββββββββββββββββββ
|
| 38 |
+
#
|
| 39 |
+
# `GET /notifications` gains `subject`, `kind` and `target` per item (`read` was always there).
|
| 40 |
+
#
|
| 41 |
+
# β DERIVED, NEVER STORED, AND THAT IS THE WHOLE OF WHY THIS WAVE EXISTS. Stamping the three
|
| 42 |
+
# keys onto the record at write time would give them to notifications minted AFTER the deploy and
|
| 43 |
+
# to nothing else β every notification already sitting in every tenant's inbox would open nothing,
|
| 44 |
+
# and the feature would be correct in the source and absent from the product
|
| 45 |
+
# ([[a-migration-that-runs-on-the-next-write]], D-201). A read-side derivation reaches a
|
| 46 |
+
# notification queued last month. It also keeps the store shape out of `core/alerts.py`, which is
|
| 47 |
+
# another lane's file this wave β but that is the convenience, not the reason.
|
| 48 |
+
#
|
| 49 |
+
# β TWO PRODUCERS WRITE TWO SHAPES into one inbox, and the vocabulary below is what tells them
|
| 50 |
+
# apart. `_queue` (a record ENTERED a watched view) sets `topic`+`viewId`. `notify()` sets
|
| 51 |
+
# `topic='automation'` and puts the producer's key in `alertId`, leaving `viewId` empty. Deciding
|
| 52 |
+
# here means the client branches on ONE field instead of re-deriving the same split.
|
| 53 |
+
#
|
| 54 |
+
# β **D-101 IS CLOSED HERE, BY SUBTRACTION.** There was a THIRD shape β `kind='automation_review'`
|
| 55 |
+
# + `autoId`, a card arriving at a review stage β and its producer `notify_review` was deleted by
|
| 56 |
+
# W27/R3 with the review lanes. `automation_engine.py`'s own tombstone (search `notify_review`)
|
| 57 |
+
# records the 2026-08-12 sweep: **no `.py` file anywhere produces one**, while the client branch,
|
| 58 |
+
# its route and three gate legs stayed fully alive. D-101's exit condition is *"the client review
|
| 59 |
+
# branch is deleted in the same change as any remaining residue, OR `notify_review` gains its real
|
| 60 |
+
# caller"* β the residue is zero, so the branch goes. It is not carried into the Inbox: a stored
|
| 61 |
+
# review notification (if any survives in a tenant from the wave-23 era) derives as an ordinary
|
| 62 |
+
# `alert` with no target, i.e. an honest unclickable row, which is correct β the board it pointed
|
| 63 |
+
# at was deleted two waves ago.
|
| 64 |
+
|
| 65 |
+
#: C3's `kind` vocabulary. Plain strings on the wire β the client must never union over them
|
| 66 |
+
#: (alertsModel's wave-9 law: a client union turns "the server grew a kind" into a dropped row).
|
| 67 |
+
NOTIF_KIND_ALERT = "alert"
|
| 68 |
+
NOTIF_KIND_AUTOMATION = "automation"
|
| 69 |
+
NOTIF_KIND_SHARE = "share"
|
| 70 |
+
|
| 71 |
+
#: C3's `target.module` vocabulary, and the automation sub-selection.
|
| 72 |
+
TARGET_MODULE_DATABASE = "database"
|
| 73 |
+
TARGET_MODULE_AUTOMATION = "automation"
|
| 74 |
+
TARGET_TAB_RUNS = "runs"
|
| 75 |
+
|
| 76 |
+
#: The topic `notify()` carries for a SHARE (W32-T28 writes it; nothing does yet, and a kind with
|
| 77 |
+
#: no producer is a string that reads as a feature β the reason this constant is named here and
|
| 78 |
+
#: cited from `routes_shares` rather than typed twice).
|
| 79 |
+
SHARE_TOPIC = "share"
|
| 80 |
+
|
| 81 |
+
#: `core.alerts.notify`'s default topic for a run outcome. Mirrors `inboxModel.AUTOMATION_TOPIC`.
|
| 82 |
+
AUTOMATION_TOPIC = "automation"
|
| 83 |
+
|
| 84 |
+
_UT_TOPIC = re.compile(r"ut_[A-Za-z0-9_]+\Z")
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
def route_for_topic(topic):
|
| 88 |
+
"""A grid SCOPE key -> the registry route that renders it, or None.
|
| 89 |
+
|
| 90 |
+
β THE SAME TABLE AS `alertsModel.routeForTopic`, and the parity is GATED
|
| 91 |
+
(`verify_alerts.py`'s vocabulary scan) rather than trusted. The two built-ins are the only
|
| 92 |
+
pair that differ β the registry names the surface (`customer_data`) while the grid names the
|
| 93 |
+
scope (`customer`) β so a topic passed through as a route sends every click to a page that
|
| 94 |
+
does not exist. `None` for anything else: a target this product cannot resolve must be ABSENT
|
| 95 |
+
rather than plausible, because an absent target renders as a row that does not pretend to be
|
| 96 |
+
clickable, and a wrong one renders as a click that silently goes nowhere.
|
| 97 |
+
"""
|
| 98 |
+
t = str(topic or "").strip()
|
| 99 |
+
if t == "customer":
|
| 100 |
+
return "customer_data"
|
| 101 |
+
if t == "product":
|
| 102 |
+
return "product_data"
|
| 103 |
+
if _UT_TOPIC.match(t):
|
| 104 |
+
return t
|
| 105 |
+
return None
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
def _refusal_code(exc):
|
| 109 |
+
"""The `error.code` an `HTTPException` raised by `deps.err()` carries, or `""`.
|
| 110 |
+
|
| 111 |
+
β W32-T22. Four refusals travel up the assembly chain β `unknown_table` (404), `forbidden`
|
| 112 |
+
(403), `window_required` (409) and `store_not_ready` (503) β and each already names its own
|
| 113 |
+
cause. Anything that reduces all four to one word is throwing away the only information the
|
| 114 |
+
reader could have acted on. Returns `""` for a plain exception, so a caller can tell
|
| 115 |
+
"refused, and here is why" apart from "broke, and we do not know why".
|
| 116 |
+
"""
|
| 117 |
+
detail = getattr(exc, "detail", None)
|
| 118 |
+
if isinstance(detail, dict):
|
| 119 |
+
inner = detail.get("error")
|
| 120 |
+
if isinstance(inner, dict):
|
| 121 |
+
return str(inner.get("code") or "")
|
| 122 |
+
return ""
|
| 123 |
+
|
| 124 |
+
|
| 125 |
+
def notification_view(item):
|
| 126 |
+
"""One STORED notification -> the shape the Inbox renders. PURE, and total.
|
| 127 |
+
|
| 128 |
+
Never raises and never drops a row: an item it cannot classify comes back as an `alert` with
|
| 129 |
+
no `target`, which the client renders as an unclickable row rather than hiding. An inbox that
|
| 130 |
+
silently omits what it does not understand is the one failure a reader cannot detect.
|
| 131 |
+
"""
|
| 132 |
+
if not isinstance(item, dict):
|
| 133 |
+
return item
|
| 134 |
+
topic = str(item.get("topic") or "").strip()
|
| 135 |
+
alert_id = str(item.get("alertId") or "").strip()
|
| 136 |
+
|
| 137 |
+
# β THE ID TEST IS HALF OF EVERY BRANCH, and it is the load-bearing half. A row whose topic
|
| 138 |
+
# says `automation` but whose producer key never arrived (a truncated payload, a server
|
| 139 |
+
# mid-deploy) would otherwise be handed a target naming NOTHING β a click that appears to work
|
| 140 |
+
# and silently does not, which is this repo's most-repeated failure shape. Failing the test
|
| 141 |
+
# drops it to the `alert` branch, where `route_for_topic` refuses out loud by answering None.
|
| 142 |
+
if topic == AUTOMATION_TOPIC and alert_id:
|
| 143 |
+
kind = NOTIF_KIND_AUTOMATION
|
| 144 |
+
target = {"module": TARGET_MODULE_AUTOMATION, "id": alert_id, "tab": TARGET_TAB_RUNS}
|
| 145 |
+
elif topic == SHARE_TOPIC and alert_id:
|
| 146 |
+
# β W32-T28: the sharer writes `key=<the ROUTE to open>` and, for a shared VIEW,
|
| 147 |
+
# `row_id=<the view to select>`.
|
| 148 |
+
#
|
| 149 |
+
# β `key` IS ALREADY A ROUTE, NOT A RAW OBJECT ID, and the first version of this got it
|
| 150 |
+
# wrong in a way worth recording: a shared VIEW put the VIEW's id in `alertId`, so the
|
| 151 |
+
# target read `{module: "database", id: "view_42"}` β an instruction to open a database
|
| 152 |
+
# called `view_42`. It looked right in the payload and would have opened nothing. The
|
| 153 |
+
# producer resolves the object to its topic and hands over the route; this branch only
|
| 154 |
+
# shapes what it is given.
|
| 155 |
+
kind = NOTIF_KIND_SHARE
|
| 156 |
+
row_id = str(item.get("rowId") or "").strip()
|
| 157 |
+
target = {"module": TARGET_MODULE_DATABASE, "id": alert_id,
|
| 158 |
+
**({"tab": row_id} if row_id else {})}
|
| 159 |
+
else:
|
| 160 |
+
kind = NOTIF_KIND_ALERT
|
| 161 |
+
route = route_for_topic(topic)
|
| 162 |
+
view_id = str(item.get("viewId") or "").strip()
|
| 163 |
+
target = None if route is None else (
|
| 164 |
+
{"module": TARGET_MODULE_DATABASE, "id": route,
|
| 165 |
+
**({"tab": view_id} if view_id else {})})
|
| 166 |
+
|
| 167 |
+
# The email split: `subject` is the HEADER (what this is about β the alert, the automation,
|
| 168 |
+
# the database), `label` stays the BODY (what happened β the record that entered, the run
|
| 169 |
+
# summary). They were one field, which is why a notification read as a sentence with no
|
| 170 |
+
# sender and the pane could not be laid out like mail.
|
| 171 |
+
subject = str(item.get("alertLabel") or "").strip() or str(item.get("label") or "").strip()
|
| 172 |
+
# β `kind` is OVERWRITTEN, not merged. There was one stored value (`automation_review`) and it
|
| 173 |
+
# is D-101's dead one; leaving it through would give the client two vocabularies for one
|
| 174 |
+
# question, which is the defect this wave's item 6 is about in a different file.
|
| 175 |
+
# ββ W33-T28 (owner: "the Inbox reads like email") β THE SENDER, WHICH DID NOT EXIST.
|
| 176 |
+
#
|
| 177 |
+
# β A `verifier` reading the finished wave-32 surface found that the row's sender POSITION was
|
| 178 |
+
# occupied by `kindLabel(n.kind)` β the literals "Alert" / "Automation" / "Shared with you" β
|
| 179 |
+
# i.e. a CATEGORY standing where a who belongs, and no sender field anywhere on the wire, in
|
| 180 |
+
# the model or in the markup. Mail has a from. This is it.
|
| 181 |
+
#
|
| 182 |
+
# β IT IS DERIVED HERE, NOT STORED, FOR EVERY KIND BUT ONE β and the exception is the point.
|
| 183 |
+
# An alert firing and an automation landing rows have no person behind them; their honest
|
| 184 |
+
# sender is the machine that did it, named as the thing the reader recognises. A SHARE has a
|
| 185 |
+
# real person, and only the producer knows who: `routes_shares.py` writes it as `actor` and
|
| 186 |
+
# this reads it back. β It is NOT parsed out of the body prose ("<name> shared this with
|
| 187 |
+
# you") β a sender recovered by regexing a sentence breaks the first time the sentence is
|
| 188 |
+
# reworded, and it would break silently, in the header.
|
| 189 |
+
#
|
| 190 |
+
# β FALLS BACK, NEVER BLANK. A share queued BEFORE `actor` existed has none, and a row with an
|
| 191 |
+
# empty from column reads as a broken inbox rather than as an old notification.
|
| 192 |
+
actor = str(item.get("actor") or "").strip()
|
| 193 |
+
if kind == NOTIF_KIND_SHARE:
|
| 194 |
+
sender = actor or "A teammate"
|
| 195 |
+
elif kind == NOTIF_KIND_AUTOMATION:
|
| 196 |
+
# β "Agents", not "Automation" (W34-T40, corrected at QA 2026-08-17). The client's
|
| 197 |
+
# `senderOf` already falls back to `AGENTS_MODULE_LABEL` β but `if (sent) return sent`
|
| 198 |
+
# runs FIRST, so this server literal won and every actor-less automation notification
|
| 199 |
+
# showed the retired module name in the inbox's From column.
|
| 200 |
+
sender = actor or "Agents"
|
| 201 |
+
else:
|
| 202 |
+
sender = actor or "Alerts"
|
| 203 |
+
out = {**item, "read": bool(item.get("read")), "kind": kind,
|
| 204 |
+
"subject": subject or "Notification", "sender": sender}
|
| 205 |
+
if target is not None:
|
| 206 |
+
out["target"] = target
|
| 207 |
+
return out
|
| 208 |
+
|
| 209 |
+
|
| 210 |
+
def inbox_view(box):
|
| 211 |
+
"""`core.alerts.inbox()`'s answer, with every item put through {@link notification_view}.
|
| 212 |
+
|
| 213 |
+
β `unread` IS NOT RECOUNTED. It is the ACCOUNT's number and `items` is one page of it; a
|
| 214 |
+
recount here would make the badge a function of whatever this page happened to include, which
|
| 215 |
+
is the exact defect `alertsModel.parseInbox`'s own header records from the other side.
|
| 216 |
+
"""
|
| 217 |
+
if not isinstance(box, dict):
|
| 218 |
+
return box
|
| 219 |
+
items = box.get("items")
|
| 220 |
+
if not isinstance(items, list):
|
| 221 |
+
return box
|
| 222 |
+
# ββ W33-T28 / D-208 β THE SERVER'S CLOCK RIDES WITH THE PAGE, and it is what lets the client
|
| 223 |
+
# render a mail-shaped stamp ("09:41" today, "Aug 12" beyond) instead of `2026-08-13 09:41`.
|
| 224 |
+
#
|
| 225 |
+
# β THE CLIENT MUST NOT READ ITS OWN CLOCK, which is D-208's exit condition word for word and
|
| 226 |
+
# is why this key exists rather than a `new Date()` in the browser. `at` is sent as UTC WITH
|
| 227 |
+
# its offset (D-18) precisely so every reader sees the same instant; deciding "is this today?"
|
| 228 |
+
# against a browser clock would re-introduce the drift the offset exists to remove β a reader a
|
| 229 |
+
# day ahead being told an event happened tomorrow [[date-window-vocabulary]]. Both operands
|
| 230 |
+
# now come from the same machine.
|
| 231 |
+
# β Same funnel as the enrichment, so the read door and the mark-read door cannot disagree β
|
| 232 |
+
# the note two lines up records what happened last time only one of them was enriched.
|
| 233 |
+
return {**box, "now": alerts._now_iso(),
|
| 234 |
+
"items": [notification_view(n) for n in items]}
|
| 235 |
+
|
| 236 |
+
|
| 237 |
+
def _view_by_id(g, view_id):
|
| 238 |
+
"""One saved view out of an assembly, by id. `None` when there is no such view.
|
| 239 |
+
|
| 240 |
+
ββ W33-T29 (owner: *"Alert me about new records"* answering "Something went wrong") β THIS
|
| 241 |
+
FUNCTION EXISTS BECAUSE TWO CALL SITES BOTH WROTE `(g.get("views") or {}).get(view_id)`, AND
|
| 242 |
+
`g["views"] IS A LIST`. `aios_grid.views_from_defs` returns `[{...}]`, `workspace_wire` passes
|
| 243 |
+
it straight out and both `ut_assembly` and `grid_assembly` return it unchanged β so `.get` on
|
| 244 |
+
it raises `AttributeError`, and `views_from_defs` always returns at least one element, so the
|
| 245 |
+
`or {}` never fires. **It raised on EVERY call, on every topic, since wave 20.**
|
| 246 |
+
|
| 247 |
+
β AND THE TWO SITES FAILED DIFFERENTLY, WHICH IS WHY ONLY ONE WAS EVER REPORTED. In
|
| 248 |
+
`_require_filtered_view` the raise lands ABOVE the handler's own `try`, so it leaves as a bare
|
| 249 |
+
FastAPI 500 and the client's `errorMessage` turns any 5xx into *"Something went wrong on our
|
| 250 |
+
side"* β the exact sentence the owner reported (D-107's shape, again: an attribute error above
|
| 251 |
+
the guard arrives as plain text rather than as our envelope). In `_evaluate` the identical
|
| 252 |
+
line is swallowed by `/notifications`' `except Exception: continue`, so **every stored
|
| 253 |
+
view-alert was silently dropped from the Inbox** and nobody had anything to report at all.
|
| 254 |
+
One expression, one loud symptom and one silent one.
|
| 255 |
+
|
| 256 |
+
β SO IT IS A FUNCTION, NOT TWO FIXED LINES. Two copies of "find the view" is what let one site
|
| 257 |
+
be discussed for three waves while its twin went unnoticed [[one-question-two-normalizers]].
|
| 258 |
+
|
| 259 |
+
β It accepts a dict too, and that is not defensive noise: `verify_alerts`' door fixture was
|
| 260 |
+
keyed `{id: view}` β which is precisely why the gate was green while production raised on
|
| 261 |
+
every call. The fixture is moving to the production shape in this same change, and tolerating
|
| 262 |
+
both here means a caller that legitimately holds one cannot resurrect the bug.
|
| 263 |
+
"""
|
| 264 |
+
want = str(view_id or "")
|
| 265 |
+
if not want:
|
| 266 |
+
return None
|
| 267 |
+
views = (g or {}).get("views")
|
| 268 |
+
if isinstance(views, dict):
|
| 269 |
+
found = views.get(want)
|
| 270 |
+
return found if isinstance(found, dict) else None
|
| 271 |
+
if not isinstance(views, list):
|
| 272 |
+
return None
|
| 273 |
+
for v in views:
|
| 274 |
+
if isinstance(v, dict) and str(v.get("id") or "") == want:
|
| 275 |
+
return v
|
| 276 |
+
return None
|
| 277 |
+
|
| 278 |
+
|
| 279 |
+
def _topic_or_400(raw):
|
| 280 |
+
topic = str(raw or "").strip().lower()
|
| 281 |
+
if topic.startswith("ut_") or topic in _TOPICS:
|
| 282 |
+
return topic
|
| 283 |
+
raise err(400, "bad_topic", f"topic must be one of {', '.join(_TOPICS)} or a ut_ table")
|
| 284 |
+
|
| 285 |
+
|
| 286 |
+
def _owner_session(session: Session, owner: str):
|
| 287 |
+
"""A `Session` for the alert's OWNER (see the module note on why the owner, not the caller).
|
| 288 |
+
|
| 289 |
+
β `Session` exposes `uname`/`admin` as PROPERTIES derived from `user`, not as fields β so an
|
| 290 |
+
owner session is built by swapping the `user` RECORD and letting both derive themselves. An
|
| 291 |
+
earlier version passed `uname=`/`admin=` to the constructor, which would have raised on the
|
| 292 |
+
first write hook of the wave; the properties are the single definition of who a session is,
|
| 293 |
+
and going around them is how a session with an admin flag and a non-admin record exists.
|
| 294 |
+
|
| 295 |
+
Returns None when the owner is gone or deactivated β their alerts then stop evaluating rather
|
| 296 |
+
than evaluating as somebody else, which is the fail-closed direction.
|
| 297 |
+
"""
|
| 298 |
+
import core.users as users
|
| 299 |
+
|
| 300 |
+
if str(owner) == str(session.uname):
|
| 301 |
+
return session
|
| 302 |
+
rec = (users.registry() or {}).get(str(owner))
|
| 303 |
+
if not isinstance(rec, dict) or not rec.get("active", True):
|
| 304 |
+
return None
|
| 305 |
+
# `_public` is THE definition of what a session may know about its own account (never a hash
|
| 306 |
+
# or a salt) β the same one `routes_auth` uses. Building the dict by hand here would be a
|
| 307 |
+
# second definition, and the one that leaks is always the copy.
|
| 308 |
+
return Session(tenant=session.tenant, user=users._public(str(owner), rec),
|
| 309 |
+
claims=session.claims, runtime=session.runtime)
|
| 310 |
+
|
| 311 |
+
|
| 312 |
+
def _evaluate(session: Session, rec: dict, assemblies=None):
|
| 313 |
+
"""Resolve `rec`'s view over its topic AS THE ALERT'S OWNER, then fold the result in.
|
| 314 |
+
|
| 315 |
+
ββ W31-T24 β `assemblies` IS A PER-REQUEST MEMO, KEYED `(topic, owner)`, and it is the whole
|
| 316 |
+
of this ticket's server half. `/notifications` re-evaluates EVERY alert inline on read and each
|
| 317 |
+
one built a FULL assembly β the pool, the workspace, `rows_from_pool` over every row. Two
|
| 318 |
+
alerts on one view built that table twice; ten built it ten times. Nothing dedupes them,
|
| 319 |
+
because each `_evaluate` was a closed call.
|
| 320 |
+
β `(topic, owner)` and not `topic`: the assembly is built as the alert's OWNER (see the module
|
| 321 |
+
note β evaluating a BU-scoped user's alert on a full-access admin's pool is a permission leak
|
| 322 |
+
wearing a notification's clothes), so two owners on one topic are two DIFFERENT tables and
|
| 323 |
+
must never share an entry. Getting that key wrong is the one way this optimisation could leak.
|
| 324 |
+
β Passing nothing keeps the old behaviour exactly, which is what the create/run doors want:
|
| 325 |
+
they evaluate ONE alert and a memo for a single call is pure overhead.
|
| 326 |
+
"""
|
| 327 |
+
import aios_grid
|
| 328 |
+
from harness import filter_eval
|
| 329 |
+
|
| 330 |
+
owner_sess = _owner_session(session, rec.get("owner"))
|
| 331 |
+
if owner_sess is None:
|
| 332 |
+
return {"skipped": "owner_unavailable"}
|
| 333 |
+
topic = str(rec.get("topic") or "")
|
| 334 |
+
memo_key = (topic, str(owner_sess.uname))
|
| 335 |
+
g = assemblies.get(memo_key) if isinstance(assemblies, dict) else None
|
| 336 |
+
if g is None:
|
| 337 |
+
try:
|
| 338 |
+
if topic.startswith("ut_"):
|
| 339 |
+
from routes_tables import ut_assembly
|
| 340 |
+
# β `consume_corrections=False`, and the default was a REAL BUG, not a tidy-up.
|
| 341 |
+
# `ut_assembly` defaults it True, so every `/notifications` read CONSUMED the
|
| 342 |
+
# one-shot field-name correction acks for every `ut_` topic that has an alert β
|
| 343 |
+
# taking them from the `/workspace` refresh that exists to show them to the person
|
| 344 |
+
# who made the edit. The customer branch below has always passed False; this one
|
| 345 |
+
# inherited a default nobody re-read. An inbox poll must never consume a one-shot.
|
| 346 |
+
g = ut_assembly(owner_sess, topic,
|
| 347 |
+
storage_key=f"{owner_sess.tenant}:{topic}:{owner_sess.uname}",
|
| 348 |
+
consume_corrections=False)
|
| 349 |
+
else:
|
| 350 |
+
from routes_customers import grid_assembly
|
| 351 |
+
g = grid_assembly(owner_sess, scope=topic, consume_corrections=False)
|
| 352 |
+
except Exception as e: # noqa: BLE001
|
| 353 |
+
# β W32-T22 β SKIPPING IS FINE HERE; SKIPPING ANONYMOUSLY IS NOT. This one must not
|
| 354 |
+
# raise (one bad alert cannot empty an inbox), so unlike `_require_filtered_view` it
|
| 355 |
+
# keeps a blanket catch β but it now reports the refusal's OWN code where there is
|
| 356 |
+
# one. `type(e).__name__` said `HTTPException` for four different causes, and
|
| 357 |
+
# `lastError` is the only place a user ever learns why an alert stopped firing.
|
| 358 |
+
#
|
| 359 |
+
# β `with_rows=True` STAYS on this path, deliberately: unlike the create door, an
|
| 360 |
+
# evaluation genuinely needs the rows to run the filter over. So an alert on a
|
| 361 |
+
# read-through grid is created (T22) and then skips at evaluation with
|
| 362 |
+
# `window_required` naming why β which is D-184's remaining half, and it is a
|
| 363 |
+
# SENTENCE now rather than silence.
|
| 364 |
+
return {"skipped": _refusal_code(e) or "unavailable", "detail": type(e).__name__}
|
| 365 |
+
if isinstance(assemblies, dict):
|
| 366 |
+
assemblies[memo_key] = g
|
| 367 |
+
|
| 368 |
+
view = _view_by_id(g, rec.get("viewId"))
|
| 369 |
+
if not isinstance(view, dict):
|
| 370 |
+
# Deleted, or un-shared out from under the alert. Say so on the RECORD rather than
|
| 371 |
+
# deleting the alert: an alert that silently vanishes is indistinguishable from one that
|
| 372 |
+
# never fires, and the user cannot debug what is not there.
|
| 373 |
+
return {"skipped": "view_missing"}
|
| 374 |
+
|
| 375 |
+
# The SAME row build the grid and `/customers` use β `rows_from_pool` is what puts derived
|
| 376 |
+
# and overlay values on a row. Evaluating a filter against raw pool dicts would silently
|
| 377 |
+
# never match any condition on a user-created or measure column.
|
| 378 |
+
#
|
| 379 |
+
# ββ AND "THE SAME ROW BUILD" WAS NOT TRUE, WHICH MADE EVERY ALERT ON A `ut_*` DATABASE BLIND
|
| 380 |
+
# TO IMPORTED DATA. Found by a verifier driving one real assembly through both paths.
|
| 381 |
+
#
|
| 382 |
+
# `routes_tables.table_rows` β the grid the person is looking at β merges the DEFINITION rows
|
| 383 |
+
# underneath the overlay ("base first, overlay wins"; that merge is itself the fix for owner
|
| 384 |
+
# item 3, *"it all got reseted"*). `_evaluate` is a second copy of that read and never got it:
|
| 385 |
+
# it handed `ws['overlays']` to `rows_from_pool` raw, so for a `ut_*` table every base cell
|
| 386 |
+
# evaluated as BLANK. Measured on one assembly, same view, same rows:
|
| 387 |
+
# rows_src state='unpaid' / 'paid'
|
| 388 |
+
# _evaluate saw state='' / '' β every base cell blank
|
| 389 |
+
# the GRID saw state='unpaid' / 'paid'
|
| 390 |
+
# so `state eq unpaid` matched NOTHING while the view showed one row, and `state isEmpty`
|
| 391 |
+
# matched EVERYTHING while the view showed none. **The alert did not merely miss rows β it
|
| 392 |
+
# inverted.** End to end: a row whose value arrived by import, automation, paste or the create
|
| 393 |
+
# door never fired; only a value typed as a hand EDIT did.
|
| 394 |
+
#
|
| 395 |
+
# β Scope, so nobody widens the fix past its cause: materialised `ut_*` tables are hit;
|
| 396 |
+
# `customer`/`product` are not (their fields are `source: "odoo"` and read off `rows_src`);
|
| 397 |
+
# `ut_odoo_*` never reaches here (`with_rows=True` refuses first and returns
|
| 398 |
+
# `skipped: window_required`).
|
| 399 |
+
# β ORDER IS LOAD-BEARING AND IS THE GRID'S: base underneath, overlay ON TOP. Inverting it
|
| 400 |
+
# would let a stale definition value shadow an edit the user has just made β the same defect
|
| 401 |
+
# `table_rows`' own note records, arriving from the other side.
|
| 402 |
+
_ov = (g.get("ws") or {}).get("overlays") or {}
|
| 403 |
+
_merged = {}
|
| 404 |
+
for _r in g["rows_src"]:
|
| 405 |
+
_pid = str(_r.get("pid"))
|
| 406 |
+
_cells = {k: v for k, v in _r.items() if k != "pid"}
|
| 407 |
+
_o = _ov.get(_pid)
|
| 408 |
+
if isinstance(_o, dict):
|
| 409 |
+
_cells.update(_o)
|
| 410 |
+
_merged[_pid] = _cells
|
| 411 |
+
rows = aios_grid.rows_from_pool(g["rows_src"], g["fields"], _merged,
|
| 412 |
+
derived=g.get("derived"))
|
| 413 |
+
config = view.get("config") or view
|
| 414 |
+
ctx = filter_eval.EvalCtx(
|
| 415 |
+
cohort_sets={str(k): {str(p) for p in (v.get("memberPids") or ())}
|
| 416 |
+
for k, v in (g.get("lists") or {}).items() if isinstance(v, dict)},
|
| 417 |
+
measure_sets=g.get("measure_sets") or {},
|
| 418 |
+
today=g.get("today"))
|
| 419 |
+
pids = filter_eval.visible_pids(config.get("filters") or [], rows, g["fields"], ctx,
|
| 420 |
+
member_pids=config.get("memberPids"))
|
| 421 |
+
labels = {str(r.get("pid")): str(r.get("name") or r.get("pid")) for r in rows}
|
| 422 |
+
return alerts.evaluate(rec.get("id"), [str(p) for p in pids],
|
| 423 |
+
labels=labels, partial=False, st=session.runtime)
|
| 424 |
+
|
| 425 |
+
|
| 426 |
+
@router.get("/alerts")
|
| 427 |
+
def list_alerts(session: Session = Depends(require_session)):
|
| 428 |
+
return {"alerts": alerts.list_alerts(user=session.uname, is_admin=session.admin,
|
| 429 |
+
st=session.runtime)}
|
| 430 |
+
|
| 431 |
+
|
| 432 |
+
@router.post("/alerts")
|
| 433 |
+
def create_alert(body: dict = Body(default=None), session: Session = Depends(require_session)):
|
| 434 |
+
body = body or {}
|
| 435 |
+
view_id = str(body.get("viewId") or "").strip()
|
| 436 |
+
if not view_id:
|
| 437 |
+
raise err(400, "bad_view", "an alert needs the id of the view it watches")
|
| 438 |
+
topic = _topic_or_400(body.get("topic"))
|
| 439 |
+
_require_filtered_view(session, topic, view_id)
|
| 440 |
+
import uuid
|
| 441 |
+
aid = f"al_{uuid.uuid4().hex[:12]}"
|
| 442 |
+
rec = alerts.create(aid, view_id=view_id, topic=topic, owner=session.uname,
|
| 443 |
+
label=body.get("label") or "", st=session.runtime)
|
| 444 |
+
# SEED IMMEDIATELY, so the alert starts from "everything currently matching is old news".
|
| 445 |
+
# Deferring this to the first write hook would mean the next edit announces the whole view.
|
| 446 |
+
outcome = _evaluate(session, rec)
|
| 447 |
+
return {"alert": {**rec, "seeded": True}, "first": outcome}
|
| 448 |
+
|
| 449 |
+
|
| 450 |
+
def _require_filtered_view(session: Session, topic: str, view_id: str):
|
| 451 |
+
"""400 unless `view_id` exists on `topic` AND actually narrows something.
|
| 452 |
+
|
| 453 |
+
β AN ALERT ON AN UNFILTERED VIEW IS SILENTLY INCAPABLE OF ALERTING, which is worse than one
|
| 454 |
+
that is refused. `filter_eval` treats an inactive tree as "no narrowing, every row shows"
|
| 455 |
+
(`visible_pids`'s own rule), so such an alert seeds with the entire table and can never see an
|
| 456 |
+
entrant again β there is nothing left to enter. The owner's words are *"when a Record gets
|
| 457 |
+
into that Filter's criteria"*: no criteria, no alert, and said at creation rather than
|
| 458 |
+
discovered by never being notified.
|
| 459 |
+
|
| 460 |
+
`is_rule_active` is the SAME activeness predicate the engine and the column tints use β a
|
| 461 |
+
half-typed rule is not a filter, and this must agree with what actually narrows or it would
|
| 462 |
+
accept a view whose one rule the engine then ignores.
|
| 463 |
+
|
| 464 |
+
ββ WAVE 32 Β· T22 (owner item 17) β THIS FUNCTION WAS THE ERROR. Two defects, stacked, and
|
| 465 |
+
the second one hid the first.
|
| 466 |
+
|
| 467 |
+
(1) **IT ASKED FOR EVERY ROW OF A TABLE IT NEVER LOOKS AT.** The only thing read below is
|
| 468 |
+
`g["views"]`. `ut_assembly` defaults `with_rows=True`, so creating an alert on a
|
| 469 |
+
read-through grid built the whole pool β and `scoped_pool` refuses that with
|
| 470 |
+
`409 window_required` over 963,783 rows, exactly as it is supposed to. `with_rows=False`
|
| 471 |
+
(W31-T20's flag, built for precisely this) answers the same question with `scoped_pids`,
|
| 472 |
+
runs the SAME `_defn_or_refuse` wall, and does not refuse. **That is D-184's create half,
|
| 473 |
+
closed** β an alert on a read-through grid can now be made at all.
|
| 474 |
+
(2) **A BLANKET `except Exception` TURNED EVERY NAMED REFUSAL INTO A 503.** `HTTPException`
|
| 475 |
+
is an `Exception`, so `404 unknown_table`, `403 forbidden`, `409 window_required` and
|
| 476 |
+
`503 store_not_ready` β four refusals that each say what is wrong β were all replaced by
|
| 477 |
+
*"the table is unavailable β try again in a moment"*. β AND THAT SENTENCE NEVER REACHED
|
| 478 |
+
A USER EITHER: `alertsApi.errorMessage` discards the text of any status β₯ 500 by design
|
| 479 |
+
(a 5xx body is the server's internals), substituting *"Something went wrong on our
|
| 480 |
+
side."* β which is the owner's screenshot, word for word. A knowable cause returned as a
|
| 481 |
+
5xx is invisible by construction, so re-wording the 503 could never have fixed this.
|
| 482 |
+
β The except is narrowed, not deleted: an UNEXPECTED failure is still a 503, because that is
|
| 483 |
+
honest. What it may no longer do is catch a refusal that already knows its own name.
|
| 484 |
+
"""
|
| 485 |
+
from fastapi import HTTPException
|
| 486 |
+
|
| 487 |
+
from harness import filter_eval
|
| 488 |
+
|
| 489 |
+
try:
|
| 490 |
+
if topic.startswith("ut_"):
|
| 491 |
+
from routes_tables import ut_assembly
|
| 492 |
+
# β `consume_corrections=False` β the customer branch has always passed it and this
|
| 493 |
+
# one inherited a default nobody re-read. Creating an alert must not eat the one-shot
|
| 494 |
+
# field-name correction acks belonging to the `/workspace` refresh that exists to show
|
| 495 |
+
# them to the person who made the edit. Same defect `_evaluate`'s header records.
|
| 496 |
+
g = ut_assembly(session, topic,
|
| 497 |
+
storage_key=f"{session.tenant}:{topic}:{session.uname}",
|
| 498 |
+
consume_corrections=False, with_rows=False)
|
| 499 |
+
else:
|
| 500 |
+
from routes_customers import grid_assembly
|
| 501 |
+
g = grid_assembly(session, scope=topic, consume_corrections=False)
|
| 502 |
+
except HTTPException:
|
| 503 |
+
raise # it already names its own cause
|
| 504 |
+
except Exception as e: # noqa: BLE001
|
| 505 |
+
# Genuinely unexpected. Still a 503, and now it carries the exception TYPE β without it,
|
| 506 |
+
# the one path that reaches this branch is also the one path with nothing to debug from.
|
| 507 |
+
raise err(503, "unavailable",
|
| 508 |
+
f"the table could not be read ({type(e).__name__}) β try again in a moment")
|
| 509 |
+
view = _view_by_id(g, view_id)
|
| 510 |
+
if not isinstance(view, dict):
|
| 511 |
+
raise err(404, "no_view", "that view does not exist on this table")
|
| 512 |
+
nodes, _conj = filter_eval.tree_parts((view.get("config") or view).get("filters") or [])
|
| 513 |
+
|
| 514 |
+
# ββ WAVE 33 Β· T29 β **CORRECTION: THE BLOCK BELOW IS TRUE ABOUT THE CODE AND FALSE ABOUT
|
| 515 |
+
# PRODUCTION, AND IT MUST BE READ SECOND.** It claims the missing-argument `TypeError` "IS
|
| 516 |
+
# owner item 17" β the owner's *"Something went wrong"*. It was not, and it could not have
|
| 517 |
+
# been: at `cbcf005`, the build the owner was using, the dict-read on `views` sat ~10 lines
|
| 518 |
+
# ABOVE this call and raised `AttributeError` on EVERY request, so the walk never reached the
|
| 519 |
+
# leaf and the arity bug was unreachable. `_view_by_id`'s own header records that fix.
|
| 520 |
+
#
|
| 521 |
+
# β WHY THE STALE PARAGRAPH STAYS RATHER THAN GETTING DELETED: the arity bug was real, the
|
| 522 |
+
# fix was right, and the three reasons it hid are the most transferable thing in this file.
|
| 523 |
+
# What was wrong is only its CLAIM TO BE THE CAUSE. Two comment blocks in one function each
|
| 524 |
+
# naming themselves as the origin of the same screenshot are mutually exclusive, and the next
|
| 525 |
+
# reader believes whichever they meet first β which is why this correction sits above rather
|
| 526 |
+
# than below. Caught by a verifier that read the SHIPPED file at the deployed commit instead
|
| 527 |
+
# of the working tree. [[grep-output-is-not-source]]
|
| 528 |
+
#
|
| 529 |
+
# ββ WAVE 32 Β· T22 β **THE CALL BELOW WAS MISSING AN ARGUMENT** (and wave 32 believed, wrongly,
|
| 530 |
+
# that this was owner item 17 β see the correction directly above).
|
| 531 |
+
#
|
| 532 |
+
# `is_rule_active(rule, columns)` takes TWO parameters (`harness/filter_sql.py`; every other
|
| 533 |
+
# caller in the repo passes both). This one passed ONE, so the moment the walk reached a LEAF
|
| 534 |
+
# rule it raised `TypeError: is_rule_active() missing 1 required positional argument`.
|
| 535 |
+
#
|
| 536 |
+
# β READ WHAT THAT MEANS BEFORE FIXING ANYTHING ELSE: the walk only reaches a leaf when the
|
| 537 |
+
# view HAS a condition β and a view with a condition is the only kind an alert is allowed on.
|
| 538 |
+
# A view with no filters yields an empty `nodes`, so `_any_active` returns False without ever
|
| 539 |
+
# calling this, and the reader gets the honest 400 `no_filter`. **So the only path that
|
| 540 |
+
# worked was the refusal path: "Alert me about new records" had never once created an alert
|
| 541 |
+
# on a filtered view.** β And the raise lands OUTSIDE the `try` above, so it was not even the
|
| 542 |
+
# 503 β it was a bare FastAPI 500, which `alertsApi.errorMessage` renders as *"Something went
|
| 543 |
+
# wrong on our side. Try again in a moment."*, the owner's screenshot word for word.
|
| 544 |
+
#
|
| 545 |
+
# β THREE THINGS HID IT, and they are worth more than the fix. (1) Python does not check
|
| 546 |
+
# arity until the line RUNS, and this line runs only on the success path of a feature whose
|
| 547 |
+
# every test exercised its refusals. (2) The `no_filter` 400 above it is a real, correct,
|
| 548 |
+
# well-tested refusal, so the door looked alive. (3) `verify_alerts.py` asserts the refusal
|
| 549 |
+
# (`no_filter` reaches the user) and the transport β never a creation. A gate can be green,
|
| 550 |
+
# thorough and honest about everything except the one path the feature exists for.
|
| 551 |
+
#
|
| 552 |
+
# `_columns_map` is the DEFINITION of fields -> the membership set `is_rule_active` looks a
|
| 553 |
+
# column up in; building a second dict here would be a second answer to one question, which
|
| 554 |
+
# is this wave's other headline defect in a different file. Its leading underscore is a real
|
| 555 |
+
# smell and is BOOKED (PENDING, mailbox/C.md) rather than worked around.
|
| 556 |
+
columns = filter_eval._columns_map(g.get("fields") or [])
|
| 557 |
+
|
| 558 |
+
def _any_active(ns):
|
| 559 |
+
for n in ns or ():
|
| 560 |
+
if isinstance(n, dict) and isinstance(n.get("children"), list):
|
| 561 |
+
if _any_active(n["children"]):
|
| 562 |
+
return True
|
| 563 |
+
elif filter_eval.is_rule_active(n, columns):
|
| 564 |
+
return True
|
| 565 |
+
return False
|
| 566 |
+
|
| 567 |
+
if not _any_active(nodes):
|
| 568 |
+
raise err(400, "no_filter",
|
| 569 |
+
"this view has no active filter, so no record can ever ENTER it β add a "
|
| 570 |
+
"condition to the view first, then create the alert")
|
| 571 |
+
|
| 572 |
+
|
| 573 |
+
@router.delete("/alerts/{alert_id}")
|
| 574 |
+
def delete_alert(alert_id: str, session: Session = Depends(require_session)):
|
| 575 |
+
rec = next((r for r in alerts.list_alerts(st=session.runtime)
|
| 576 |
+
if str(r.get("id")) == str(alert_id)), None)
|
| 577 |
+
if rec is None:
|
| 578 |
+
raise err(404, "no_alert", "that alert does not exist")
|
| 579 |
+
if str(rec.get("owner")) != str(session.uname) and not session.admin:
|
| 580 |
+
raise err(403, "not_yours", "only the alert's owner (or an administrator) can delete it")
|
| 581 |
+
alerts.delete(alert_id, st=session.runtime)
|
| 582 |
+
return {"ok": True}
|
| 583 |
+
|
| 584 |
+
|
| 585 |
+
@router.post("/alerts/{alert_id}/run")
|
| 586 |
+
def run_alert(alert_id: str, session: Session = Depends(require_session)):
|
| 587 |
+
rec = next((r for r in alerts.list_alerts(user=session.uname, is_admin=session.admin,
|
| 588 |
+
st=session.runtime)
|
| 589 |
+
if str(r.get("id")) == str(alert_id)), None)
|
| 590 |
+
if rec is None:
|
| 591 |
+
raise err(404, "no_alert", "that alert does not exist")
|
| 592 |
+
return _evaluate(session, rec)
|
| 593 |
+
|
| 594 |
+
|
| 595 |
+
@router.get("/notifications")
|
| 596 |
+
def notifications(session: Session = Depends(require_session)):
|
| 597 |
+
"""The inbox β RE-EVALUATED on read, which is a deliberate design choice.
|
| 598 |
+
|
| 599 |
+
β A-S1-2 RESOLVED THE OTHER WAY, and the reason is structural rather than a shortcut. The
|
| 600 |
+
plan was a push hook: the automation engine calls `after_write` when it lands rows. But
|
| 601 |
+
`run_async` runs on a BACKGROUND THREAD with no `Session` in scope, and an alert must be
|
| 602 |
+
evaluated as its OWNER (see `_evaluate`) β so a push hook would have to mint a session inside
|
| 603 |
+
a worker thread from a tenant runtime, which is exactly the kind of ad-hoc identity
|
| 604 |
+
construction that leaks scope.
|
| 605 |
+
|
| 606 |
+
Pulling on read has none of that: the caller IS a session, the assemblies are already
|
| 607 |
+
scope-cached, and the user cannot observe the difference β an inbox is only ever read by
|
| 608 |
+
someone opening it. The cost is that a notification is minted when you LOOK rather than when
|
| 609 |
+
the row landed, so the `at` stamp is detection time, not arrival time.
|
| 610 |
+
|
| 611 |
+
`after_write` stays exported for the day the engine can hand over a real identity.
|
| 612 |
+
|
| 613 |
+
ββ W31-T24 β ONE ASSEMBLY PER (TOPIC, OWNER), NOT ONE PER ALERT.
|
| 614 |
+
β MEASURED FIRST, AND THE MEASUREMENT CORRECTS AN EARLIER READING OF IT. This route is
|
| 615 |
+
**20 ms in-process and 3,280 ms live** on tenant #0 β but tenant #0 has **ZERO alerts**
|
| 616 |
+
(censused 2026-08-12), so the 20 ms is an EMPTY LOOP and says nothing at all about what the
|
| 617 |
+
re-evaluation costs. The live 3,280 ms is the two store reads either side of that loop. So the
|
| 618 |
+
body below is not slow today; it is UNEXERCISED, and every alert a tenant creates adds a whole
|
| 619 |
+
grid assembly to an inbox poll. The memo turns O(alerts) into O(distinct topic Γ owner), which
|
| 620 |
+
is the difference between "fine" and "three seconds per alert" the day somebody uses the
|
| 621 |
+
feature. β Making the read cheap by evaluating LESS is the obvious wrong fix and is not what
|
| 622 |
+
this does: every alert is still evaluated, against the same rows, in the same order.
|
| 623 |
+
"""
|
| 624 |
+
assemblies = {}
|
| 625 |
+
for rec in alerts.list_alerts(user=session.uname, is_admin=False, st=session.runtime):
|
| 626 |
+
try:
|
| 627 |
+
_evaluate(session, rec, assemblies=assemblies)
|
| 628 |
+
except Exception: # noqa: BLE001
|
| 629 |
+
continue # one bad alert must not empty the pane
|
| 630 |
+
# β W32-T20 (C3): every item leaves through `inbox_view`, so a notification queued before
|
| 631 |
+
# this wave carries a `target` too. See `notification_view`'s header for why it is derived.
|
| 632 |
+
return inbox_view(alerts.inbox(session.uname, st=session.runtime))
|
| 633 |
+
|
| 634 |
+
|
| 635 |
+
@router.post("/notifications/read")
|
| 636 |
+
def read_notifications(body: dict = Body(default=None),
|
| 637 |
+
session: Session = Depends(require_session)):
|
| 638 |
+
body = body or {}
|
| 639 |
+
ids = body.get("ids")
|
| 640 |
+
if ids is not None and not isinstance(ids, list):
|
| 641 |
+
raise err(400, "bad_ids", "ids must be a list, or null to mark every notification")
|
| 642 |
+
# β THE SAME ENRICHMENT ON BOTH DOORS. `mark_read` returns a fresh inbox, and the Inbox
|
| 643 |
+
# module re-renders from it β an un-enriched answer here would strip `target` off every row
|
| 644 |
+
# the moment somebody marked one read, i.e. the feature would work until first use.
|
| 645 |
+
return inbox_view(alerts.mark_read(session.uname, ids, read=bool(body.get("read", True)),
|
| 646 |
+
st=session.runtime))
|
| 647 |
+
|
| 648 |
+
|
| 649 |
+
def after_write(session: Session, topic_key: str):
|
| 650 |
+
"""THE WRITE HOOK β call after a write that could change what a view matches.
|
| 651 |
+
|
| 652 |
+
Exported as a plain function (not a route) so `core.grid_events`' callers and S2's automation
|
| 653 |
+
upserts reach it the same way. It never raises: an alert evaluation failing must not fail the
|
| 654 |
+
edit that triggered it.
|
| 655 |
+
|
| 656 |
+
β W31-T24 β it shares `/notifications`' memo shape for the same reason: a write that changes
|
| 657 |
+
one view can trip several alerts on the SAME topic, and each would otherwise rebuild the table.
|
| 658 |
+
β STILL ZERO PRODUCTION CALLERS (W31-T24 confirmed it; the route docstring above says why the
|
| 659 |
+
push hook was resolved the other way). Booked rather than wired: minting a session inside the
|
| 660 |
+
engine's worker thread is the ad-hoc identity construction this file exists to avoid.
|
| 661 |
+
"""
|
| 662 |
+
try:
|
| 663 |
+
assemblies = {}
|
| 664 |
+
return alerts.after_write(topic_key, st=session.runtime,
|
| 665 |
+
runner=lambda rec: _evaluate(session, rec,
|
| 666 |
+
assemblies=assemblies))
|
| 667 |
+
except Exception: # noqa: BLE001
|
| 668 |
+
return {"evaluated": 0}
|
api/routes_automation.py
CHANGED
|
@@ -14,6 +14,7 @@ trigger becomes a public one β the same class of mistake as an empty-200 permi
|
|
| 14 |
import os
|
| 15 |
import re
|
| 16 |
import time
|
|
|
|
| 17 |
|
| 18 |
from fastapi import APIRouter, Body, Depends, Header, Request
|
| 19 |
|
|
@@ -53,7 +54,7 @@ MODULE = "automation"
|
|
| 53 |
_GATE = module_gate(MODULE)
|
| 54 |
|
| 55 |
|
| 56 |
-
def _wire(defn, tenant):
|
| 57 |
"""One automation, as the client reads it. `running` is PROCESS state, never store state β
|
| 58 |
see the engine header on why a persisted 'running' is a permanent lock."""
|
| 59 |
live = engine.running(tenant, defn.get("id"))
|
|
@@ -141,6 +142,16 @@ def _wire(defn, tenant):
|
|
| 141 |
# back. What needs the target database's schema (an enrich binding resolved by the profile
|
| 142 |
# FLAG) stays a run-time refusal β see `ACTION_REQUIRED`'s note.
|
| 143 |
"unconfigured": engine.unconfigured_actions(defn),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 144 |
}
|
| 145 |
|
| 146 |
|
|
@@ -243,7 +254,7 @@ def _field_agent_rows(session):
|
|
| 243 |
# β the exact way `awaitingResults` shipped inert for a whole wave. Passing the
|
| 244 |
# synthetic DEFINITION through the same function makes them identical by
|
| 245 |
# construction; only `system` is stamped afterwards, because no stored row has it.
|
| 246 |
-
row = _wire(defn_syn, session.tenant)
|
| 247 |
row["system"] = SYSTEM_FIELD_AGENT
|
| 248 |
out.append(row)
|
| 249 |
return out
|
|
@@ -334,7 +345,7 @@ def _odoo_sync_row(session, detail=False):
|
|
| 334 |
"config": {"connector": "odoo", "every": every,
|
| 335 |
"frozen": frozen}}]},
|
| 336 |
}
|
| 337 |
-
row = _wire(defn, session.tenant)
|
| 338 |
row["system"] = SYSTEM_ODOO_SYNC
|
| 339 |
# β THE SENTENCE IS OVERRIDDEN, AND IT IS A FIX RATHER THAN A PREFERENCE. `compose_sentence`
|
| 340 |
# speaks the automation vocabulary β it builds "When <cron phrase>, run N actions" out of
|
|
@@ -745,7 +756,7 @@ def list_automations(session: Session = Depends(_GATE)):
|
|
| 745 |
pass
|
| 746 |
_STATEMENTS_SEEDED.add(session.tenant)
|
| 747 |
defs = engine.all_definitions(session.runtime)
|
| 748 |
-
items = [_wire(d, session.tenant) for _, d in
|
| 749 |
sorted(defs.items(), key=lambda kv: (kv[1].get("name") or "").lower())]
|
| 750 |
# ββ WAVE 34 Β· CONTRACT C3 (W34-T48) β field agents join the list as SYNTHETIC rows.
|
| 751 |
# β MERGED AND RE-SORTED, not appended in a block at the end. R13 asks for a field agent to be
|
|
@@ -1133,6 +1144,17 @@ def draft_automation(body: dict = Body(default=None), session: Session = Depends
|
|
| 1133 |
prompt = str((body or {}).get("prompt") or "").strip()
|
| 1134 |
if not prompt:
|
| 1135 |
raise err(400, "no_prompt", "type what you want the automation to do")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1136 |
# β THE TENANT'S OWN TABLES, THROUGH THE EXISTING WALL. `automation_tables` applies `may_open`
|
| 1137 |
# per table, so the model is shown exactly the databases this caller may already see and cannot
|
| 1138 |
# name one they were not granted β the permission wall re-used, never a second one built beside
|
|
@@ -1170,6 +1192,7 @@ def draft_automation(body: dict = Body(default=None), session: Session = Depends
|
|
| 1170 |
# sites, this one has a real person behind it: somebody typed the sentence, so `user` is
|
| 1171 |
# the caller rather than the automation's owner.
|
| 1172 |
st=session.runtime, user=getattr(session, "uname", "") or "",
|
|
|
|
| 1173 |
chat=_DRAFT_CHAT[0])
|
| 1174 |
if refusal or not draft:
|
| 1175 |
raise err(400, "draft_refused", refusal or "no automation could be drafted from that")
|
|
@@ -1263,6 +1286,244 @@ def draft_automation(body: dict = Body(default=None), session: Session = Depends
|
|
| 1263 |
"provider": provider, "dropped": dropped, "notes": _notes, "saved": False}
|
| 1264 |
|
| 1265 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1266 |
@router.get("/automations/{auto_id}")
|
| 1267 |
def get_automation(auto_id: str, session: Session = Depends(_GATE)):
|
| 1268 |
"""One automation, by id.
|
|
@@ -1295,7 +1556,7 @@ def get_automation(auto_id: str, session: Session = Depends(_GATE)):
|
|
| 1295 |
defn = engine.all_definitions(session.runtime).get(str(auto_id))
|
| 1296 |
if defn is None:
|
| 1297 |
raise err(404, "unknown_automation", "no automation with that id")
|
| 1298 |
-
return {"automation": _wire(defn, session.tenant)}
|
| 1299 |
|
| 1300 |
|
| 1301 |
@router.post("/automations")
|
|
@@ -1317,7 +1578,7 @@ def create_automation(body: dict = Body(default=None), session: Session = Depend
|
|
| 1317 |
notes=_notes)
|
| 1318 |
if error:
|
| 1319 |
raise err(400, "invalid_automation", error)
|
| 1320 |
-
return {"automation": _wire(defn, session.tenant), "notes": _notes}
|
| 1321 |
|
| 1322 |
|
| 1323 |
@router.patch("/automations/{auto_id}")
|
|
@@ -1339,6 +1600,10 @@ def patch_automation(auto_id: str, body: dict = Body(default=None),
|
|
| 1339 |
return _patch_odoo_sync(session, body or {})
|
| 1340 |
# D-75, same channel as `create` above β an EDIT is where this matters most, because the
|
| 1341 |
# person has just typed the thing that gets dropped.
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1342 |
_notes = []
|
| 1343 |
defn, error = engine.patch(session.runtime, auto_id, body or {}, username=session.uname,
|
| 1344 |
notes=_notes)
|
|
@@ -1346,7 +1611,10 @@ def patch_automation(auto_id: str, body: dict = Body(default=None),
|
|
| 1346 |
raise err(400 if error != "no such automation" else 404,
|
| 1347 |
"invalid_automation" if error != "no such automation" else "unknown_automation",
|
| 1348 |
error)
|
| 1349 |
-
|
|
|
|
|
|
|
|
|
|
| 1350 |
|
| 1351 |
|
| 1352 |
@router.delete("/automations/{auto_id}")
|
|
@@ -1384,6 +1652,9 @@ def delete_automation(auto_id: str, session: Session = Depends(_GATE)):
|
|
| 1384 |
if _sys:
|
| 1385 |
raise err(409, "system_agent", engine.system_agent_refusal(_sys))
|
| 1386 |
engine.remove(session.runtime, auto_id)
|
|
|
|
|
|
|
|
|
|
| 1387 |
return {"deleted": str(auto_id)}
|
| 1388 |
|
| 1389 |
|
|
@@ -1439,7 +1710,7 @@ def toggle_automation_node(auto_id: str, node_id: str, session: Session = Depend
|
|
| 1439 |
raise err(404 if error == "no such automation" else 400,
|
| 1440 |
"unknown_automation" if error == "no such automation" else "node_not_toggleable",
|
| 1441 |
error)
|
| 1442 |
-
return {"automation": _wire(defn, session.tenant)}
|
| 1443 |
|
| 1444 |
|
| 1445 |
@router.post("/automations/{auto_id}/hook/{token}")
|
|
|
|
| 14 |
import os
|
| 15 |
import re
|
| 16 |
import time
|
| 17 |
+
from datetime import datetime, timezone
|
| 18 |
|
| 19 |
from fastapi import APIRouter, Body, Depends, Header, Request
|
| 20 |
|
|
|
|
| 54 |
_GATE = module_gate(MODULE)
|
| 55 |
|
| 56 |
|
| 57 |
+
def _wire(defn, tenant, rt=None):
|
| 58 |
"""One automation, as the client reads it. `running` is PROCESS state, never store state β
|
| 59 |
see the engine header on why a persisted 'running' is a permanent lock."""
|
| 60 |
live = engine.running(tenant, defn.get("id"))
|
|
|
|
| 142 |
# back. What needs the target database's schema (an enrich binding resolved by the profile
|
| 143 |
# FLAG) stays a run-time refusal β see `ACTION_REQUIRED`'s note.
|
| 144 |
"unconfigured": engine.unconfigured_actions(defn),
|
| 145 |
+
# ββ WAVE 36 Β· W36-T38 (owner item 8 / R9) β WHICH ACTIONS AN AGENT BUILT.
|
| 146 |
+
# `{action_id: {agent, agentName, created, by}}`, empty for an ordinary automation.
|
| 147 |
+
# β THE CONFIG IS NOT HIDDEN AND MUST NOT BE. R9 is a WRITE rule: the client uses this to
|
| 148 |
+
# draw a lock and explain who owns the step, never to withhold what the step does β an
|
| 149 |
+
# automation nobody can audit is worse than one nobody can edit.
|
| 150 |
+
# β `rt=None` YIELDS `{}` RATHER THAN OMITTING THE KEY, so a caller that forgets the
|
| 151 |
+
# runtime produces "nothing is agent-owned" (a visible, wrong-but-safe answer) instead of
|
| 152 |
+
# an absent prop the client silently reads as undefined
|
| 153 |
+
# [[flag-shipped-without-its-writer]].
|
| 154 |
+
"agentActions": (_agent_marks(rt).get(str(defn.get("id") or "")) or {}) if rt else {},
|
| 155 |
}
|
| 156 |
|
| 157 |
|
|
|
|
| 254 |
# β the exact way `awaitingResults` shipped inert for a whole wave. Passing the
|
| 255 |
# synthetic DEFINITION through the same function makes them identical by
|
| 256 |
# construction; only `system` is stamped afterwards, because no stored row has it.
|
| 257 |
+
row = _wire(defn_syn, session.tenant, rt=session.runtime)
|
| 258 |
row["system"] = SYSTEM_FIELD_AGENT
|
| 259 |
out.append(row)
|
| 260 |
return out
|
|
|
|
| 345 |
"config": {"connector": "odoo", "every": every,
|
| 346 |
"frozen": frozen}}]},
|
| 347 |
}
|
| 348 |
+
row = _wire(defn, session.tenant, rt=session.runtime)
|
| 349 |
row["system"] = SYSTEM_ODOO_SYNC
|
| 350 |
# β THE SENTENCE IS OVERRIDDEN, AND IT IS A FIX RATHER THAN A PREFERENCE. `compose_sentence`
|
| 351 |
# speaks the automation vocabulary β it builds "When <cron phrase>, run N actions" out of
|
|
|
|
| 756 |
pass
|
| 757 |
_STATEMENTS_SEEDED.add(session.tenant)
|
| 758 |
defs = engine.all_definitions(session.runtime)
|
| 759 |
+
items = [_wire(d, session.tenant, rt=session.runtime) for _, d in
|
| 760 |
sorted(defs.items(), key=lambda kv: (kv[1].get("name") or "").lower())]
|
| 761 |
# ββ WAVE 34 Β· CONTRACT C3 (W34-T48) β field agents join the list as SYNTHETIC rows.
|
| 762 |
# β MERGED AND RE-SORTED, not appended in a block at the end. R13 asks for a field agent to be
|
|
|
|
| 1144 |
prompt = str((body or {}).get("prompt") or "").strip()
|
| 1145 |
if not prompt:
|
| 1146 |
raise err(400, "no_prompt", "type what you want the automation to do")
|
| 1147 |
+
# ββ ASK D-18 (2026-08-18) β THE KEY THE CLIENT SENDS IS NOW READ. This door took `prompt` off
|
| 1148 |
+
# the body and nothing else, so the Agent chat's model toggle (W36-T34) was a control that
|
| 1149 |
+
# configured nothing: the value was accepted and dropped, which is the shape three tickets in
|
| 1150 |
+
# this wave already tripped over. D held T34 rather than mark a picker BUILT while its value
|
| 1151 |
+
# stopped at the browser, and was right to.
|
| 1152 |
+
# β UNKNOWN OR UNCONFIGURED FALLS BACK TO THE LADDER, never refuses (D-18's asked-for posture,
|
| 1153 |
+
# and C5's everywhere else): a model the ladder stops offering must not turn every later draft
|
| 1154 |
+
# into an error. The response already names the rung that ANSWERED, which is the honest half.
|
| 1155 |
+
_model = str((body or {}).get("model") or "").strip().lower()
|
| 1156 |
+
if _model in ("", "auto"):
|
| 1157 |
+
_model = None
|
| 1158 |
# β THE TENANT'S OWN TABLES, THROUGH THE EXISTING WALL. `automation_tables` applies `may_open`
|
| 1159 |
# per table, so the model is shown exactly the databases this caller may already see and cannot
|
| 1160 |
# name one they were not granted β the permission wall re-used, never a second one built beside
|
|
|
|
| 1192 |
# sites, this one has a real person behind it: somebody typed the sentence, so `user` is
|
| 1193 |
# the caller rather than the automation's owner.
|
| 1194 |
st=session.runtime, user=getattr(session, "uname", "") or "",
|
| 1195 |
+
model=_model,
|
| 1196 |
chat=_DRAFT_CHAT[0])
|
| 1197 |
if refusal or not draft:
|
| 1198 |
raise err(400, "draft_refused", refusal or "no automation could be drafted from that")
|
|
|
|
| 1286 |
"provider": provider, "dropped": dropped, "notes": _notes, "saved": False}
|
| 1287 |
|
| 1288 |
|
| 1289 |
+
# ββββββββββββββββββ WAVE 36 Β· W36-T38 (owner item 8, ruling R9) β AGENT-AUTHORED ACTIONS ββββββ
|
| 1290 |
+
#
|
| 1291 |
+
# Owner, verbatim (2026-08-18): *"Create the ability for an automation agent to create any
|
| 1292 |
+
# 'Action' under the Canvas, so its configuration can only be touched by the agent. Be it a tool a
|
| 1293 |
+
# script etc. We need to really guardrail the reach of this script."*
|
| 1294 |
+
#
|
| 1295 |
+
# ββ R9 IS A **WRITE** RULE, NOT A VISIBILITY RULE, and the ticket says so in as many words: the
|
| 1296 |
+
# user MAY read an agent-authored action's configuration and MAY delete it; what they may not do
|
| 1297 |
+
# is hand-edit it. Hiding a config from the person whose workspace it runs in is a different
|
| 1298 |
+
# product, and a worse one β nobody can audit what they cannot see.
|
| 1299 |
+
#
|
| 1300 |
+
# β AND IT IS ENFORCED AT THE DOOR, NEVER IN A `disabled` ATTRIBUTE. `routes_automation` already
|
| 1301 |
+
# carries that lesson for a different control; a client-side lock is a suggestion, and the whole
|
| 1302 |
+
# point of R9 is that the agent's configuration stays coherent with what the agent believes it
|
| 1303 |
+
# built.
|
| 1304 |
+
#
|
| 1305 |
+
# β WHY THE MARK IS A SEPARATE BUCKET RATHER THAN A KEY ON THE ACTION. `clean_actions` builds every
|
| 1306 |
+
# action KEY BY KEY from an allowlist and drops anything it does not recognise (D-75) β so a
|
| 1307 |
+
# marker stored inside the action would be silently erased on the next save, and the wall would
|
| 1308 |
+
# quietly stop existing with every gate still green. This is an ANNOTATION layer keyed by
|
| 1309 |
+
# `(automation id, action id)`; the configuration itself stays where it always was, in the
|
| 1310 |
+
# automation, with exactly one writer.
|
| 1311 |
+
AGENT_ACTIONS_KEY = "automation_agent_actions"
|
| 1312 |
+
|
| 1313 |
+
#: Ids this door mints. β It must match `clean_actions`' `act_[a-z0-9_]{1,32}` or the engine
|
| 1314 |
+
#: re-mints it and the annotation points at an action that no longer carries that id.
|
| 1315 |
+
_AGENT_ACTION_PREFIX = "act_ag"
|
| 1316 |
+
|
| 1317 |
+
|
| 1318 |
+
def _agent_marks(rt):
|
| 1319 |
+
"""`{auto_id: {action_id: mark}}` for one tenant. `{}` on any failure β an unreadable
|
| 1320 |
+
annotation must degrade to "nothing is agent-owned", never to a 500 on the automations rail.
|
| 1321 |
+
|
| 1322 |
+
β AND "DEGRADE TO NOTHING IS OWNED" IS THE SAFE DIRECTION HERE, which is worth stating because
|
| 1323 |
+
it usually is not. The wall this feeds protects the AGENT's coherence, not the tenant's data: a
|
| 1324 |
+
lost mark lets a person edit a config they own anyway, in their own workspace, on a step they
|
| 1325 |
+
could always have deleted outright. A wall that failed CLOSED would instead make an automation
|
| 1326 |
+
permanently unsavable because one annotation read timed out.
|
| 1327 |
+
"""
|
| 1328 |
+
try:
|
| 1329 |
+
found = rt.get(AGENT_ACTIONS_KEY) or {}
|
| 1330 |
+
except Exception: # noqa: BLE001
|
| 1331 |
+
return {}
|
| 1332 |
+
return found if isinstance(found, dict) else {}
|
| 1333 |
+
|
| 1334 |
+
|
| 1335 |
+
def _flat_actions(actions, out=None, depth=0):
|
| 1336 |
+
"""Every action in a flow, INCLUDING the ones nested inside a group's arms.
|
| 1337 |
+
|
| 1338 |
+
β NESTED ACTIONS ARE THE ONES THIS MUST NOT MISS. `unconfigured_actions` walks them for the
|
| 1339 |
+
same reason: a step inside an If/then branch is exactly the one a person cannot see, and a
|
| 1340 |
+
wall that only looked at the top level would leave the agent's own nested step editable.
|
| 1341 |
+
"""
|
| 1342 |
+
out = [] if out is None else out
|
| 1343 |
+
if depth > 6 or not isinstance(actions, list):
|
| 1344 |
+
return out
|
| 1345 |
+
for action in actions:
|
| 1346 |
+
if not isinstance(action, dict):
|
| 1347 |
+
continue
|
| 1348 |
+
out.append(action)
|
| 1349 |
+
for key in ("then", "else", "actions"):
|
| 1350 |
+
_flat_actions(action.get(key), out, depth + 1)
|
| 1351 |
+
return out
|
| 1352 |
+
|
| 1353 |
+
|
| 1354 |
+
def _flow_actions(defn):
|
| 1355 |
+
return _flat_actions(((defn or {}).get("flow") or {}).get("actions") or [])
|
| 1356 |
+
|
| 1357 |
+
|
| 1358 |
+
def _same_action(left, right, rt):
|
| 1359 |
+
"""Do these two actions carry the SAME kind and configuration?
|
| 1360 |
+
|
| 1361 |
+
β COMPARED AFTER `clean_actions`, ON BOTH SIDES, and that is the difference between a wall and
|
| 1362 |
+
a nuisance. The stored action has already been through the cleaner; a client round-trip has
|
| 1363 |
+
not, so it may carry a key order, a blank string or a dropped condition that means nothing.
|
| 1364 |
+
Comparing raw shapes would refuse an edit that changes nothing, and a wall that fires on a
|
| 1365 |
+
no-op is one an operator learns to route around [[one-question-two-normalizers]].
|
| 1366 |
+
"""
|
| 1367 |
+
def _clean(action):
|
| 1368 |
+
cleaned, error = engine.clean_actions([dict(action or {}, id="act_1")], rt=rt)
|
| 1369 |
+
if error or not cleaned:
|
| 1370 |
+
return None
|
| 1371 |
+
one = dict(cleaned[0])
|
| 1372 |
+
one.pop("id", None)
|
| 1373 |
+
return one
|
| 1374 |
+
|
| 1375 |
+
a, b = _clean(left), _clean(right)
|
| 1376 |
+
return a is not None and a == b
|
| 1377 |
+
|
| 1378 |
+
|
| 1379 |
+
def _agent_owned_guard(session, auto_id, body):
|
| 1380 |
+
"""R9 AT THE DOOR: a user may not change an agent-authored action's kind or configuration.
|
| 1381 |
+
|
| 1382 |
+
Three outcomes, and the middle one is the ruling:
|
| 1383 |
+
* the action is ABSENT from the incoming flow -> a DELETE, and R9 allows it
|
| 1384 |
+
* the action is present and CHANGED -> **409**, naming the action and the agent
|
| 1385 |
+
* the action is present and identical -> nothing happens, so an ordinary save of a
|
| 1386 |
+
flow that merely CONTAINS an agent action
|
| 1387 |
+
is not refused
|
| 1388 |
+
"""
|
| 1389 |
+
marks = _agent_marks(session.runtime).get(str(auto_id)) or {}
|
| 1390 |
+
if not isinstance(marks, dict) or not marks:
|
| 1391 |
+
return
|
| 1392 |
+
incoming = (body or {}).get("flow")
|
| 1393 |
+
if not isinstance(incoming, dict) or "actions" not in incoming:
|
| 1394 |
+
return
|
| 1395 |
+
stored = (engine.all_definitions(session.runtime) or {}).get(str(auto_id)) or {}
|
| 1396 |
+
was = {str(a.get("id") or ""): a for a in _flow_actions(stored)}
|
| 1397 |
+
now = {str(a.get("id") or ""): a
|
| 1398 |
+
for a in _flat_actions(incoming.get("actions") or [])}
|
| 1399 |
+
for action_id, mark in marks.items():
|
| 1400 |
+
action_id = str(action_id)
|
| 1401 |
+
if action_id not in now or action_id not in was:
|
| 1402 |
+
continue # removed, or never stored: not an EDIT
|
| 1403 |
+
if _same_action(was[action_id], now[action_id], session.runtime):
|
| 1404 |
+
continue
|
| 1405 |
+
who = str((mark or {}).get("agentName") or (mark or {}).get("agent") or "an agent")
|
| 1406 |
+
raise err(409, "agent_owned",
|
| 1407 |
+
f"this step was built by {who} and only {who} can change how it is set up. "
|
| 1408 |
+
f"You can read it, and you can delete it, but it cannot be edited by hand")
|
| 1409 |
+
|
| 1410 |
+
|
| 1411 |
+
def _prune_marks(session, auto_id):
|
| 1412 |
+
"""Drop annotations whose action is gone, and the whole entry when the automation is.
|
| 1413 |
+
|
| 1414 |
+
β CALLED AFTER EVERY WRITE, because the alternative is an annotation bucket that only ever
|
| 1415 |
+
grows: a mark on a deleted action would keep refusing an id nobody can see, and a mark on a
|
| 1416 |
+
deleted automation would sit in the tenant document forever.
|
| 1417 |
+
"""
|
| 1418 |
+
auto_id = str(auto_id)
|
| 1419 |
+
|
| 1420 |
+
def _set(cur):
|
| 1421 |
+
cur = dict(cur or {})
|
| 1422 |
+
marks = cur.get(auto_id)
|
| 1423 |
+
if not isinstance(marks, dict):
|
| 1424 |
+
cur.pop(auto_id, None)
|
| 1425 |
+
return cur
|
| 1426 |
+
defn = (engine.all_definitions(session.runtime) or {}).get(auto_id)
|
| 1427 |
+
if defn is None:
|
| 1428 |
+
cur.pop(auto_id, None)
|
| 1429 |
+
return cur
|
| 1430 |
+
alive = {str(a.get("id") or "") for a in _flow_actions(defn)}
|
| 1431 |
+
kept = {k: v for k, v in marks.items() if str(k) in alive}
|
| 1432 |
+
if kept:
|
| 1433 |
+
cur[auto_id] = kept
|
| 1434 |
+
else:
|
| 1435 |
+
cur.pop(auto_id, None)
|
| 1436 |
+
return cur
|
| 1437 |
+
|
| 1438 |
+
session.runtime.update(AGENT_ACTIONS_KEY, _set, flush="sync")
|
| 1439 |
+
|
| 1440 |
+
|
| 1441 |
+
@router.post("/automations/{auto_id}/agent-actions")
|
| 1442 |
+
def agent_author_action(auto_id: str, body: dict = Body(default=None),
|
| 1443 |
+
session: Session = Depends(_GATE)):
|
| 1444 |
+
"""THE AGENT'S DOOR: add or replace one Action under the Canvas, marked agent-owned (R9).
|
| 1445 |
+
|
| 1446 |
+
{agent: "<agent id>", action: {kind, config, when?, id?}}
|
| 1447 |
+
-> {automation, actionId, agent}
|
| 1448 |
+
|
| 1449 |
+
β THE ACTION GOES THROUGH `clean_actions` LIKE EVERY OTHER ONE, and that is what "guardrail
|
| 1450 |
+
the reach of this script" means in code: an agent cannot invent a config key, cannot name a
|
| 1451 |
+
kind that is not in the catalog, and cannot reach a kind this tenant is not entitled to. The
|
| 1452 |
+
agent gets a different WALL on editing, never a wider vocabulary.
|
| 1453 |
+
|
| 1454 |
+
β AND IT CANNOT AUTHOR A KIND THAT IS NOT BUILT. `run_script` is `ready: False` in the
|
| 1455 |
+
catalog, so `clean_actions` refuses it here exactly as it refuses it for a person β see this
|
| 1456 |
+
lane's mailbox for why the script ARM is booked rather than half-built: a per-row script
|
| 1457 |
+
contract and a client card are both missing, and a step that reports success and does nothing
|
| 1458 |
+
is the failure this repo has already paid for.
|
| 1459 |
+
|
| 1460 |
+
β THE AGENT ID IS CHECKED AGAINST THE TENANT'S OWN AGENTS. A mark naming an agent that does
|
| 1461 |
+
not exist would refuse every future edit with a sentence naming nobody.
|
| 1462 |
+
"""
|
| 1463 |
+
import routes_slack # noqa: PLC0415
|
| 1464 |
+
|
| 1465 |
+
body = body if isinstance(body, dict) else {}
|
| 1466 |
+
agent_id = str(body.get("agent") or "").strip()
|
| 1467 |
+
agent = routes_slack._agents(session.runtime).get(agent_id)
|
| 1468 |
+
if not isinstance(agent, dict):
|
| 1469 |
+
raise err(404, "no_agent", "there is no agent with that id in this workspace")
|
| 1470 |
+
action = body.get("action")
|
| 1471 |
+
if not isinstance(action, dict) or not str(action.get("kind") or "").strip():
|
| 1472 |
+
raise err(400, "no_action", "an action needs a kind")
|
| 1473 |
+
|
| 1474 |
+
defn = (engine.all_definitions(session.runtime) or {}).get(str(auto_id))
|
| 1475 |
+
if defn is None:
|
| 1476 |
+
raise err(404, "unknown_automation", "no automation with that id")
|
| 1477 |
+
if str(defn.get("system") or "").strip():
|
| 1478 |
+
raise err(409, "system_agent", engine.system_agent_refusal(str(defn["system"])))
|
| 1479 |
+
|
| 1480 |
+
existing = list(((defn.get("flow") or {}).get("actions") or []))
|
| 1481 |
+
taken = {str(a.get("id") or "") for a in _flat_actions(existing)}
|
| 1482 |
+
action_id = str(action.get("id") or "").strip()
|
| 1483 |
+
if not action_id or action_id not in taken:
|
| 1484 |
+
n = 1
|
| 1485 |
+
while f"{_AGENT_ACTION_PREFIX}{n}" in taken:
|
| 1486 |
+
n += 1
|
| 1487 |
+
action_id = f"{_AGENT_ACTION_PREFIX}{n}"
|
| 1488 |
+
fresh = dict(action, id=action_id)
|
| 1489 |
+
|
| 1490 |
+
# β VALIDATED BEFORE ANYTHING IS WRITTEN, so a refusal leaves the automation exactly as it was.
|
| 1491 |
+
checked, error = engine.clean_actions([fresh], rt=session.runtime)
|
| 1492 |
+
if error or not checked:
|
| 1493 |
+
raise err(400, "invalid_action", error or "that action could not be built")
|
| 1494 |
+
|
| 1495 |
+
replaced = False
|
| 1496 |
+
for i, one in enumerate(existing):
|
| 1497 |
+
if isinstance(one, dict) and str(one.get("id") or "") == action_id:
|
| 1498 |
+
existing[i], replaced = fresh, True
|
| 1499 |
+
break
|
| 1500 |
+
if not replaced:
|
| 1501 |
+
existing.append(fresh)
|
| 1502 |
+
|
| 1503 |
+
notes = []
|
| 1504 |
+
updated, error = engine.patch(session.runtime, str(auto_id),
|
| 1505 |
+
{"flow": {**(defn.get("flow") or {}), "actions": existing}},
|
| 1506 |
+
username=session.uname, notes=notes)
|
| 1507 |
+
if error:
|
| 1508 |
+
raise err(400, "invalid_automation", error)
|
| 1509 |
+
|
| 1510 |
+
def _set(cur):
|
| 1511 |
+
cur = dict(cur or {})
|
| 1512 |
+
marks = dict(cur.get(str(auto_id)) or {})
|
| 1513 |
+
marks[action_id] = {"agent": agent_id,
|
| 1514 |
+
"agentName": str(agent.get("label") or agent.get("channelName")
|
| 1515 |
+
or agent_id),
|
| 1516 |
+
"created": datetime.now(timezone.utc).isoformat(timespec="seconds"),
|
| 1517 |
+
"by": session.uname}
|
| 1518 |
+
cur[str(auto_id)] = marks
|
| 1519 |
+
return cur
|
| 1520 |
+
|
| 1521 |
+
session.runtime.update(AGENT_ACTIONS_KEY, _set, flush="sync")
|
| 1522 |
+
_prune_marks(session, auto_id)
|
| 1523 |
+
return {"automation": _wire(updated, session.tenant, rt=session.runtime),
|
| 1524 |
+
"actionId": action_id, "agent": agent_id, "notes": notes}
|
| 1525 |
+
|
| 1526 |
+
|
| 1527 |
@router.get("/automations/{auto_id}")
|
| 1528 |
def get_automation(auto_id: str, session: Session = Depends(_GATE)):
|
| 1529 |
"""One automation, by id.
|
|
|
|
| 1556 |
defn = engine.all_definitions(session.runtime).get(str(auto_id))
|
| 1557 |
if defn is None:
|
| 1558 |
raise err(404, "unknown_automation", "no automation with that id")
|
| 1559 |
+
return {"automation": _wire(defn, session.tenant, rt=session.runtime)}
|
| 1560 |
|
| 1561 |
|
| 1562 |
@router.post("/automations")
|
|
|
|
| 1578 |
notes=_notes)
|
| 1579 |
if error:
|
| 1580 |
raise err(400, "invalid_automation", error)
|
| 1581 |
+
return {"automation": _wire(defn, session.tenant, rt=session.runtime), "notes": _notes}
|
| 1582 |
|
| 1583 |
|
| 1584 |
@router.patch("/automations/{auto_id}")
|
|
|
|
| 1600 |
return _patch_odoo_sync(session, body or {})
|
| 1601 |
# D-75, same channel as `create` above β an EDIT is where this matters most, because the
|
| 1602 |
# person has just typed the thing that gets dropped.
|
| 1603 |
+
# ββ W36-T38 / R9 β THE AGENT-OWNED WALL, AT THE DOOR, BEFORE ANY WRITE. A `disabled`
|
| 1604 |
+
# attribute in the builder is a suggestion; this is the refusal. It permits a DELETE of the
|
| 1605 |
+
# step and permits an ordinary save of a flow that merely contains one.
|
| 1606 |
+
_agent_owned_guard(session, auto_id, body or {})
|
| 1607 |
_notes = []
|
| 1608 |
defn, error = engine.patch(session.runtime, auto_id, body or {}, username=session.uname,
|
| 1609 |
notes=_notes)
|
|
|
|
| 1611 |
raise err(400 if error != "no such automation" else 404,
|
| 1612 |
"invalid_automation" if error != "no such automation" else "unknown_automation",
|
| 1613 |
error)
|
| 1614 |
+
# An action the user removed takes its annotation with it, or the mark outlives its subject
|
| 1615 |
+
# and refuses an id nobody can see.
|
| 1616 |
+
_prune_marks(session, auto_id)
|
| 1617 |
+
return {"automation": _wire(defn, session.tenant, rt=session.runtime), "notes": _notes}
|
| 1618 |
|
| 1619 |
|
| 1620 |
@router.delete("/automations/{auto_id}")
|
|
|
|
| 1652 |
if _sys:
|
| 1653 |
raise err(409, "system_agent", engine.system_agent_refusal(_sys))
|
| 1654 |
engine.remove(session.runtime, auto_id)
|
| 1655 |
+
# R9's other half: the user MAY delete an agent-authored step, and deleting the whole
|
| 1656 |
+
# automation takes its annotations with it rather than stranding them in the document.
|
| 1657 |
+
_prune_marks(session, auto_id)
|
| 1658 |
return {"deleted": str(auto_id)}
|
| 1659 |
|
| 1660 |
|
|
|
|
| 1710 |
raise err(404 if error == "no such automation" else 400,
|
| 1711 |
"unknown_automation" if error == "no such automation" else "node_not_toggleable",
|
| 1712 |
error)
|
| 1713 |
+
return {"automation": _wire(defn, session.tenant, rt=session.runtime)}
|
| 1714 |
|
| 1715 |
|
| 1716 |
@router.post("/automations/{auto_id}/hook/{token}")
|
api/routes_grid.py
CHANGED
|
@@ -1000,3 +1000,84 @@ def grid_events_route(body: dict = Body(default=None),
|
|
| 1000 |
# `pool_cache` after an `overlay_patch`, which threw away an expensive Odoo pull to refresh
|
| 1001 |
# data that was never in it.
|
| 1002 |
return out
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1000 |
# `pool_cache` after an `overlay_patch`, which threw away an expensive Odoo pull to refresh
|
| 1001 |
# data that was never in it.
|
| 1002 |
return out
|
| 1003 |
+
|
| 1004 |
+
|
| 1005 |
+
# ββ CONTRACT C1 (W36-T20): THE REGISTRY TOPICS' ROW READERS βββββββββββββββββββββββββββββββββββ
|
| 1006 |
+
# ββ R6 β *"EVERY database gets the same permission logic, always."* `core.perm_scope.scoped_table`
|
| 1007 |
+
# is the ONE door to any database's rows, and it cannot import a topic's pool builder: `core` never
|
| 1008 |
+
# imports up (`platform/ARCHITECTURE.md`) and these pools are built by `modules/` + `aios_grid`
|
| 1009 |
+
# behind this layer's per-tenant cache. So the app layer DECLARES its readers, exactly as
|
| 1010 |
+
# `routes_odoo_tables` declares connected tables to `user_tables.register_connected`.
|
| 1011 |
+
#
|
| 1012 |
+
# β REGISTERED HERE RATHER THAN IN `routes_customers`/`routes_products` because those two files
|
| 1013 |
+
# are outside wave 36's lane-C fence. The readers themselves are three lines each and call the
|
| 1014 |
+
# SAME `_pool_for` + `derive_pool_scope` pair those routes call, so there is no second pool and no
|
| 1015 |
+
# second scope derivation β only a second CALLER of the one that exists.
|
| 1016 |
+
#
|
| 1017 |
+
# β AND THE TOPIC ROUTES STILL HAVE THEIR OWN DOOR TODAY. Contract C1 says `apply_row_scope` +
|
| 1018 |
+
# `visible_fields` "move behind" `scoped_table`; moving `routes_customers.grid_assembly` and
|
| 1019 |
+
# `routes_products.scoped_pool` is booked as a PENDING row (mailbox/C.md, C-1) rather than done
|
| 1020 |
+
# here, because neither file is in this fence. What ships now is the arm the wave is load-bearing
|
| 1021 |
+
# on β every `ut_*` database, plus E's sandbox β and a topic arm that is REAL rather than stubbed,
|
| 1022 |
+
# so `scoped_table`'s topic leg is exercised by the product instead of only by a gate.
|
| 1023 |
+
def _topic_rt(st, module):
|
| 1024 |
+
"""The tenant runtime a topic pool must be built against, or a REPORTED refusal.
|
| 1025 |
+
|
| 1026 |
+
β A topic pool is per TENANT (`rt.pool_cache`), so `st=None` cannot be resolved to "the
|
| 1027 |
+
default" without picking a tenant at random β which on this box is tenant #0's PRODUCTION
|
| 1028 |
+
data. Standing rule 1's second sentence: say why, and say what to do instead.
|
| 1029 |
+
"""
|
| 1030 |
+
if st is None:
|
| 1031 |
+
import core.perm_scope as perm_scope
|
| 1032 |
+
raise perm_scope.Unresolvable(
|
| 1033 |
+
subject="rows", effect="unreadable",
|
| 1034 |
+
cause=f"'{module}' is a registry topic whose pool is built per tenant and no tenant "
|
| 1035 |
+
f"runtime was passed",
|
| 1036 |
+
recommendation="pass the session's runtime as `st=`. A topic pool cannot be "
|
| 1037 |
+
"resolved without knowing which tenant is asking")
|
| 1038 |
+
return st
|
| 1039 |
+
|
| 1040 |
+
|
| 1041 |
+
def _customer_rows(table_key, user, st):
|
| 1042 |
+
"""`(fields, rows)` for the customer topic β the SAME derivation `_team_agent` uses."""
|
| 1043 |
+
import aios_grid
|
| 1044 |
+
import core.perm_scope as perm_scope
|
| 1045 |
+
from routes_customers import _pool_for
|
| 1046 |
+
|
| 1047 |
+
rt = _topic_rt(st, table_key)
|
| 1048 |
+
team_id, agent = perm_scope.derive_pool_scope(user, table_key)
|
| 1049 |
+
return list(aios_grid.FIELDS), _pool_for(rt, team_id, agent)
|
| 1050 |
+
|
| 1051 |
+
|
| 1052 |
+
def _product_rows(table_key, user, st):
|
| 1053 |
+
"""`(fields, rows)` for the product topic. `consolidated=` follows the derived scope, so a
|
| 1054 |
+
BU-pinned reader gets that BU's field contract rather than the consolidated one."""
|
| 1055 |
+
import core.perm_scope as perm_scope
|
| 1056 |
+
from routes_products import _pool_for, pd_fields
|
| 1057 |
+
|
| 1058 |
+
rt = _topic_rt(st, table_key)
|
| 1059 |
+
team_id, _agent = perm_scope.derive_pool_scope(user, table_key)
|
| 1060 |
+
return pd_fields(consolidated=team_id is None), _pool_for(rt, team_id)
|
| 1061 |
+
|
| 1062 |
+
|
| 1063 |
+
def _register_topic_rows():
|
| 1064 |
+
"""Declare both topic readers to C1. Called at import; returns the registered key set.
|
| 1065 |
+
|
| 1066 |
+
β THE KEYS ARE LITERALS AND THE ROUTE IMPORTS ARE INSIDE THE READERS, on purpose: this runs at
|
| 1067 |
+
module import, and `from routes_products import MODULE` here would pull a sibling router in
|
| 1068 |
+
before its own imports have settled. Every other cross-router reference in this file is lazy
|
| 1069 |
+
for the same reason. The literals are held to their sources by `verify_scopes`, so they cannot
|
| 1070 |
+
drift into naming a topic that does not exist.
|
| 1071 |
+
"""
|
| 1072 |
+
import core.perm_scope as perm_scope
|
| 1073 |
+
|
| 1074 |
+
perm_scope.register_rows(_customer_rows, MODULE)
|
| 1075 |
+
return perm_scope.register_rows(_product_rows, _PRODUCT_MODULE)
|
| 1076 |
+
|
| 1077 |
+
|
| 1078 |
+
#: β `_`-prefixed, because three functions in this file already bind the name `PRODUCT_MODULE`
|
| 1079 |
+
#: LOCALLY from `routes_products`. A module-level twin of that spelling would read as the same
|
| 1080 |
+
#: thing and be a different one β [[constant-two-features-share]] waiting to happen.
|
| 1081 |
+
_PRODUCT_MODULE = "product_data"
|
| 1082 |
+
|
| 1083 |
+
_C1_ROW_SOURCES = _register_topic_rows()
|
api/routes_keychain.py
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
api/routes_nav.py
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
api/routes_oauth.py
CHANGED
|
@@ -1,114 +1,114 @@
|
|
| 1 |
-
"""routes_oauth.py β the OAuth connector surface (wave 22, contract C5 + A2/A3 / R12).
|
| 2 |
-
|
| 3 |
-
Thin over `oauth_connect`, the way `routes_automation` is thin over the engine: sessions,
|
| 4 |
-
shapes and status codes here; every decision that could be wrong lives in the module a gate
|
| 5 |
-
can drive without a server. GENERIC over `{provider}` (C5-A2): the routes read the registry,
|
| 6 |
-
so the day a second provider lands here is the day nothing in this file changes.
|
| 7 |
-
|
| 8 |
-
MOUNTED FROM `routes_automation` (not `main.py`): this wave's ownership fence gives no session
|
| 9 |
-
`main.py`, and `routes_automation` is already included there β so this router rides inside it
|
| 10 |
-
(`/api/v1` + `/oauth/...`). Lifting the include into `main.py` later is a two-line change that
|
| 11 |
-
alters no path.
|
| 12 |
-
|
| 13 |
-
β THE TWO REDIRECT LAWS (A3): `/{provider}/start` answers **302 to the provider's consent
|
| 14 |
-
screen** β it is a top-level navigation the client reaches by `<a href>`, never JSON. The
|
| 15 |
-
callback 302s BACK to the return path the `state` carried (relative-only, sanitised by
|
| 16 |
-
`oauth_connect.safe_next`), so the user lands where they left β connected or not, whatever
|
| 17 |
-
went wrong rides in the query string; a dead-end error page where the app used to be reads as
|
| 18 |
-
"the product broke", not "the connect failed".
|
| 19 |
-
"""
|
| 20 |
-
import os
|
| 21 |
-
|
| 22 |
-
from fastapi import APIRouter, Depends, Request
|
| 23 |
-
from fastapi.responses import RedirectResponse
|
| 24 |
-
|
| 25 |
-
import oauth_connect
|
| 26 |
-
from deps import Session, err, require_session
|
| 27 |
-
|
| 28 |
-
router = APIRouter(prefix="/oauth")
|
| 29 |
-
|
| 30 |
-
|
| 31 |
-
def _redirect_uri(request: Request, provider: str) -> str:
|
| 32 |
-
"""The redirect URI this deployment registers at the provider β env-pinned when the
|
| 33 |
-
container sits behind a proxy that rewrites the scheme (the HF Space), else derived from
|
| 34 |
-
the request. MUST match a console-registered URI verbatim, so it is computed in exactly
|
| 35 |
-
one place.
|
| 36 |
-
|
| 37 |
-
β WAVE 29 (R4): `deploy_web.py` now PUSHES `AIOS_PUBLIC_BASE` on every deploy, defaulted to
|
| 38 |
-
the same URL as `APP_BASE_URL`, so the pinned branch is the one that runs in production and
|
| 39 |
-
the request-derived fallback below is effectively dev-only.
|
| 40 |
-
β THAT MAKES THIS FUNCTION A CUSTOM-DOMAIN COUPLING, not merely a scheme fix. Whatever host
|
| 41 |
-
this returns is where the provider sends the user BACK, and the session cookie is host-only
|
| 42 |
-
(`aios_session.py:114-117`, no `domain=`) β so a callback base that disagrees with the host
|
| 43 |
-
the user actually browsed plants the session on the wrong hostname and they return logged
|
| 44 |
-
out. Moving the app to a new hostname means moving this value AND re-registering the
|
| 45 |
-
resulting URI in the provider console; one without the other fails closed.
|
| 46 |
-
Runbook: `.claude/wiki/research/loopable-domain-runbook.md`."""
|
| 47 |
-
base = (os.environ.get("AIOS_PUBLIC_BASE") or "").strip().rstrip("/")
|
| 48 |
-
if not base:
|
| 49 |
-
base = f"{request.url.scheme}://{request.url.netloc}"
|
| 50 |
-
return f"{base}/api/v1/oauth/{provider}/callback"
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
@router.get("/status")
|
| 54 |
-
def oauth_status(session: Session = Depends(require_session)):
|
| 55 |
-
"""C5's status shape for the SESSION user, one entry per registry provider:
|
| 56 |
-
`{google: {connected, email, reconnect, configured}}` today. The bit the email trigger's
|
| 57 |
-
`ready` reads through."""
|
| 58 |
-
return oauth_connect.status(session.runtime, session.uname)
|
| 59 |
-
|
| 60 |
-
|
| 61 |
-
def _offered_or_404(provider: str):
|
| 62 |
-
"""ββ W32-T14 / OWNER ITEM 12 / R6 β A PROVIDER THE PRODUCT DOES NOT OFFER HAS NO DOOR.
|
| 63 |
-
|
| 64 |
-
The owner pasted the failure this replaces: clicking Connect on the Google card answered
|
| 65 |
-
`503 oauth_unavailable` as raw JSON. R6's fix is not a nicer error β it is that the flow is
|
| 66 |
-
not offered at all, so the honest status is the one for a URL that does not exist. `404`
|
| 67 |
-
rather than `503`, deliberately: a 503 says *"come back later"* about a door that is not
|
| 68 |
-
coming back until somebody pays for CASA verification (D-45, ~$540β1,800/yr).
|
| 69 |
-
|
| 70 |
-
β Every door in this router goes through it, START included, because the JSON the owner saw
|
| 71 |
-
came from the start route and a guard on one leg is a guard on one leg.
|
| 72 |
-
"""
|
| 73 |
-
if not oauth_connect.offered(provider):
|
| 74 |
-
raise err(404, "unknown_provider", f"{provider!r} is not a connectable provider")
|
| 75 |
-
|
| 76 |
-
|
| 77 |
-
@router.get("/{provider}/start")
|
| 78 |
-
def oauth_start(provider: str, request: Request, next: str = "",
|
| 79 |
-
session: Session = Depends(require_session)):
|
| 80 |
-
"""302 to the provider's consent screen (A3 β a navigation, never JSON). `?next=` is the
|
| 81 |
-
RELATIVE path the callback returns the browser to; it rides inside the single-use state,
|
| 82 |
-
sanitised, so the round trip cannot be steered off-origin."""
|
| 83 |
-
_offered_or_404(provider)
|
| 84 |
-
url, problem = oauth_connect.start(provider, session.uname,
|
| 85 |
-
_redirect_uri(request, provider), next_path=next)
|
| 86 |
-
if problem:
|
| 87 |
-
raise err(503 if "not configured" in problem else 404, "oauth_unavailable", problem)
|
| 88 |
-
return RedirectResponse(url, status_code=302)
|
| 89 |
-
|
| 90 |
-
|
| 91 |
-
@router.get("/{provider}/callback")
|
| 92 |
-
def oauth_callback(provider: str, request: Request,
|
| 93 |
-
session: Session = Depends(require_session),
|
| 94 |
-
state: str = "", code: str = "", error: str = ""):
|
| 95 |
-
"""The provider's redirect target. Exchanges the code, stores the per-user slot, and sends
|
| 96 |
-
the browser back to the state's return path β connected or not (see module header)."""
|
| 97 |
-
_offered_or_404(provider)
|
| 98 |
-
if error:
|
| 99 |
-
home = "/#/"
|
| 100 |
-
return RedirectResponse(f"{home}?oauthError={error[:80]}", status_code=302)
|
| 101 |
-
email, home, problem = oauth_connect.callback(session.runtime, session.uname, state, code)
|
| 102 |
-
sep = "&" if "?" in home else "?"
|
| 103 |
-
if problem:
|
| 104 |
-
return RedirectResponse(f"{home}{sep}oauthError=connect_failed", status_code=302)
|
| 105 |
-
return RedirectResponse(f"{home}{sep}connected={provider}", status_code=302)
|
| 106 |
-
|
| 107 |
-
|
| 108 |
-
@router.post("/{provider}/disconnect")
|
| 109 |
-
def oauth_disconnect(provider: str, session: Session = Depends(require_session)):
|
| 110 |
-
_offered_or_404(provider)
|
| 111 |
-
if oauth_connect.provider_def(provider) is None:
|
| 112 |
-
raise err(404, "unknown_provider", f"{provider!r} is not a connectable provider")
|
| 113 |
-
oauth_connect.disconnect(session.runtime, session.uname, provider)
|
| 114 |
-
return {"disconnected": provider}
|
|
|
|
| 1 |
+
"""routes_oauth.py β the OAuth connector surface (wave 22, contract C5 + A2/A3 / R12).
|
| 2 |
+
|
| 3 |
+
Thin over `oauth_connect`, the way `routes_automation` is thin over the engine: sessions,
|
| 4 |
+
shapes and status codes here; every decision that could be wrong lives in the module a gate
|
| 5 |
+
can drive without a server. GENERIC over `{provider}` (C5-A2): the routes read the registry,
|
| 6 |
+
so the day a second provider lands here is the day nothing in this file changes.
|
| 7 |
+
|
| 8 |
+
MOUNTED FROM `routes_automation` (not `main.py`): this wave's ownership fence gives no session
|
| 9 |
+
`main.py`, and `routes_automation` is already included there β so this router rides inside it
|
| 10 |
+
(`/api/v1` + `/oauth/...`). Lifting the include into `main.py` later is a two-line change that
|
| 11 |
+
alters no path.
|
| 12 |
+
|
| 13 |
+
β THE TWO REDIRECT LAWS (A3): `/{provider}/start` answers **302 to the provider's consent
|
| 14 |
+
screen** β it is a top-level navigation the client reaches by `<a href>`, never JSON. The
|
| 15 |
+
callback 302s BACK to the return path the `state` carried (relative-only, sanitised by
|
| 16 |
+
`oauth_connect.safe_next`), so the user lands where they left β connected or not, whatever
|
| 17 |
+
went wrong rides in the query string; a dead-end error page where the app used to be reads as
|
| 18 |
+
"the product broke", not "the connect failed".
|
| 19 |
+
"""
|
| 20 |
+
import os
|
| 21 |
+
|
| 22 |
+
from fastapi import APIRouter, Depends, Request
|
| 23 |
+
from fastapi.responses import RedirectResponse
|
| 24 |
+
|
| 25 |
+
import oauth_connect
|
| 26 |
+
from deps import Session, err, require_session
|
| 27 |
+
|
| 28 |
+
router = APIRouter(prefix="/oauth")
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def _redirect_uri(request: Request, provider: str) -> str:
|
| 32 |
+
"""The redirect URI this deployment registers at the provider β env-pinned when the
|
| 33 |
+
container sits behind a proxy that rewrites the scheme (the HF Space), else derived from
|
| 34 |
+
the request. MUST match a console-registered URI verbatim, so it is computed in exactly
|
| 35 |
+
one place.
|
| 36 |
+
|
| 37 |
+
β WAVE 29 (R4): `deploy_web.py` now PUSHES `AIOS_PUBLIC_BASE` on every deploy, defaulted to
|
| 38 |
+
the same URL as `APP_BASE_URL`, so the pinned branch is the one that runs in production and
|
| 39 |
+
the request-derived fallback below is effectively dev-only.
|
| 40 |
+
β THAT MAKES THIS FUNCTION A CUSTOM-DOMAIN COUPLING, not merely a scheme fix. Whatever host
|
| 41 |
+
this returns is where the provider sends the user BACK, and the session cookie is host-only
|
| 42 |
+
(`aios_session.py:114-117`, no `domain=`) β so a callback base that disagrees with the host
|
| 43 |
+
the user actually browsed plants the session on the wrong hostname and they return logged
|
| 44 |
+
out. Moving the app to a new hostname means moving this value AND re-registering the
|
| 45 |
+
resulting URI in the provider console; one without the other fails closed.
|
| 46 |
+
Runbook: `.claude/wiki/research/loopable-domain-runbook.md`."""
|
| 47 |
+
base = (os.environ.get("AIOS_PUBLIC_BASE") or "").strip().rstrip("/")
|
| 48 |
+
if not base:
|
| 49 |
+
base = f"{request.url.scheme}://{request.url.netloc}"
|
| 50 |
+
return f"{base}/api/v1/oauth/{provider}/callback"
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
@router.get("/status")
|
| 54 |
+
def oauth_status(session: Session = Depends(require_session)):
|
| 55 |
+
"""C5's status shape for the SESSION user, one entry per registry provider:
|
| 56 |
+
`{google: {connected, email, reconnect, configured}}` today. The bit the email trigger's
|
| 57 |
+
`ready` reads through."""
|
| 58 |
+
return oauth_connect.status(session.runtime, session.uname)
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
def _offered_or_404(provider: str):
|
| 62 |
+
"""ββ W32-T14 / OWNER ITEM 12 / R6 β A PROVIDER THE PRODUCT DOES NOT OFFER HAS NO DOOR.
|
| 63 |
+
|
| 64 |
+
The owner pasted the failure this replaces: clicking Connect on the Google card answered
|
| 65 |
+
`503 oauth_unavailable` as raw JSON. R6's fix is not a nicer error β it is that the flow is
|
| 66 |
+
not offered at all, so the honest status is the one for a URL that does not exist. `404`
|
| 67 |
+
rather than `503`, deliberately: a 503 says *"come back later"* about a door that is not
|
| 68 |
+
coming back until somebody pays for CASA verification (D-45, ~$540β1,800/yr).
|
| 69 |
+
|
| 70 |
+
β Every door in this router goes through it, START included, because the JSON the owner saw
|
| 71 |
+
came from the start route and a guard on one leg is a guard on one leg.
|
| 72 |
+
"""
|
| 73 |
+
if not oauth_connect.offered(provider):
|
| 74 |
+
raise err(404, "unknown_provider", f"{provider!r} is not a connectable provider")
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
@router.get("/{provider}/start")
|
| 78 |
+
def oauth_start(provider: str, request: Request, next: str = "",
|
| 79 |
+
session: Session = Depends(require_session)):
|
| 80 |
+
"""302 to the provider's consent screen (A3 β a navigation, never JSON). `?next=` is the
|
| 81 |
+
RELATIVE path the callback returns the browser to; it rides inside the single-use state,
|
| 82 |
+
sanitised, so the round trip cannot be steered off-origin."""
|
| 83 |
+
_offered_or_404(provider)
|
| 84 |
+
url, problem = oauth_connect.start(provider, session.uname,
|
| 85 |
+
_redirect_uri(request, provider), next_path=next)
|
| 86 |
+
if problem:
|
| 87 |
+
raise err(503 if "not configured" in problem else 404, "oauth_unavailable", problem)
|
| 88 |
+
return RedirectResponse(url, status_code=302)
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
@router.get("/{provider}/callback")
|
| 92 |
+
def oauth_callback(provider: str, request: Request,
|
| 93 |
+
session: Session = Depends(require_session),
|
| 94 |
+
state: str = "", code: str = "", error: str = ""):
|
| 95 |
+
"""The provider's redirect target. Exchanges the code, stores the per-user slot, and sends
|
| 96 |
+
the browser back to the state's return path β connected or not (see module header)."""
|
| 97 |
+
_offered_or_404(provider)
|
| 98 |
+
if error:
|
| 99 |
+
home = "/#/"
|
| 100 |
+
return RedirectResponse(f"{home}?oauthError={error[:80]}", status_code=302)
|
| 101 |
+
email, home, problem = oauth_connect.callback(session.runtime, session.uname, state, code)
|
| 102 |
+
sep = "&" if "?" in home else "?"
|
| 103 |
+
if problem:
|
| 104 |
+
return RedirectResponse(f"{home}{sep}oauthError=connect_failed", status_code=302)
|
| 105 |
+
return RedirectResponse(f"{home}{sep}connected={provider}", status_code=302)
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
@router.post("/{provider}/disconnect")
|
| 109 |
+
def oauth_disconnect(provider: str, session: Session = Depends(require_session)):
|
| 110 |
+
_offered_or_404(provider)
|
| 111 |
+
if oauth_connect.provider_def(provider) is None:
|
| 112 |
+
raise err(404, "unknown_provider", f"{provider!r} is not a connectable provider")
|
| 113 |
+
oauth_connect.disconnect(session.runtime, session.uname, provider)
|
| 114 |
+
return {"disconnected": provider}
|
api/routes_odoo_tables.py
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
api/routes_publish.py
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
api/routes_query.py
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
api/routes_records.py
CHANGED
|
@@ -1,211 +1,211 @@
|
|
| 1 |
-
"""Record detail routes: durable comments, scoped to the caller's book β ON EVERY DATABASE.
|
| 2 |
-
|
| 3 |
-
β WAVE 19 (owner item 12). This file used to be the CUSTOMER record's comment routes with a
|
| 4 |
-
customer-shaped wall bolted to the module import line: `_in_book` asked
|
| 5 |
-
`routes_customers.allowed_pids` whatever surface the browser was on. Opening a PRODUCT record and
|
| 6 |
-
typing a comment therefore asked the customer book about a CRC32 hash of a SKU code, and the
|
| 7 |
-
panel answered "that customer is not in your book" β the owner's report. The dangerous half is
|
| 8 |
-
the one nobody sees: a hash that collides with a real partner id passes the wall, and the comment
|
| 9 |
-
is filed against somebody's customer where the whole team can read it.
|
| 10 |
-
|
| 11 |
-
THE SHAPE NOW: `?scope=` names the database (the same vocabulary `/workspace?scope=` and the
|
| 12 |
-
events route's `scopeKey` already speak), and `_pool_or_refuse` resolves BOTH halves of the wall
|
| 13 |
-
per scope β the GRANT and the ROW SET β by asking that topic's own route, never by re-deriving
|
| 14 |
-
one here:
|
| 15 |
-
|
| 16 |
-
customer / cohort `routes_customers.allowed_pids` behind the `customer_data` grant
|
| 17 |
-
product `routes_products.scoped_pool` behind the `product_data` grant
|
| 18 |
-
ut_<slug> `routes_tables.scoped_pids`, whose `_defn_or_refuse` IS the wall
|
| 19 |
-
(404 unknown / 403 not yours β a user table has no module grant).
|
| 20 |
-
β W33-T03/D-183: `scoped_pIDs`, not `scoped_pOOL` β the pool builds every
|
| 21 |
-
ROW to derive a pid set this module discards, and RAISES 409 on a
|
| 22 |
-
read-through grid past one window, which is why the record drawer painted
|
| 23 |
-
an error page on `ut_odoo_gl_lines`.
|
| 24 |
-
|
| 25 |
-
β THE PATH KEEPS ITS `/customers/` SEGMENT. It is the shipped URL and `verify_api.py`'s E1a
|
| 26 |
-
section pins it; the scope now travels beside it explicitly. A nicer noun is not worth churning
|
| 27 |
-
another session's gate mid-wave β the WALL is the query parameter, not the word.
|
| 28 |
-
|
| 29 |
-
β NO DEFAULT BEYOND THE LEGACY ONE. An absent `scope` means `customer`, which is what every
|
| 30 |
-
shipped client sent and what keeps the old callers byte-identical; an UNRECOGNISED scope is a
|
| 31 |
-
400, never a silent fallback to the customer book (`routes_grid._scope_or_400`'s rule, and for
|
| 32 |
-
the same reason: a typo served as `customer` answers a question nobody asked).
|
| 33 |
-
"""
|
| 34 |
-
from fastapi import APIRouter, Body, Depends, Query
|
| 35 |
-
|
| 36 |
-
from deps import Session, err, require_session
|
| 37 |
-
|
| 38 |
-
router = APIRouter(prefix="/api/v1")
|
| 39 |
-
|
| 40 |
-
#: The customer topic's two names β one book, two surfaces (the Cohort page is the customer table
|
| 41 |
-
#: over hand-curated sets). Mirrors `modules.cohort.LEGACY_SCOPES` / `core.record_comments`.
|
| 42 |
-
_CUSTOMER_SCOPES = ("", "customer", "cohort")
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
def _scope_or_400(raw):
|
| 46 |
-
scope = str(raw or "customer").strip().lower()
|
| 47 |
-
if scope in _CUSTOMER_SCOPES or scope == "product" or scope.startswith("ut_"):
|
| 48 |
-
return "customer" if scope in _CUSTOMER_SCOPES else scope
|
| 49 |
-
raise err(400, "bad_scope",
|
| 50 |
-
"scope must be customer, cohort, product or a ut_ database β refusing to guess")
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
def _pool_or_refuse(session: Session, scope: str):
|
| 54 |
-
"""The pids this session may attach comments to ON THIS DATABASE β grant wall included.
|
| 55 |
-
|
| 56 |
-
Returns **`(pids, unbounded)`** β a 2-tuple on EVERY branch. Raises the topic's own 403/404/503,
|
| 57 |
-
so a caller who may not open the surface never learns anything about the row they asked about.
|
| 58 |
-
|
| 59 |
-
β `unbounded` is TRUE only when the row set could not be ENUMERATED (a read-through grid past
|
| 60 |
-
one window), never when it is merely EMPTY. Those are opposite answers and `scoped_pids` returns
|
| 61 |
-
`frozenset()` for both β see the branch below.
|
| 62 |
-
β THE SHAPE IS A CONTRACT EVEN THOUGH THIS FUNCTION IS PRIVATE, and it has two consumers that
|
| 63 |
-
do not travel together: `_in_book` here, and a NEGATIVE CONTROL in `aios-web/api/verify_scopes.py`
|
| 64 |
-
that REPLACES this function with its own lambda. A gate's test double is a caller
|
| 65 |
-
([[test-double-patched-by-a-name-list]]); when this signature moved, that double kept returning a
|
| 66 |
-
bare frozenset and the section died on `ValueError: too many values to unpack` β no tally, no
|
| 67 |
-
failing name. Change the shape here and that double changes with it.
|
| 68 |
-
"""
|
| 69 |
-
if scope == "product":
|
| 70 |
-
from routes_products import MODULE as PRODUCT_MODULE, scoped_pool
|
| 71 |
-
|
| 72 |
-
session.require(PRODUCT_MODULE)
|
| 73 |
-
pids, _team, _rows, _fields = scoped_pool(session)
|
| 74 |
-
# β `(pids, unbounded)` on EVERY branch. This one returned a bare frozenset for ten minutes
|
| 75 |
-
# after the `ut_` branch grew its second element, and `_in_book`'s unpack would have raised
|
| 76 |
-
# a `TypeError` β a 500 on every product comment β while both other branches worked. A
|
| 77 |
-
# return shape is a contract even when the function is private.
|
| 78 |
-
return (pids, False)
|
| 79 |
-
if scope.startswith("ut_"):
|
| 80 |
-
# No module grant exists for a user table β `_defn_or_refuse` inside `scoped_pids` IS
|
| 81 |
-
# the wall (creator or admin, fail-closed), and it answers 404 before 403 exactly as the
|
| 82 |
-
# rows routes do.
|
| 83 |
-
#
|
| 84 |
-
# ββ W33-T03 / D-183 β `scoped_pids`, NOT `scoped_pool`, AND THAT ONE WORD IS THE BUG.
|
| 85 |
-
# `scoped_pool` builds every ROW to derive a pid set this function then throws away, and on
|
| 86 |
-
# a read-through grid larger than one window it RAISES `409 window_required`. So opening the
|
| 87 |
-
# record drawer on `ut_odoo_gl_lines` (975,137 rows) painted an error page β for a panel
|
| 88 |
-
# that renders comments about ONE row it already has. `scoped_pids` answers the identical
|
| 89 |
-
# question (its docstring: *"the pid set is IDENTICAL, not merely equivalent"*) and takes
|
| 90 |
-
# W31-T20's `limits` OUT-PARAMETER instead of raising, which is the same shape `/workspace`
|
| 91 |
-
# used to become openable on those two grids.
|
| 92 |
-
from routes_tables import scoped_pids
|
| 93 |
-
|
| 94 |
-
limits = []
|
| 95 |
-
pids, _fields, _defn = scoped_pids(session, scope, limits=limits)
|
| 96 |
-
# β AN EMPTY PID SET AND AN UNRESOLVABLE ONE ARE OPPOSITE ANSWERS, and collapsing them is
|
| 97 |
-
# how a fail-closed default becomes a lie. `scoped_pids` returns `frozenset()` BOTH for a
|
| 98 |
-
# database with no rows and for a read-through grid too big to enumerate β it distinguishes
|
| 99 |
-
# them by APPENDING R6's sentence to `limits`. Without this branch the drawer would move
|
| 100 |
-
# from a 409 error page to a 403 "not in your book" on a row the user is looking at, which
|
| 101 |
-
# is the same defect wearing a politer message ([[empty-answer-vs-unfinished-answer]]).
|
| 102 |
-
# β ADMITTING HERE IS NOT A WIDENING, and the ruling is wave 27 / D-72: on a `ut_*` database
|
| 103 |
-
# THE TENANT IS THE UNIT β `scoped_pool` itself carries "no per-row owner filter; the
|
| 104 |
-
# table-level wall is the WHOLE wall". `_defn_or_refuse` has already run inside
|
| 105 |
-
# `scoped_pids` and answered 404/403. The pid set was only ever an existence check.
|
| 106 |
-
return (pids, bool(limits))
|
| 107 |
-
from routes_customers import MODULE as CUSTOMER_MODULE, allowed_pids
|
| 108 |
-
|
| 109 |
-
session.require(CUSTOMER_MODULE)
|
| 110 |
-
return (frozenset(allowed_pids(session)), False)
|
| 111 |
-
|
| 112 |
-
|
| 113 |
-
def _in_book(pid, session, scope):
|
| 114 |
-
pids, unbounded = _pool_or_refuse(session, scope)
|
| 115 |
-
if unbounded:
|
| 116 |
-
# The table wall passed and the row set is larger than this process will enumerate. Said
|
| 117 |
-
# out loud rather than silently admitting: R6's second sentence is that a limit which
|
| 118 |
-
# cannot be removed gets REPORTED, and this is the one place the report has no envelope to
|
| 119 |
-
# ride in.
|
| 120 |
-
print(f"[records] {scope}: pid membership unresolved (read-through beyond one window) β "
|
| 121 |
-
f"admitting on the table wall alone, per D-72")
|
| 122 |
-
return
|
| 123 |
-
if pid not in pids:
|
| 124 |
-
# 403, not 404: the record may exist, but this session may not inspect it.
|
| 125 |
-
raise err(403, "out_of_scope", "that record is not in your book")
|
| 126 |
-
|
| 127 |
-
|
| 128 |
-
def _unavailable():
|
| 129 |
-
return err(
|
| 130 |
-
503,
|
| 131 |
-
"store_unavailable",
|
| 132 |
-
"record comments are temporarily unavailable β no change was saved",
|
| 133 |
-
)
|
| 134 |
-
|
| 135 |
-
|
| 136 |
-
# β WAVE 21 (D-17): the CANONICAL path is /records/{pid}/comments β comments hang off a RECORD
|
| 137 |
-
# in whatever topic `?scope=` names, and the customer-flavoured noun was wave-19 residue (the
|
| 138 |
-
# wall was always the query param). The old path stays as an ALIAS because the shipped client
|
| 139 |
-
# still calls it; verify_api pins the canonical path AND that the alias answers, so removing
|
| 140 |
-
# the alias later is a decision, never an accident.
|
| 141 |
-
@router.get("/records/{pid}/comments")
|
| 142 |
-
@router.get("/customers/{pid}/comments")
|
| 143 |
-
def comments(pid: int, scope: str = Query(default="customer"),
|
| 144 |
-
session: Session = Depends(require_session)):
|
| 145 |
-
from core import record_comments
|
| 146 |
-
|
| 147 |
-
scope = _scope_or_400(scope)
|
| 148 |
-
_in_book(pid, session, scope)
|
| 149 |
-
try:
|
| 150 |
-
rows = record_comments.list_comments(session.runtime, pid, scope=scope)
|
| 151 |
-
except record_comments.CommentsUnavailable:
|
| 152 |
-
raise _unavailable()
|
| 153 |
-
return {"comments": rows}
|
| 154 |
-
|
| 155 |
-
|
| 156 |
-
@router.post("/records/{pid}/comments", status_code=201)
|
| 157 |
-
@router.post("/customers/{pid}/comments", status_code=201)
|
| 158 |
-
def create_comment(
|
| 159 |
-
pid: int,
|
| 160 |
-
body: dict = Body(default=None),
|
| 161 |
-
scope: str = Query(default="customer"),
|
| 162 |
-
session: Session = Depends(require_session),
|
| 163 |
-
):
|
| 164 |
-
from core import record_comments
|
| 165 |
-
|
| 166 |
-
scope = _scope_or_400(scope)
|
| 167 |
-
_in_book(pid, session, scope)
|
| 168 |
-
try:
|
| 169 |
-
comment = record_comments.add_comment(
|
| 170 |
-
session.runtime,
|
| 171 |
-
pid,
|
| 172 |
-
(body or {}).get("body"),
|
| 173 |
-
session.uname,
|
| 174 |
-
session.user.get("name") or session.uname,
|
| 175 |
-
scope=scope,
|
| 176 |
-
)
|
| 177 |
-
except ValueError as exc:
|
| 178 |
-
raise err(400, "bad_comment", str(exc))
|
| 179 |
-
except record_comments.CommentsUnavailable:
|
| 180 |
-
raise _unavailable()
|
| 181 |
-
return {"comment": comment}
|
| 182 |
-
|
| 183 |
-
|
| 184 |
-
@router.delete("/records/{pid}/comments/{comment_id}")
|
| 185 |
-
@router.delete("/customers/{pid}/comments/{comment_id}")
|
| 186 |
-
def remove_comment(
|
| 187 |
-
pid: int,
|
| 188 |
-
comment_id: str,
|
| 189 |
-
scope: str = Query(default="customer"),
|
| 190 |
-
session: Session = Depends(require_session),
|
| 191 |
-
):
|
| 192 |
-
from core import record_comments
|
| 193 |
-
|
| 194 |
-
scope = _scope_or_400(scope)
|
| 195 |
-
_in_book(pid, session, scope)
|
| 196 |
-
try:
|
| 197 |
-
deleted = record_comments.delete_comment(
|
| 198 |
-
session.runtime,
|
| 199 |
-
pid,
|
| 200 |
-
comment_id,
|
| 201 |
-
session.uname,
|
| 202 |
-
admin=session.admin,
|
| 203 |
-
scope=scope,
|
| 204 |
-
)
|
| 205 |
-
except record_comments.CommentForbidden:
|
| 206 |
-
raise err(403, "comment_forbidden", "only the author may delete this comment")
|
| 207 |
-
except record_comments.CommentsUnavailable:
|
| 208 |
-
raise _unavailable()
|
| 209 |
-
if not deleted:
|
| 210 |
-
raise err(404, "comment_not_found", "that comment no longer exists")
|
| 211 |
-
return {"ok": True, "id": comment_id}
|
|
|
|
| 1 |
+
"""Record detail routes: durable comments, scoped to the caller's book β ON EVERY DATABASE.
|
| 2 |
+
|
| 3 |
+
β WAVE 19 (owner item 12). This file used to be the CUSTOMER record's comment routes with a
|
| 4 |
+
customer-shaped wall bolted to the module import line: `_in_book` asked
|
| 5 |
+
`routes_customers.allowed_pids` whatever surface the browser was on. Opening a PRODUCT record and
|
| 6 |
+
typing a comment therefore asked the customer book about a CRC32 hash of a SKU code, and the
|
| 7 |
+
panel answered "that customer is not in your book" β the owner's report. The dangerous half is
|
| 8 |
+
the one nobody sees: a hash that collides with a real partner id passes the wall, and the comment
|
| 9 |
+
is filed against somebody's customer where the whole team can read it.
|
| 10 |
+
|
| 11 |
+
THE SHAPE NOW: `?scope=` names the database (the same vocabulary `/workspace?scope=` and the
|
| 12 |
+
events route's `scopeKey` already speak), and `_pool_or_refuse` resolves BOTH halves of the wall
|
| 13 |
+
per scope β the GRANT and the ROW SET β by asking that topic's own route, never by re-deriving
|
| 14 |
+
one here:
|
| 15 |
+
|
| 16 |
+
customer / cohort `routes_customers.allowed_pids` behind the `customer_data` grant
|
| 17 |
+
product `routes_products.scoped_pool` behind the `product_data` grant
|
| 18 |
+
ut_<slug> `routes_tables.scoped_pids`, whose `_defn_or_refuse` IS the wall
|
| 19 |
+
(404 unknown / 403 not yours β a user table has no module grant).
|
| 20 |
+
β W33-T03/D-183: `scoped_pIDs`, not `scoped_pOOL` β the pool builds every
|
| 21 |
+
ROW to derive a pid set this module discards, and RAISES 409 on a
|
| 22 |
+
read-through grid past one window, which is why the record drawer painted
|
| 23 |
+
an error page on `ut_odoo_gl_lines`.
|
| 24 |
+
|
| 25 |
+
β THE PATH KEEPS ITS `/customers/` SEGMENT. It is the shipped URL and `verify_api.py`'s E1a
|
| 26 |
+
section pins it; the scope now travels beside it explicitly. A nicer noun is not worth churning
|
| 27 |
+
another session's gate mid-wave β the WALL is the query parameter, not the word.
|
| 28 |
+
|
| 29 |
+
β NO DEFAULT BEYOND THE LEGACY ONE. An absent `scope` means `customer`, which is what every
|
| 30 |
+
shipped client sent and what keeps the old callers byte-identical; an UNRECOGNISED scope is a
|
| 31 |
+
400, never a silent fallback to the customer book (`routes_grid._scope_or_400`'s rule, and for
|
| 32 |
+
the same reason: a typo served as `customer` answers a question nobody asked).
|
| 33 |
+
"""
|
| 34 |
+
from fastapi import APIRouter, Body, Depends, Query
|
| 35 |
+
|
| 36 |
+
from deps import Session, err, require_session
|
| 37 |
+
|
| 38 |
+
router = APIRouter(prefix="/api/v1")
|
| 39 |
+
|
| 40 |
+
#: The customer topic's two names β one book, two surfaces (the Cohort page is the customer table
|
| 41 |
+
#: over hand-curated sets). Mirrors `modules.cohort.LEGACY_SCOPES` / `core.record_comments`.
|
| 42 |
+
_CUSTOMER_SCOPES = ("", "customer", "cohort")
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def _scope_or_400(raw):
|
| 46 |
+
scope = str(raw or "customer").strip().lower()
|
| 47 |
+
if scope in _CUSTOMER_SCOPES or scope == "product" or scope.startswith("ut_"):
|
| 48 |
+
return "customer" if scope in _CUSTOMER_SCOPES else scope
|
| 49 |
+
raise err(400, "bad_scope",
|
| 50 |
+
"scope must be customer, cohort, product or a ut_ database β refusing to guess")
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
def _pool_or_refuse(session: Session, scope: str):
|
| 54 |
+
"""The pids this session may attach comments to ON THIS DATABASE β grant wall included.
|
| 55 |
+
|
| 56 |
+
Returns **`(pids, unbounded)`** β a 2-tuple on EVERY branch. Raises the topic's own 403/404/503,
|
| 57 |
+
so a caller who may not open the surface never learns anything about the row they asked about.
|
| 58 |
+
|
| 59 |
+
β `unbounded` is TRUE only when the row set could not be ENUMERATED (a read-through grid past
|
| 60 |
+
one window), never when it is merely EMPTY. Those are opposite answers and `scoped_pids` returns
|
| 61 |
+
`frozenset()` for both β see the branch below.
|
| 62 |
+
β THE SHAPE IS A CONTRACT EVEN THOUGH THIS FUNCTION IS PRIVATE, and it has two consumers that
|
| 63 |
+
do not travel together: `_in_book` here, and a NEGATIVE CONTROL in `aios-web/api/verify_scopes.py`
|
| 64 |
+
that REPLACES this function with its own lambda. A gate's test double is a caller
|
| 65 |
+
([[test-double-patched-by-a-name-list]]); when this signature moved, that double kept returning a
|
| 66 |
+
bare frozenset and the section died on `ValueError: too many values to unpack` β no tally, no
|
| 67 |
+
failing name. Change the shape here and that double changes with it.
|
| 68 |
+
"""
|
| 69 |
+
if scope == "product":
|
| 70 |
+
from routes_products import MODULE as PRODUCT_MODULE, scoped_pool
|
| 71 |
+
|
| 72 |
+
session.require(PRODUCT_MODULE)
|
| 73 |
+
pids, _team, _rows, _fields = scoped_pool(session)
|
| 74 |
+
# β `(pids, unbounded)` on EVERY branch. This one returned a bare frozenset for ten minutes
|
| 75 |
+
# after the `ut_` branch grew its second element, and `_in_book`'s unpack would have raised
|
| 76 |
+
# a `TypeError` β a 500 on every product comment β while both other branches worked. A
|
| 77 |
+
# return shape is a contract even when the function is private.
|
| 78 |
+
return (pids, False)
|
| 79 |
+
if scope.startswith("ut_"):
|
| 80 |
+
# No module grant exists for a user table β `_defn_or_refuse` inside `scoped_pids` IS
|
| 81 |
+
# the wall (creator or admin, fail-closed), and it answers 404 before 403 exactly as the
|
| 82 |
+
# rows routes do.
|
| 83 |
+
#
|
| 84 |
+
# ββ W33-T03 / D-183 β `scoped_pids`, NOT `scoped_pool`, AND THAT ONE WORD IS THE BUG.
|
| 85 |
+
# `scoped_pool` builds every ROW to derive a pid set this function then throws away, and on
|
| 86 |
+
# a read-through grid larger than one window it RAISES `409 window_required`. So opening the
|
| 87 |
+
# record drawer on `ut_odoo_gl_lines` (975,137 rows) painted an error page β for a panel
|
| 88 |
+
# that renders comments about ONE row it already has. `scoped_pids` answers the identical
|
| 89 |
+
# question (its docstring: *"the pid set is IDENTICAL, not merely equivalent"*) and takes
|
| 90 |
+
# W31-T20's `limits` OUT-PARAMETER instead of raising, which is the same shape `/workspace`
|
| 91 |
+
# used to become openable on those two grids.
|
| 92 |
+
from routes_tables import scoped_pids
|
| 93 |
+
|
| 94 |
+
limits = []
|
| 95 |
+
pids, _fields, _defn = scoped_pids(session, scope, limits=limits)
|
| 96 |
+
# β AN EMPTY PID SET AND AN UNRESOLVABLE ONE ARE OPPOSITE ANSWERS, and collapsing them is
|
| 97 |
+
# how a fail-closed default becomes a lie. `scoped_pids` returns `frozenset()` BOTH for a
|
| 98 |
+
# database with no rows and for a read-through grid too big to enumerate β it distinguishes
|
| 99 |
+
# them by APPENDING R6's sentence to `limits`. Without this branch the drawer would move
|
| 100 |
+
# from a 409 error page to a 403 "not in your book" on a row the user is looking at, which
|
| 101 |
+
# is the same defect wearing a politer message ([[empty-answer-vs-unfinished-answer]]).
|
| 102 |
+
# β ADMITTING HERE IS NOT A WIDENING, and the ruling is wave 27 / D-72: on a `ut_*` database
|
| 103 |
+
# THE TENANT IS THE UNIT β `scoped_pool` itself carries "no per-row owner filter; the
|
| 104 |
+
# table-level wall is the WHOLE wall". `_defn_or_refuse` has already run inside
|
| 105 |
+
# `scoped_pids` and answered 404/403. The pid set was only ever an existence check.
|
| 106 |
+
return (pids, bool(limits))
|
| 107 |
+
from routes_customers import MODULE as CUSTOMER_MODULE, allowed_pids
|
| 108 |
+
|
| 109 |
+
session.require(CUSTOMER_MODULE)
|
| 110 |
+
return (frozenset(allowed_pids(session)), False)
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
def _in_book(pid, session, scope):
|
| 114 |
+
pids, unbounded = _pool_or_refuse(session, scope)
|
| 115 |
+
if unbounded:
|
| 116 |
+
# The table wall passed and the row set is larger than this process will enumerate. Said
|
| 117 |
+
# out loud rather than silently admitting: R6's second sentence is that a limit which
|
| 118 |
+
# cannot be removed gets REPORTED, and this is the one place the report has no envelope to
|
| 119 |
+
# ride in.
|
| 120 |
+
print(f"[records] {scope}: pid membership unresolved (read-through beyond one window) β "
|
| 121 |
+
f"admitting on the table wall alone, per D-72")
|
| 122 |
+
return
|
| 123 |
+
if pid not in pids:
|
| 124 |
+
# 403, not 404: the record may exist, but this session may not inspect it.
|
| 125 |
+
raise err(403, "out_of_scope", "that record is not in your book")
|
| 126 |
+
|
| 127 |
+
|
| 128 |
+
def _unavailable():
|
| 129 |
+
return err(
|
| 130 |
+
503,
|
| 131 |
+
"store_unavailable",
|
| 132 |
+
"record comments are temporarily unavailable β no change was saved",
|
| 133 |
+
)
|
| 134 |
+
|
| 135 |
+
|
| 136 |
+
# β WAVE 21 (D-17): the CANONICAL path is /records/{pid}/comments β comments hang off a RECORD
|
| 137 |
+
# in whatever topic `?scope=` names, and the customer-flavoured noun was wave-19 residue (the
|
| 138 |
+
# wall was always the query param). The old path stays as an ALIAS because the shipped client
|
| 139 |
+
# still calls it; verify_api pins the canonical path AND that the alias answers, so removing
|
| 140 |
+
# the alias later is a decision, never an accident.
|
| 141 |
+
@router.get("/records/{pid}/comments")
|
| 142 |
+
@router.get("/customers/{pid}/comments")
|
| 143 |
+
def comments(pid: int, scope: str = Query(default="customer"),
|
| 144 |
+
session: Session = Depends(require_session)):
|
| 145 |
+
from core import record_comments
|
| 146 |
+
|
| 147 |
+
scope = _scope_or_400(scope)
|
| 148 |
+
_in_book(pid, session, scope)
|
| 149 |
+
try:
|
| 150 |
+
rows = record_comments.list_comments(session.runtime, pid, scope=scope)
|
| 151 |
+
except record_comments.CommentsUnavailable:
|
| 152 |
+
raise _unavailable()
|
| 153 |
+
return {"comments": rows}
|
| 154 |
+
|
| 155 |
+
|
| 156 |
+
@router.post("/records/{pid}/comments", status_code=201)
|
| 157 |
+
@router.post("/customers/{pid}/comments", status_code=201)
|
| 158 |
+
def create_comment(
|
| 159 |
+
pid: int,
|
| 160 |
+
body: dict = Body(default=None),
|
| 161 |
+
scope: str = Query(default="customer"),
|
| 162 |
+
session: Session = Depends(require_session),
|
| 163 |
+
):
|
| 164 |
+
from core import record_comments
|
| 165 |
+
|
| 166 |
+
scope = _scope_or_400(scope)
|
| 167 |
+
_in_book(pid, session, scope)
|
| 168 |
+
try:
|
| 169 |
+
comment = record_comments.add_comment(
|
| 170 |
+
session.runtime,
|
| 171 |
+
pid,
|
| 172 |
+
(body or {}).get("body"),
|
| 173 |
+
session.uname,
|
| 174 |
+
session.user.get("name") or session.uname,
|
| 175 |
+
scope=scope,
|
| 176 |
+
)
|
| 177 |
+
except ValueError as exc:
|
| 178 |
+
raise err(400, "bad_comment", str(exc))
|
| 179 |
+
except record_comments.CommentsUnavailable:
|
| 180 |
+
raise _unavailable()
|
| 181 |
+
return {"comment": comment}
|
| 182 |
+
|
| 183 |
+
|
| 184 |
+
@router.delete("/records/{pid}/comments/{comment_id}")
|
| 185 |
+
@router.delete("/customers/{pid}/comments/{comment_id}")
|
| 186 |
+
def remove_comment(
|
| 187 |
+
pid: int,
|
| 188 |
+
comment_id: str,
|
| 189 |
+
scope: str = Query(default="customer"),
|
| 190 |
+
session: Session = Depends(require_session),
|
| 191 |
+
):
|
| 192 |
+
from core import record_comments
|
| 193 |
+
|
| 194 |
+
scope = _scope_or_400(scope)
|
| 195 |
+
_in_book(pid, session, scope)
|
| 196 |
+
try:
|
| 197 |
+
deleted = record_comments.delete_comment(
|
| 198 |
+
session.runtime,
|
| 199 |
+
pid,
|
| 200 |
+
comment_id,
|
| 201 |
+
session.uname,
|
| 202 |
+
admin=session.admin,
|
| 203 |
+
scope=scope,
|
| 204 |
+
)
|
| 205 |
+
except record_comments.CommentForbidden:
|
| 206 |
+
raise err(403, "comment_forbidden", "only the author may delete this comment")
|
| 207 |
+
except record_comments.CommentsUnavailable:
|
| 208 |
+
raise _unavailable()
|
| 209 |
+
if not deleted:
|
| 210 |
+
raise err(404, "comment_not_found", "that comment no longer exists")
|
| 211 |
+
return {"ok": True, "id": comment_id}
|
api/routes_script_views.py
ADDED
|
@@ -0,0 +1,329 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""routes_script_views.py β CONTRACT C3: a database View that is a PYTHON SCRIPT (R3 / R5 / R10).
|
| 2 |
+
|
| 3 |
+
Owner item 6, verbatim (2026-08-18): *"Add code script as an interface (database View) so a user
|
| 4 |
+
can build whatever they want through the Agent chat interface. be able to create any dashboard
|
| 5 |
+
they want. User should have the ability to see the code AND the dashboard output of course⦠Limit
|
| 6 |
+
the code script View per database⦠Any agent can add into more AI script, so we can see different
|
| 7 |
+
versions or different things the AI code for us."*
|
| 8 |
+
|
| 9 |
+
GET /api/v1/script-views?database=K the views bound to ONE database
|
| 10 |
+
POST /api/v1/script-views create one {database, name?, source}
|
| 11 |
+
GET /api/v1/script-views/{id} one view, its source and its history
|
| 12 |
+
PUT /api/v1/script-views/{id} a NEW VERSION of the source
|
| 13 |
+
DELETE /api/v1/script-views/{id} drop it
|
| 14 |
+
POST /api/v1/script-views/{id}/run run it -> {ok, spec | error, stdout, ms}
|
| 15 |
+
|
| 16 |
+
ββ **R3 IS "SCOPED, NOT CAPPED", AND THE TWO HALVES POINT OPPOSITE WAYS.** *Scoped*: a view may
|
| 17 |
+
read ONLY the database it lives in, and a script that names another database is REFUSED with a
|
| 18 |
+
message naming both. *Not capped*: there is **no limit on how many script views a database may
|
| 19 |
+
carry**, because that is how an agent offers three attempts and the owner picks one. So nothing
|
| 20 |
+
below counts views. What IS bounded is what makes them big β one source is capped, and one view's
|
| 21 |
+
edit history is capped and REPORTS what it dropped.
|
| 22 |
+
|
| 23 |
+
β **THE RUN IS THE CALLER'S, NEVER THE AUTHOR'S (R5).** `script_sandbox.run_view` is handed
|
| 24 |
+
`session.user`, so a script written by an administrator and opened by a scoped analyst reads the
|
| 25 |
+
ANALYST's rows. The author decides what the code does; the reader decides what it can see.
|
| 26 |
+
β And the reverse case is safe rather than lucky: a narrowly-scoped author cannot write a script
|
| 27 |
+
that exfiltrates anything, because the only thing a script can return is a render spec drawn on
|
| 28 |
+
the screen of the person who ran it. There is no network, no file and no second reader.
|
| 29 |
+
|
| 30 |
+
β **`run` IS `def`, NOT `async def`.** It waits on a subprocess for up to ten seconds; as a
|
| 31 |
+
coroutine that would block the event loop for every other request in the container. FastAPI runs a
|
| 32 |
+
plain `def` in the threadpool, which is what makes one slow script one slow REQUEST.
|
| 33 |
+
"""
|
| 34 |
+
import threading
|
| 35 |
+
from datetime import datetime, timezone
|
| 36 |
+
|
| 37 |
+
from fastapi import APIRouter, Body, Depends
|
| 38 |
+
|
| 39 |
+
from deps import Session, err, require_session
|
| 40 |
+
|
| 41 |
+
router = APIRouter(prefix="/api/v1")
|
| 42 |
+
|
| 43 |
+
#: The tenant's script views: `{id: record}`. Per tenant, so it rides `runtime.store_key`.
|
| 44 |
+
VIEWS_KEY = "script_views"
|
| 45 |
+
|
| 46 |
+
MAX_NAME = 80
|
| 47 |
+
MAX_SOURCE_BYTES = 128 * 1024
|
| 48 |
+
|
| 49 |
+
#: Edit history per view. β NOT a cap on the NUMBER of views (R3 forbids that) β a cap on how far
|
| 50 |
+
#: back ONE view's source is kept. Past this the oldest go and `trimmed` counts them, so a reader
|
| 51 |
+
#: can see the history is partial instead of concluding the view was only ever saved twice.
|
| 52 |
+
MAX_HISTORY = 40
|
| 53 |
+
|
| 54 |
+
#: β HOW MANY SCRIPTS MAY BE RUNNING IN THIS CONTAINER AT ONCE, and it is a REPORTED refusal
|
| 55 |
+
#: rather than a queue. Each run is a real subprocess with a ten-second wall clock; without this,
|
| 56 |
+
#: holding down refresh forks until the box gives up, and the tenant's ONE FastAPI process is what
|
| 57 |
+
#: gives up. A 429 that says so is honest; an unbounded fork is not.
|
| 58 |
+
MAX_CONCURRENT_RUNS = 4
|
| 59 |
+
_RUN_SLOTS = threading.BoundedSemaphore(MAX_CONCURRENT_RUNS)
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
def _now():
|
| 63 |
+
return datetime.now(timezone.utc).isoformat(timespec="seconds")
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def _all(rt):
|
| 67 |
+
"""`{id: record}` for one tenant. `{}` on any failure β an unreadable bucket must degrade to
|
| 68 |
+
"this database has no script views", never to a 500 on the view rail."""
|
| 69 |
+
try:
|
| 70 |
+
found = rt.get(VIEWS_KEY) or {}
|
| 71 |
+
except Exception: # noqa: BLE001
|
| 72 |
+
return {}
|
| 73 |
+
return found if isinstance(found, dict) else {}
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def _database_ok(session, database):
|
| 77 |
+
"""Does this database EXIST, and may this caller read it? Answered by C1, never by a list.
|
| 78 |
+
|
| 79 |
+
β `perm_scope.may_read` ALONE IS NOT ENOUGH and the reason is easy to miss: it answers True
|
| 80 |
+
for an ADMIN on any key at all, including one no database answers to. So a create validated
|
| 81 |
+
with `may_read` would let an administrator bind a view to a typo and leave an orphan nothing
|
| 82 |
+
can ever run. `scoped_fields` is the cheap half of C1 (a `ut_*` definition, no rows) and it
|
| 83 |
+
RAISES `UnknownTable`, which is exactly the question being asked.
|
| 84 |
+
"""
|
| 85 |
+
import core.perm_scope as perm_scope
|
| 86 |
+
try:
|
| 87 |
+
perm_scope.scoped_fields(session.user, database, st=session.runtime)
|
| 88 |
+
except perm_scope.UnknownTable:
|
| 89 |
+
raise err(404, "no_database", f"there is no database '{database}' in this workspace")
|
| 90 |
+
except perm_scope.Denied:
|
| 91 |
+
raise err(403, "forbidden", f"your account may not read '{database}'")
|
| 92 |
+
except perm_scope.Unresolvable as exc:
|
| 93 |
+
# The database is real and cannot be served under this call's constraints. Standing rule
|
| 94 |
+
# 1's second sentence: report the cause and the recommendation, never a bare refusal.
|
| 95 |
+
raise err(409, "unresolvable", str(exc)) from None
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
def _clean_source(raw):
|
| 99 |
+
source = str(raw or "")
|
| 100 |
+
if not source.strip():
|
| 101 |
+
raise err(400, "no_source", "a script view needs some code")
|
| 102 |
+
if len(source.encode("utf-8", "replace")) > MAX_SOURCE_BYTES:
|
| 103 |
+
raise err(413, "source_too_long",
|
| 104 |
+
f"a script view is at most {MAX_SOURCE_BYTES // 1024} KB of code")
|
| 105 |
+
return source
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
def _row(rec, *, source=False):
|
| 109 |
+
"""One view as the list door reports it. β NO SOURCE unless asked: the rail lists names."""
|
| 110 |
+
out = {"id": rec.get("id") or "", "database": rec.get("database") or "",
|
| 111 |
+
"name": rec.get("name") or "", "author": rec.get("author") or "",
|
| 112 |
+
"version": int(rec.get("version") or 1),
|
| 113 |
+
"created": rec.get("created") or "", "updated": rec.get("updated") or "",
|
| 114 |
+
"versions": len(rec.get("history") or []) + 1,
|
| 115 |
+
"trimmed": int(rec.get("trimmed") or 0)}
|
| 116 |
+
if source:
|
| 117 |
+
out["source"] = rec.get("source") or ""
|
| 118 |
+
out["history"] = [{"version": int(h.get("version") or 0), "author": h.get("author") or "",
|
| 119 |
+
"created": h.get("created") or "",
|
| 120 |
+
"bytes": len(str(h.get("source") or "").encode("utf-8", "replace"))}
|
| 121 |
+
for h in reversed(rec.get("history") or []) if isinstance(h, dict)]
|
| 122 |
+
return out
|
| 123 |
+
|
| 124 |
+
|
| 125 |
+
def _limits():
|
| 126 |
+
return {"maxSourceBytes": MAX_SOURCE_BYTES, "maxName": MAX_NAME,
|
| 127 |
+
"maxHistory": MAX_HISTORY, "maxConcurrentRuns": MAX_CONCURRENT_RUNS,
|
| 128 |
+
# β SAID OUT LOUD, because R3's "not capped" half is the one a reader assumes wrong.
|
| 129 |
+
"maxViewsPerDatabase": None}
|
| 130 |
+
|
| 131 |
+
|
| 132 |
+
def _put(session, view_id, mutate):
|
| 133 |
+
"""Read-modify-write ONE view, synchronously β the client re-reads the rail immediately."""
|
| 134 |
+
def _set(cur):
|
| 135 |
+
cur = dict(cur or {})
|
| 136 |
+
nxt = mutate(cur.get(view_id) if isinstance(cur.get(view_id), dict) else None)
|
| 137 |
+
if nxt is None:
|
| 138 |
+
cur.pop(view_id, None)
|
| 139 |
+
else:
|
| 140 |
+
cur[view_id] = nxt
|
| 141 |
+
return cur
|
| 142 |
+
|
| 143 |
+
session.runtime.update(VIEWS_KEY, _set, flush="sync")
|
| 144 |
+
|
| 145 |
+
|
| 146 |
+
def _mine_or_admin(session, rec):
|
| 147 |
+
"""Who may EDIT or DELETE a view: its author, or an administrator.
|
| 148 |
+
|
| 149 |
+
β RUNNING IS A DIFFERENT QUESTION and deliberately wider β anybody who may read the database
|
| 150 |
+
may run any view on it, under their OWN scope. That is the whole of "so we can see different
|
| 151 |
+
versions or different things the AI code for us": a colleague's attempt is worth nothing if
|
| 152 |
+
only its author can open it.
|
| 153 |
+
"""
|
| 154 |
+
if session.admin or str(rec.get("author") or "") == session.uname:
|
| 155 |
+
return
|
| 156 |
+
raise err(403, "forbidden", "only the author or an administrator can change this script view")
|
| 157 |
+
|
| 158 |
+
|
| 159 |
+
# ββ the routes ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 160 |
+
@router.get("/script-views")
|
| 161 |
+
def list_script_views(database: str = "", session: Session = Depends(require_session)):
|
| 162 |
+
"""Every script view bound to ONE database, newest first. `database` is required."""
|
| 163 |
+
key = str(database or "").strip()
|
| 164 |
+
if not key:
|
| 165 |
+
raise err(400, "no_database", "name the database whose script views you want")
|
| 166 |
+
_database_ok(session, key)
|
| 167 |
+
rows = [_row(rec) for rec in _all(session.runtime).values()
|
| 168 |
+
if isinstance(rec, dict) and str(rec.get("database") or "") == key]
|
| 169 |
+
rows.sort(key=lambda r: (r["created"], r["id"]), reverse=True)
|
| 170 |
+
return {"database": key, "views": rows, "limits": _limits()}
|
| 171 |
+
|
| 172 |
+
|
| 173 |
+
@router.post("/script-views")
|
| 174 |
+
def create_script_view(body: dict = Body(default=None),
|
| 175 |
+
session: Session = Depends(require_session)):
|
| 176 |
+
"""Create one. β THE SOURCE IS CHECKED BEFORE IT IS STORED, not first at run time.
|
| 177 |
+
|
| 178 |
+
A script view that cannot be run is a broken feature the reader discovers by pressing a button,
|
| 179 |
+
and the agent that wrote it is long gone by then. `check_source` is pure and costs no process,
|
| 180 |
+
so the refusal arrives while the author still has the code in front of them.
|
| 181 |
+
"""
|
| 182 |
+
import secrets # noqa: PLC0415
|
| 183 |
+
import core.script_sandbox as sandbox # noqa: PLC0415
|
| 184 |
+
|
| 185 |
+
body = body if isinstance(body, dict) else {}
|
| 186 |
+
database = str(body.get("database") or "").strip()
|
| 187 |
+
if not database:
|
| 188 |
+
raise err(400, "no_database", "a script view is bound to one database")
|
| 189 |
+
_database_ok(session, database)
|
| 190 |
+
source = _clean_source(body.get("source"))
|
| 191 |
+
refusal = sandbox.check_source(source)
|
| 192 |
+
if refusal is not None:
|
| 193 |
+
raise err(400, refusal.code, refusal.message)
|
| 194 |
+
|
| 195 |
+
view_id = "sv_" + secrets.token_urlsafe(9)
|
| 196 |
+
rec = {"id": view_id, "database": database,
|
| 197 |
+
"name": " ".join(str(body.get("name") or "Script view").split())[:MAX_NAME],
|
| 198 |
+
"source": source, "author": session.uname, "version": 1,
|
| 199 |
+
"created": _now(), "updated": _now(), "history": [], "trimmed": 0}
|
| 200 |
+
_put(session, view_id, lambda _prior: rec)
|
| 201 |
+
fresh = _all(session.runtime).get(view_id)
|
| 202 |
+
if not isinstance(fresh, dict):
|
| 203 |
+
# The store took the write and did not record it. A 200 here would tell the author their
|
| 204 |
+
# script was saved when it was not.
|
| 205 |
+
raise err(503, "store_unavailable", "the script view was NOT created")
|
| 206 |
+
return {"view": _row(fresh, source=True), "limits": _limits()}
|
| 207 |
+
|
| 208 |
+
|
| 209 |
+
@router.get("/script-views/{view_id}")
|
| 210 |
+
def get_script_view(view_id: str, session: Session = Depends(require_session)):
|
| 211 |
+
"""One view WITH its source and its edit history. Owner item 6's *"see the code"* half."""
|
| 212 |
+
rec = _all(session.runtime).get(str(view_id))
|
| 213 |
+
if not isinstance(rec, dict):
|
| 214 |
+
raise err(404, "no_view", "there is no script view with that id")
|
| 215 |
+
_database_ok(session, str(rec.get("database") or ""))
|
| 216 |
+
return {"view": _row(rec, source=True), "limits": _limits()}
|
| 217 |
+
|
| 218 |
+
|
| 219 |
+
@router.put("/script-views/{view_id}")
|
| 220 |
+
def update_script_view(view_id: str, body: dict = Body(default=None),
|
| 221 |
+
session: Session = Depends(require_session)):
|
| 222 |
+
"""A NEW VERSION of one view's source. The prior source is KEPT, never replaced in place."""
|
| 223 |
+
import core.script_sandbox as sandbox # noqa: PLC0415
|
| 224 |
+
|
| 225 |
+
view_id = str(view_id)
|
| 226 |
+
rec = _all(session.runtime).get(view_id)
|
| 227 |
+
if not isinstance(rec, dict):
|
| 228 |
+
raise err(404, "no_view", "there is no script view with that id")
|
| 229 |
+
_mine_or_admin(session, rec)
|
| 230 |
+
body = body if isinstance(body, dict) else {}
|
| 231 |
+
source = _clean_source(body.get("source"))
|
| 232 |
+
refusal = sandbox.check_source(source)
|
| 233 |
+
if refusal is not None:
|
| 234 |
+
raise err(400, refusal.code, refusal.message)
|
| 235 |
+
|
| 236 |
+
def _mutate(prior):
|
| 237 |
+
prior = dict(prior or rec)
|
| 238 |
+
history = list(prior.get("history") or [])
|
| 239 |
+
history.append({"version": int(prior.get("version") or 1),
|
| 240 |
+
"source": prior.get("source") or "",
|
| 241 |
+
"author": prior.get("author") or "", "created": prior.get("updated") or ""})
|
| 242 |
+
dropped = max(0, len(history) - MAX_HISTORY)
|
| 243 |
+
prior["history"] = history[dropped:] if dropped else history
|
| 244 |
+
prior["trimmed"] = int(prior.get("trimmed") or 0) + dropped
|
| 245 |
+
prior["source"] = source
|
| 246 |
+
prior["version"] = int(prior.get("version") or 1) + 1
|
| 247 |
+
prior["updated"] = _now()
|
| 248 |
+
if body.get("name"):
|
| 249 |
+
prior["name"] = " ".join(str(body["name"]).split())[:MAX_NAME]
|
| 250 |
+
return prior
|
| 251 |
+
|
| 252 |
+
_put(session, view_id, _mutate)
|
| 253 |
+
fresh = _all(session.runtime).get(view_id)
|
| 254 |
+
if not isinstance(fresh, dict):
|
| 255 |
+
raise err(503, "store_unavailable", "the new version was NOT saved")
|
| 256 |
+
return {"view": _row(fresh, source=True), "limits": _limits()}
|
| 257 |
+
|
| 258 |
+
|
| 259 |
+
@router.delete("/script-views/{view_id}")
|
| 260 |
+
def delete_script_view(view_id: str, session: Session = Depends(require_session)):
|
| 261 |
+
view_id = str(view_id)
|
| 262 |
+
rec = _all(session.runtime).get(view_id)
|
| 263 |
+
if not isinstance(rec, dict):
|
| 264 |
+
raise err(404, "no_view", "there is no script view with that id")
|
| 265 |
+
_mine_or_admin(session, rec)
|
| 266 |
+
_put(session, view_id, lambda _prior: None)
|
| 267 |
+
return {"deleted": view_id}
|
| 268 |
+
|
| 269 |
+
|
| 270 |
+
@router.post("/script-views/{view_id}/run")
|
| 271 |
+
def run_script_view(view_id: str, body: dict = Body(default=None),
|
| 272 |
+
session: Session = Depends(require_session)):
|
| 273 |
+
"""CONTRACT C3's run door: `{ok, spec | error, stdout, ms}`.
|
| 274 |
+
|
| 275 |
+
β `spec` IS A DESCRIPTION THE CLIENT DRAWS. It is never HTML and never text the browser
|
| 276 |
+
executes β the sandbox refuses a spec carrying an `html`, `script`, `src`, `href` or `on*` key
|
| 277 |
+
before this function ever sees it, so a renderer cannot be talked into running something by a
|
| 278 |
+
script that was itself perfectly well behaved.
|
| 279 |
+
|
| 280 |
+
β AND IT IS PLAIN `def`, NOT `async def` β see this module's header. A ten-second subprocess
|
| 281 |
+
wait on the event loop is a ten-second outage for the whole container.
|
| 282 |
+
"""
|
| 283 |
+
import core.script_sandbox as sandbox # noqa: PLC0415
|
| 284 |
+
|
| 285 |
+
rec = _all(session.runtime).get(str(view_id))
|
| 286 |
+
if not isinstance(rec, dict):
|
| 287 |
+
raise err(404, "no_view", "there is no script view with that id")
|
| 288 |
+
database = str(rec.get("database") or "")
|
| 289 |
+
# A DRAFT run: the editor sends unsaved code so the author can try it before committing to it.
|
| 290 |
+
# It is checked exactly as a stored one is, because "unsaved" is not a permission.
|
| 291 |
+
draft = (body or {}).get("source") if isinstance(body, dict) else None
|
| 292 |
+
source = _clean_source(draft) if draft else str(rec.get("source") or "")
|
| 293 |
+
# β NO `_database_ok` CALL HERE. `run_view` asks C1 the identical question a line later, and
|
| 294 |
+
# asking twice builds a registry topic's pool twice. The three refusals are translated below
|
| 295 |
+
# instead, which is the same wall reached through the same door.
|
| 296 |
+
|
| 297 |
+
if not _RUN_SLOTS.acquire(blocking=False):
|
| 298 |
+
raise err(429, "busy",
|
| 299 |
+
f"{MAX_CONCURRENT_RUNS} script views are already running on this server. "
|
| 300 |
+
f"Try again in a moment")
|
| 301 |
+
try:
|
| 302 |
+
out = sandbox.run_view(session.user, database, source, st=session.runtime)
|
| 303 |
+
finally:
|
| 304 |
+
_RUN_SLOTS.release()
|
| 305 |
+
|
| 306 |
+
# β C1'S THREE REFUSALS ARE HTTP STATUSES, NOT `ok:false`. "You may not read this database"
|
| 307 |
+
# answered 200 would be a permission decision the client has to go looking for, and every
|
| 308 |
+
# other door in this app answers 403 for it. Everything BELOW this line is a well-formed
|
| 309 |
+
# request whose ANSWER is that the script did not produce a view β that is a 200 with
|
| 310 |
+
# `ok:false`, the shape `routes_web_agent.test` already uses for the same reason.
|
| 311 |
+
if out.get("code") == "unknown_table":
|
| 312 |
+
raise err(404, "no_database", out.get("message") or f"there is no database '{database}'")
|
| 313 |
+
if out.get("code") == "denied":
|
| 314 |
+
raise err(403, "forbidden", out.get("message") or "your account may not read that database")
|
| 315 |
+
if out.get("code") == "unresolvable":
|
| 316 |
+
refusal = err(409, "unresolvable", out.get("message") or "these rows cannot be served")
|
| 317 |
+
refusal.detail["error"]["limit"] = out.get("limit") or {}
|
| 318 |
+
raise refusal
|
| 319 |
+
|
| 320 |
+
answer = {"ok": bool(out.get("ok")), "spec": out.get("spec"),
|
| 321 |
+
"stdout": out.get("stdout") or "", "truncated": bool(out.get("truncated")),
|
| 322 |
+
"ms": int(out.get("ms") or 0), "code": out.get("code") or "",
|
| 323 |
+
# β `caps` RIDES ON THE ANSWER (standing rule 1). On a POSIX host all three limits
|
| 324 |
+
# were applied; on a Windows host the memory and CPU ones were not, and a screen
|
| 325 |
+
# that claims an enforcement which did not happen is the failure the rule is about.
|
| 326 |
+
"caps": out.get("caps") or {}}
|
| 327 |
+
if not answer["ok"]:
|
| 328 |
+
answer["error"] = out.get("message") or "the script view did not produce a view"
|
| 329 |
+
return answer
|
api/routes_shares.py
CHANGED
|
@@ -1,393 +1,393 @@
|
|
| 1 |
-
"""routes_shares.py β the manage-access surface (wave 20, owner ruling R10, contract C-SHARE).
|
| 2 |
-
|
| 3 |
-
GET /api/v1/share/{kind}/{oid} -> {owner, entries:[{user,role}], mayAdminister, people}
|
| 4 |
-
PUT /api/v1/share/{kind}/{oid} <- {entries:[{user,role}]} (REPLACES the set)
|
| 5 |
-
GET /api/v1/share/mine -> {view:[id], folder:[id], database:[id]}
|
| 6 |
-
|
| 7 |
-
`kind` β view | folder | database. Roles are `view` | `edit` β the same two words the view rail
|
| 8 |
-
already speaks, now extended to folders and databases so there is ONE vocabulary in the UI
|
| 9 |
-
(R10: "the same picker views use").
|
| 10 |
-
|
| 11 |
-
β **RE-SHARING IS THE OWNER'S, AND THAT IS ENFORCED HERE, NOT IN THE CLIENT.** `PUT` requires
|
| 12 |
-
`shares.may_administer` (owner or admin). A collaborator with `edit` may change an object's
|
| 13 |
-
CONTENT and may not change who else can reach it β otherwise anyone you shared a view with could
|
| 14 |
-
widen it to everyone, or grant themselves ownership and lock you out. The client greys the editor
|
| 15 |
-
for non-administrators; that is a courtesy, and this check is the wall.
|
| 16 |
-
|
| 17 |
-
β **THE GRANT NEVER WIDENS PAST THE MODULE WALL β ON A GOVERNED MODULE.** `*` ("everyone") means
|
| 18 |
-
every account that can already open the surface: `require_session` plus the topic's own gate run
|
| 19 |
-
first, and for `customer_data` / `product_data` the receiver's own row scope and hidden-field
|
| 20 |
-
closure run BEFORE any foreign view is merged. Sharing there can only narrow-or-equal the set that
|
| 21 |
-
could already reach the data ([[aios-permissioning]]).
|
| 22 |
-
|
| 23 |
-
ββ **AND THAT SENTENCE IS FALSE FOR `kind='database'`, WHICH IS WHY IT NOW SAYS "ON A GOVERNED
|
| 24 |
-
MODULE" (W32-T26, audit S-8).** `routes_admin._PERM_MODULES` is `("customer_data","product_data")`
|
| 25 |
-
and `_clean_perms` **400s** on anything else, so **no row filter and no hidden field can even be
|
| 26 |
-
DECLARED for a `ut_*` database** β `routes_tables.py` makes zero `perm_scope` calls and passes
|
| 27 |
-
`hidden_keys=frozenset()`. There is no module wall behind a user table for a grant to be bounded
|
| 28 |
-
by: **this registry IS the wall.** So a `database` grant is ALL-OR-NOTHING β every row, every
|
| 29 |
-
column β and an `*` database grant admits every account in the tenant to all of it.
|
| 30 |
-
That is a real capability, deliberately kept; what was wrong was a docstring promising a second
|
| 31 |
-
wall that does not exist for this kind. Scoping user tables is booked, not done
|
| 32 |
-
(`waves/wave32/sharing-audit.md` S-8).
|
| 33 |
-
|
| 34 |
-
β **TWO SYSTEMS ANSWER "IS THIS SHARED", AND THEY ARE NOT THE SAME ONE (audit S-4).** THIS
|
| 35 |
-
registry decides who appears in *"Shared with me"* and who may re-share. **`table_store.is_shared`
|
| 36 |
-
β the view's own `permissions` β is what actually decides who may OPEN a view.** A grant here
|
| 37 |
-
whose object is invisible under that one is a row in a list that opens a refusal, which is what
|
| 38 |
-
made item 18 worth auditing. `_entries_or_400` closes the common cause (a name nobody has), but
|
| 39 |
-
the two vocabularies are still two.
|
| 40 |
-
"""
|
| 41 |
-
from fastapi import APIRouter, Body, Depends
|
| 42 |
-
|
| 43 |
-
import core.shares as shares
|
| 44 |
-
import core.users as users
|
| 45 |
-
from deps import Session, err, require_session
|
| 46 |
-
# β W32-T28 (C3) β the SHARE notification's topic word, imported from the module that CLASSIFIES
|
| 47 |
-
# it (`routes_alerts.notification_view`) rather than typed again here. The producer and the
|
| 48 |
-
# reader agreeing about one string is the whole difference between an Inbox row that opens the
|
| 49 |
-
# shared database and one that is quietly unclickable.
|
| 50 |
-
from routes_alerts import SHARE_TOPIC as _SHARE_TOPIC
|
| 51 |
-
|
| 52 |
-
router = APIRouter(prefix="/api/v1")
|
| 53 |
-
|
| 54 |
-
|
| 55 |
-
def _kind_or_400(raw):
|
| 56 |
-
try:
|
| 57 |
-
return shares._check_kind(raw)
|
| 58 |
-
except ValueError as e:
|
| 59 |
-
raise err(400, "bad_kind", str(e))
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
# ββ ββ WAVE 32 Β· T26 (owner item 18, ruling R12) β THE WALL THIS FILE SAID IT HAD βββββββββββββ
|
| 63 |
-
#
|
| 64 |
-
# `put_share`'s comment used to justify the first-claim rule with *"reaching this route at all
|
| 65 |
-
# means passing the surface's own wall"*. **There was no such wall.** `kind` and `oid` are free
|
| 66 |
-
# strings off the URL and the only dependency was `require_session`, so any signed-in account
|
| 67 |
-
# could `PUT` a grant on an id it had never seen. Because the 403 sat behind `if rec["owner"]`,
|
| 68 |
-
# an object with no grant record skipped the check entirely and the caller was stamped OWNER β
|
| 69 |
-
# sticky, so **the real creator was then refused on their own view, permanently.** Driven, not
|
| 70 |
-
# argued: `waves/wave32/sharing-audit.md` S-1 carries the four-step transcript.
|
| 71 |
-
#
|
| 72 |
-
# β AND IT WAS SILENT ON BOTH SIDES. The claimant does not even see the object in their own
|
| 73 |
-
# "Shared with me" (`shared_with` excludes what you own), so nothing appears anywhere until the
|
| 74 |
-
# victim next opens the dialog.
|
| 75 |
-
|
| 76 |
-
#: The built-in grid topics. A view or folder lives in `{topic}_table_workspace`, and the share
|
| 77 |
-
#: route is not told which topic β so resolving one means asking each.
|
| 78 |
-
_BUILTIN_TOPICS = ("customer", "product")
|
| 79 |
-
|
| 80 |
-
|
| 81 |
-
def _topics(session):
|
| 82 |
-
"""Every topic whose workspace could hold a view or folder for this tenant.
|
| 83 |
-
|
| 84 |
-
β `all_defs`, never `all_tables` β the latter is the whole 28.6 MB row payload (~703 ms on
|
| 85 |
-
tenant #0) to answer a question about KEYS (D-185).
|
| 86 |
-
"""
|
| 87 |
-
try:
|
| 88 |
-
import core.user_tables as ut
|
| 89 |
-
return (*_BUILTIN_TOPICS, *(ut.all_defs(st=session.runtime) or {}))
|
| 90 |
-
except Exception: # noqa: BLE001
|
| 91 |
-
return _BUILTIN_TOPICS
|
| 92 |
-
|
| 93 |
-
|
| 94 |
-
def _owns_object(session, kind, oid):
|
| 95 |
-
"""May this caller CLAIM an object that has no grant record yet β i.e. do they own it?
|
| 96 |
-
|
| 97 |
-
β THIS GUARDS THE CLAIM, NOT THE READ, AND THAT IS DELIBERATE. Resolving a view means asking
|
| 98 |
-
each topic's workspace in turn, which is N store reads; making every share call pay that
|
| 99 |
-
would put a loop on a route the manage-access dialog opens. The dangerous path is the one
|
| 100 |
-
where a caller is about to be stamped OWNER of something nobody owns β so the resolution runs
|
| 101 |
-
exactly there, and the common path (a record exists, `may_administer` decides) is untouched.
|
| 102 |
-
"""
|
| 103 |
-
if session.admin:
|
| 104 |
-
return True
|
| 105 |
-
if kind == "database":
|
| 106 |
-
# β `may_open` is THE resolver for a user table (its own docstring says so) and already
|
| 107 |
-
# admits creator, admin, or a `database` grantee. Re-implementing "who owns a table"
|
| 108 |
-
# here would be the second definition this wave keeps finding.
|
| 109 |
-
try:
|
| 110 |
-
import core.user_tables as ut
|
| 111 |
-
return bool(ut.may_open(oid, session.uname, is_admin=session.admin,
|
| 112 |
-
st=session.runtime))
|
| 113 |
-
except Exception: # noqa: BLE001
|
| 114 |
-
return False
|
| 115 |
-
try:
|
| 116 |
-
import core.table_store as table_store
|
| 117 |
-
except Exception: # noqa: BLE001
|
| 118 |
-
return False
|
| 119 |
-
for topic in _topics(session):
|
| 120 |
-
try:
|
| 121 |
-
ops = table_store.make(f"{topic}_table_workspace", st=session.runtime)
|
| 122 |
-
hit = ops.find_view(oid) if kind == "view" else ops.find_folder(oid)
|
| 123 |
-
except Exception: # noqa: BLE001
|
| 124 |
-
continue
|
| 125 |
-
if hit:
|
| 126 |
-
# `find_view`/`find_folder` answer `(owner_username, β¦)`. The claim belongs to the
|
| 127 |
-
# person whose personal stratum holds it β anybody else reaching this line is
|
| 128 |
-
# exactly the case S-1 describes.
|
| 129 |
-
return str(hit[0]) == str(session.uname)
|
| 130 |
-
return False
|
| 131 |
-
|
| 132 |
-
|
| 133 |
-
def _can_see_object(session, kind, oid):
|
| 134 |
-
"""May this caller READ an object's grant list β i.e. can they reach the object at all?
|
| 135 |
-
|
| 136 |
-
ββ THIS IS DELIBERATELY WIDER THAN {@link _owns_object}, AND CONFLATING THE TWO IS A
|
| 137 |
-
REGRESSION I SHIPPED AND CAUGHT. The first version of T26 guarded BOTH doors with the
|
| 138 |
-
ownership test, which reads sensibly and is wrong for the read, because **`find_view` searches
|
| 139 |
-
PERSONAL STRATA ONLY** (its own docstring says so). So a view living in alice's stratum with
|
| 140 |
-
`permissions.edit = "collaborative"` and no grant record yet β a view bob **can open and edit
|
| 141 |
-
in the grid** β answered `404` when bob opened its manage-access dialog. Measured before
|
| 142 |
-
fixing: `table_store._may_see(view, "bob") is True` while `GET /share/view/vc` said
|
| 143 |
-
`404 no_object`.
|
| 144 |
-
β THAT IS THE AUDIT'S OWN S-4 BITING THE AUDIT'S OWN FIX: two systems answer "is this shared",
|
| 145 |
-
and the wall consulted the grant registry (system A) plus stratum ownership, never the view's
|
| 146 |
-
`permissions` (system B) β which is the one that actually decides who may OPEN it.
|
| 147 |
-
β And it hides the ANSWER, not just the editor. `ViewSidebar`'s Share row is deliberately not
|
| 148 |
-
gated on edit rights because *"hiding the row from everyone else would hide the ANSWER too β
|
| 149 |
-
'who has this?' is a fair question for anyone the view was shared with"*. A 404 there tells a
|
| 150 |
-
legitimate collaborator their view does not exist.
|
| 151 |
-
|
| 152 |
-
β THE CLAIM KEEPS THE NARROW TEST. Being able to SEE an object must not let you become its
|
| 153 |
-
owner β that is S-1, and widening this predicate onto `put_share` would re-open it.
|
| 154 |
-
"""
|
| 155 |
-
if _owns_object(session, kind, oid):
|
| 156 |
-
return True
|
| 157 |
-
if kind != "view":
|
| 158 |
-
# A folder carries no per-object visibility flag of its own, and a database's `may_open`
|
| 159 |
-
# (inside `_owns_object`) already admits grantees. Nothing wider to ask.
|
| 160 |
-
return False
|
| 161 |
-
try:
|
| 162 |
-
import core.table_store as table_store
|
| 163 |
-
for topic in _topics(session):
|
| 164 |
-
hit = table_store.make(f"{topic}_table_workspace", st=session.runtime).find_view(oid)
|
| 165 |
-
if hit:
|
| 166 |
-
return bool(table_store._may_see(hit[1] if len(hit) > 1 else {},
|
| 167 |
-
session.uname, is_admin=session.admin))
|
| 168 |
-
except Exception: # noqa: BLE001
|
| 169 |
-
return False
|
| 170 |
-
return False
|
| 171 |
-
|
| 172 |
-
|
| 173 |
-
def _entries_or_400(session, entries):
|
| 174 |
-
"""Validate a grant list against the tenant's REAL, ACTIVE accounts β and refuse BY NAME.
|
| 175 |
-
|
| 176 |
-
β `core.shares._clean_entries` silently drops junk, and its docstring argues that correctly:
|
| 177 |
-
a UI mid-save must not lose the whole list to one malformed row. **But it validates the SHAPE
|
| 178 |
-
of a string and the role word β never that the user EXISTS, is ACTIVE, or is in this tenant**,
|
| 179 |
-
so a typo'd name is stored, reported as a successful save, and never reaches anybody. The
|
| 180 |
-
sharer believes the person has access. That is item 18's plain reading.
|
| 181 |
-
β The correct population is computed THREE FUNCTIONS BELOW and served to the picker
|
| 182 |
-
(`_people`). One route, two populations, and the write door was the permissive one.
|
| 183 |
-
β `*` (everyone) is not a user and is admitted deliberately β it is R10's vocabulary for
|
| 184 |
-
"every account that can already open the surface".
|
| 185 |
-
"""
|
| 186 |
-
known = {p["username"].strip().lower() for p in _people(session.tenant)}
|
| 187 |
-
unknown = []
|
| 188 |
-
for e in entries or ():
|
| 189 |
-
if not isinstance(e, dict):
|
| 190 |
-
continue
|
| 191 |
-
user = str(e.get("user") or "").strip().lower()
|
| 192 |
-
if user and user != shares.EVERYONE and user not in known:
|
| 193 |
-
unknown.append(user)
|
| 194 |
-
if unknown:
|
| 195 |
-
raise err(400, "unknown_people",
|
| 196 |
-
"no active account in this workspace is named "
|
| 197 |
-
+ ", ".join(sorted(set(unknown)))
|
| 198 |
-
+ " β nothing was shared. Pick people from the list rather than typing a name.")
|
| 199 |
-
|
| 200 |
-
|
| 201 |
-
@router.get("/share/mine")
|
| 202 |
-
def my_shares(session: Session = Depends(require_session)):
|
| 203 |
-
"""Everything shared WITH me, by kind β the "Shared with me" rail section (R10).
|
| 204 |
-
|
| 205 |
-
Registered before `/share/{kind}/{oid}` so the literal path wins the match; FastAPI resolves
|
| 206 |
-
in declaration order and `mine` would otherwise be read as a `kind`, answering 400 for a URL
|
| 207 |
-
that is not malformed at all.
|
| 208 |
-
"""
|
| 209 |
-
return shares.shared_with(session.uname, st=session.runtime)
|
| 210 |
-
|
| 211 |
-
|
| 212 |
-
@router.get("/share/{kind}/{oid}")
|
| 213 |
-
def get_share(kind: str, oid: str, session: Session = Depends(require_session)):
|
| 214 |
-
kind = _kind_or_400(kind)
|
| 215 |
-
rec = shares.grants(kind, oid, st=session.runtime)
|
| 216 |
-
role = shares.role_for(kind, oid, session.uname, is_admin=session.admin, st=session.runtime)
|
| 217 |
-
may_admin = shares.may_administer(kind, oid, session.uname, is_admin=session.admin,
|
| 218 |
-
st=session.runtime)
|
| 219 |
-
# β W32-T26 (audit S-3) β A STRANGER LEARNS NOTHING. This route used to answer for ANY id:
|
| 220 |
-
# who owns it, everyone it is granted to, and the tenant's whole usernameβname directory β
|
| 221 |
-
# to any signed-in session, about objects it cannot open. Now a caller with no role on an
|
| 222 |
-
# object must prove they can reach it, and gets a 404 otherwise: the same answer a
|
| 223 |
-
# non-existent id gives, so the route cannot be used to probe which ids are real.
|
| 224 |
-
# β `role is None` is the cheap pre-test, so the N-topic resolution below runs only for a
|
| 225 |
-
# caller who has no relationship with the object at all.
|
| 226 |
-
if role is None and not _can_see_object(session, kind, oid):
|
| 227 |
-
raise err(404, "no_object", "no such item, or it is not shared with this account")
|
| 228 |
-
return {
|
| 229 |
-
**rec,
|
| 230 |
-
"role": role,
|
| 231 |
-
"mayAdminister": may_admin,
|
| 232 |
-
# β WAVE 21 (C1 identity fix): grant entries BIND on USERNAMES, so the picker must carry
|
| 233 |
-
# them. `assignable_people` serves bare display names because `user`-kind CELLS store
|
| 234 |
-
# display names β that list's shape cannot change without migrating cell values β so
|
| 235 |
-
# this route serves objects of its own. Existing grants that were written as lowercased
|
| 236 |
-
# display names are normalised by the wave-21 cleanup script.
|
| 237 |
-
# β W32-T26 (audit S-3) β the roster is the EDITOR's data, so it rides only for a caller
|
| 238 |
-
# who may open the editor. A read-only grantee gets the grant list (their fair question is
|
| 239 |
-
# "who else has this?") and not a directory of every account in the workspace.
|
| 240 |
-
"people": _people(session.tenant) if may_admin else [],
|
| 241 |
-
}
|
| 242 |
-
|
| 243 |
-
|
| 244 |
-
def _people(tenant):
|
| 245 |
-
"""[{username, name}] for this tenant β same population as `assignable_people`, with the
|
| 246 |
-
BINDING identity alongside the display one."""
|
| 247 |
-
try:
|
| 248 |
-
reg = users.registry() or {}
|
| 249 |
-
except Exception:
|
| 250 |
-
return []
|
| 251 |
-
want = str(tenant or '').strip().lower()
|
| 252 |
-
out = []
|
| 253 |
-
for uname, u in reg.items():
|
| 254 |
-
if not isinstance(u, dict) or u.get('active') is False:
|
| 255 |
-
continue
|
| 256 |
-
if want and str(u.get('tenant') or 'royal-imports').strip().lower() != want:
|
| 257 |
-
continue
|
| 258 |
-
out.append({"username": str(uname), "name": str(u.get('name') or uname)})
|
| 259 |
-
return sorted(out, key=lambda p: p["name"].lower())
|
| 260 |
-
|
| 261 |
-
|
| 262 |
-
@router.put("/share/{kind}/{oid}")
|
| 263 |
-
def put_share(kind: str, oid: str, body: dict = Body(default=None),
|
| 264 |
-
session: Session = Depends(require_session)):
|
| 265 |
-
kind = _kind_or_400(kind)
|
| 266 |
-
body = body or {}
|
| 267 |
-
rec = shares.grants(kind, oid, st=session.runtime)
|
| 268 |
-
# An object with NO grant record yet has no owner β the first person to share it claims it.
|
| 269 |
-
# That is safe because reaching this route at all means passing the surface's own wall, and
|
| 270 |
-
# the alternative (refusing until somebody seeds an owner) would make a brand-new folder
|
| 271 |
-
# unshareable by the person who just made it.
|
| 272 |
-
if rec["owner"]:
|
| 273 |
-
if not shares.may_administer(kind, oid, session.uname, is_admin=session.admin,
|
| 274 |
-
st=session.runtime):
|
| 275 |
-
raise err(403, "not_owner",
|
| 276 |
-
"only the owner of this item (or an administrator) can change who it is "
|
| 277 |
-
"shared with")
|
| 278 |
-
# ββ W32-T26 (audit S-1) β THE CLAIM NOW HAS A PRECONDITION. An object with no grant record
|
| 279 |
-
# is still claimed by the first person to share it β that rule is right, and refusing until
|
| 280 |
-
# somebody seeds an owner would make a brand-new folder unshareable by the person who just
|
| 281 |
-
# made it. What was missing is the half the old comment ASSERTED and the code never did: the
|
| 282 |
-
# claimant has to be able to reach the object. Without this, any signed-in account could
|
| 283 |
-
# stamp itself owner of an id it had never seen and lock the real creator out for good.
|
| 284 |
-
elif not _owns_object(session, kind, oid):
|
| 285 |
-
raise err(404, "no_object", "no such item, or it is not shared with this account")
|
| 286 |
-
entries = body.get("entries")
|
| 287 |
-
if not isinstance(entries, list):
|
| 288 |
-
raise err(400, "bad_entries",
|
| 289 |
-
"entries must be a list of {user, role} β send [] to un-share, which is how "
|
| 290 |
-
"revoking is expressed")
|
| 291 |
-
_entries_or_400(session, entries)
|
| 292 |
-
out = shares.set_grants(kind, oid, entries, owner=rec["owner"] or session.uname,
|
| 293 |
-
st=session.runtime)
|
| 294 |
-
_notify_new_grantees(session, kind, oid, before=rec["entries"], after=out.get("entries") or [])
|
| 295 |
-
return out
|
| 296 |
-
|
| 297 |
-
|
| 298 |
-
def _notify_new_grantees(session, kind, oid, before, after):
|
| 299 |
-
"""ββ W32-T28 (owner item 18's last clause, contract C3)
|
| 300 |
-
|
| 301 |
-
Owner item 18 ends *"being shared a database notifies the receiver"*. Until now sharing was
|
| 302 |
-
silent: the grant landed in a rail section the receiver had to notice on their own, which is
|
| 303 |
-
why "I shared it with you" and "I never saw it" were both true.
|
| 304 |
-
|
| 305 |
-
β WRITTEN ON THE SHARE, NEVER POLLED. `/notifications` re-evaluates view-ALERTS on read
|
| 306 |
-
because an alert is a live question about rows; a share is an EVENT that happened once, and
|
| 307 |
-
polling for it would mean re-deriving "was this new?" on every inbox open β the diff below
|
| 308 |
-
only exists here, at the moment the set changes.
|
| 309 |
-
|
| 310 |
-
β ONLY THE NEWLY ADDED. `PUT` REPLACES the whole entry set (revoking is expressed by absence),
|
| 311 |
-
so every save re-sends everyone who was already there. Diffing against `before` is what stops
|
| 312 |
-
a rename or a role change from ringing the bell for people whose access did not change.
|
| 313 |
-
β `*` IS NOT NOTIFIED: there is no user to name, and minting one notification per account in
|
| 314 |
-
the tenant on a single click is a broadcast nobody asked for. The rail still shows it.
|
| 315 |
-
β IT NEVER RAISES. A notification that fails must not fail the share that triggered it β the
|
| 316 |
-
grant is the user's actual intent, and `core.alerts.notify` writes with `flush='async'`.
|
| 317 |
-
"""
|
| 318 |
-
try:
|
| 319 |
-
was = {e.get("user") for e in (before or ()) if isinstance(e, dict)}
|
| 320 |
-
fresh = [str(e.get("user")) for e in (after or ())
|
| 321 |
-
if isinstance(e, dict) and e.get("user") not in was
|
| 322 |
-
and e.get("user") != shares.EVERYONE]
|
| 323 |
-
if not fresh:
|
| 324 |
-
return
|
| 325 |
-
import core.alerts as alerts
|
| 326 |
-
|
| 327 |
-
label, route, view_id = _object_ref(session, kind, oid)
|
| 328 |
-
if not route:
|
| 329 |
-
# β NO ROUTE, NO NOTIFICATION β the receiver would get a row that opens nothing, and
|
| 330 |
-
# `notification_view` would have to invent a target. Silence is the honest answer
|
| 331 |
-
# here; the rail still shows the grant under "Shared with me".
|
| 332 |
-
return
|
| 333 |
-
sharer = str(session.user.get("name") or session.uname)
|
| 334 |
-
for user in fresh:
|
| 335 |
-
# β THE SHAPE IS `routes_alerts.notification_view`'s SHARE BRANCH, and the two must
|
| 336 |
-
# agree or the Inbox row is unclickable: `topic` selects the branch and `key` becomes
|
| 337 |
-
# `alertId`, which that branch reads as the id to open. Both constants are IMPORTED
|
| 338 |
-
# from there rather than typed again β one vocabulary, one owner.
|
| 339 |
-
# ββ W33-T28 (`ASK C-14`, answered) β `actor` IS THE SENDER, AND IT IS THE ONLY WAY
|
| 340 |
-
# THE INBOX CAN NAME ONE. An alert and an automation have no person behind them and
|
| 341 |
-
# are honestly named by their machine; a SHARE has a real person, and only this call
|
| 342 |
-
# site knows who. β It is passed as its OWN field rather than recovered from the
|
| 343 |
-
# `detail` prose below: a sender parsed out of "<name> shared this with you" breaks
|
| 344 |
-
# the first time the sentence is reworded, silently, in the header
|
| 345 |
-
# [[grep-output-is-not-source]]. The prose stays as the body; this is the From.
|
| 346 |
-
alerts.notify(user, label, topic=_SHARE_TOPIC, key=route, row_id=view_id,
|
| 347 |
-
detail=f"{sharer} shared this with you", actor=sharer,
|
| 348 |
-
st=session.runtime)
|
| 349 |
-
except Exception: # noqa: BLE001
|
| 350 |
-
return
|
| 351 |
-
|
| 352 |
-
|
| 353 |
-
def _object_ref(session, kind, oid):
|
| 354 |
-
"""`(label, route, view_id)` β what to CALL the shared thing, and where it OPENS.
|
| 355 |
-
|
| 356 |
-
β THE ROUTE IS RESOLVED HERE, NOT SHAPED IN THE CONSUMER, AND THE FIRST VERSION GOT IT
|
| 357 |
-
WRONG: it put the raw `oid` in the notification's key, so a shared VIEW produced
|
| 358 |
-
`target: {module: "database", id: "view_42"}` β an instruction to open a database named
|
| 359 |
-
`view_42`. It read perfectly in the payload and would have opened nothing. **A view is not
|
| 360 |
-
addressable on its own; it is a SELECTION inside a topic's grid**, so the pair is what has to
|
| 361 |
-
travel. Caught by looking at the notification the driver actually produced, not by reading
|
| 362 |
-
the code back.
|
| 363 |
-
|
| 364 |
-
β `label` never falls back to a raw id. A notification headed `ut_leads_3f2a` tells the
|
| 365 |
-
receiver nothing they can act on, and the id is already in the target.
|
| 366 |
-
β An unresolvable object answers `route=None`, and the caller then sends NOTHING rather than
|
| 367 |
-
a row that opens nowhere.
|
| 368 |
-
"""
|
| 369 |
-
try:
|
| 370 |
-
if kind == "database":
|
| 371 |
-
import core.user_tables as ut
|
| 372 |
-
defn = (ut.all_defs(st=session.runtime) or {}).get(str(oid)) or {}
|
| 373 |
-
# A user table IS its own route key in both vocabularies (`route_for_topic`).
|
| 374 |
-
return (str(defn.get("label") or "").strip() or "A database", str(oid), "")
|
| 375 |
-
import core.table_store as table_store
|
| 376 |
-
from routes_alerts import route_for_topic
|
| 377 |
-
for topic in _topics(session):
|
| 378 |
-
ops = table_store.make(f"{topic}_table_workspace", st=session.runtime)
|
| 379 |
-
hit = ops.find_view(oid) if kind == "view" else ops.find_folder(oid)
|
| 380 |
-
if not hit:
|
| 381 |
-
continue
|
| 382 |
-
route = route_for_topic(topic)
|
| 383 |
-
if not route:
|
| 384 |
-
break
|
| 385 |
-
row = hit[1] if len(hit) > 1 else {}
|
| 386 |
-
name = str((row or {}).get("name") or "").strip()
|
| 387 |
-
# β Only a VIEW carries a selection. A folder is a rail grouping, so the target opens
|
| 388 |
-
# the grid and stops there rather than naming a view the receiver did not get.
|
| 389 |
-
return (name or ("A view" if kind == "view" else "A folder"),
|
| 390 |
-
route, str(oid) if kind == "view" else "")
|
| 391 |
-
except Exception: # noqa: BLE001
|
| 392 |
-
pass
|
| 393 |
-
return ({"view": "A view", "folder": "A folder"}.get(kind, "An item"), None, "")
|
|
|
|
| 1 |
+
"""routes_shares.py β the manage-access surface (wave 20, owner ruling R10, contract C-SHARE).
|
| 2 |
+
|
| 3 |
+
GET /api/v1/share/{kind}/{oid} -> {owner, entries:[{user,role}], mayAdminister, people}
|
| 4 |
+
PUT /api/v1/share/{kind}/{oid} <- {entries:[{user,role}]} (REPLACES the set)
|
| 5 |
+
GET /api/v1/share/mine -> {view:[id], folder:[id], database:[id]}
|
| 6 |
+
|
| 7 |
+
`kind` β view | folder | database. Roles are `view` | `edit` β the same two words the view rail
|
| 8 |
+
already speaks, now extended to folders and databases so there is ONE vocabulary in the UI
|
| 9 |
+
(R10: "the same picker views use").
|
| 10 |
+
|
| 11 |
+
β **RE-SHARING IS THE OWNER'S, AND THAT IS ENFORCED HERE, NOT IN THE CLIENT.** `PUT` requires
|
| 12 |
+
`shares.may_administer` (owner or admin). A collaborator with `edit` may change an object's
|
| 13 |
+
CONTENT and may not change who else can reach it β otherwise anyone you shared a view with could
|
| 14 |
+
widen it to everyone, or grant themselves ownership and lock you out. The client greys the editor
|
| 15 |
+
for non-administrators; that is a courtesy, and this check is the wall.
|
| 16 |
+
|
| 17 |
+
β **THE GRANT NEVER WIDENS PAST THE MODULE WALL β ON A GOVERNED MODULE.** `*` ("everyone") means
|
| 18 |
+
every account that can already open the surface: `require_session` plus the topic's own gate run
|
| 19 |
+
first, and for `customer_data` / `product_data` the receiver's own row scope and hidden-field
|
| 20 |
+
closure run BEFORE any foreign view is merged. Sharing there can only narrow-or-equal the set that
|
| 21 |
+
could already reach the data ([[aios-permissioning]]).
|
| 22 |
+
|
| 23 |
+
ββ **AND THAT SENTENCE IS FALSE FOR `kind='database'`, WHICH IS WHY IT NOW SAYS "ON A GOVERNED
|
| 24 |
+
MODULE" (W32-T26, audit S-8).** `routes_admin._PERM_MODULES` is `("customer_data","product_data")`
|
| 25 |
+
and `_clean_perms` **400s** on anything else, so **no row filter and no hidden field can even be
|
| 26 |
+
DECLARED for a `ut_*` database** β `routes_tables.py` makes zero `perm_scope` calls and passes
|
| 27 |
+
`hidden_keys=frozenset()`. There is no module wall behind a user table for a grant to be bounded
|
| 28 |
+
by: **this registry IS the wall.** So a `database` grant is ALL-OR-NOTHING β every row, every
|
| 29 |
+
column β and an `*` database grant admits every account in the tenant to all of it.
|
| 30 |
+
That is a real capability, deliberately kept; what was wrong was a docstring promising a second
|
| 31 |
+
wall that does not exist for this kind. Scoping user tables is booked, not done
|
| 32 |
+
(`waves/wave32/sharing-audit.md` S-8).
|
| 33 |
+
|
| 34 |
+
β **TWO SYSTEMS ANSWER "IS THIS SHARED", AND THEY ARE NOT THE SAME ONE (audit S-4).** THIS
|
| 35 |
+
registry decides who appears in *"Shared with me"* and who may re-share. **`table_store.is_shared`
|
| 36 |
+
β the view's own `permissions` β is what actually decides who may OPEN a view.** A grant here
|
| 37 |
+
whose object is invisible under that one is a row in a list that opens a refusal, which is what
|
| 38 |
+
made item 18 worth auditing. `_entries_or_400` closes the common cause (a name nobody has), but
|
| 39 |
+
the two vocabularies are still two.
|
| 40 |
+
"""
|
| 41 |
+
from fastapi import APIRouter, Body, Depends
|
| 42 |
+
|
| 43 |
+
import core.shares as shares
|
| 44 |
+
import core.users as users
|
| 45 |
+
from deps import Session, err, require_session
|
| 46 |
+
# β W32-T28 (C3) β the SHARE notification's topic word, imported from the module that CLASSIFIES
|
| 47 |
+
# it (`routes_alerts.notification_view`) rather than typed again here. The producer and the
|
| 48 |
+
# reader agreeing about one string is the whole difference between an Inbox row that opens the
|
| 49 |
+
# shared database and one that is quietly unclickable.
|
| 50 |
+
from routes_alerts import SHARE_TOPIC as _SHARE_TOPIC
|
| 51 |
+
|
| 52 |
+
router = APIRouter(prefix="/api/v1")
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def _kind_or_400(raw):
|
| 56 |
+
try:
|
| 57 |
+
return shares._check_kind(raw)
|
| 58 |
+
except ValueError as e:
|
| 59 |
+
raise err(400, "bad_kind", str(e))
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
# ββ ββ WAVE 32 Β· T26 (owner item 18, ruling R12) β THE WALL THIS FILE SAID IT HAD βββββββββββββ
|
| 63 |
+
#
|
| 64 |
+
# `put_share`'s comment used to justify the first-claim rule with *"reaching this route at all
|
| 65 |
+
# means passing the surface's own wall"*. **There was no such wall.** `kind` and `oid` are free
|
| 66 |
+
# strings off the URL and the only dependency was `require_session`, so any signed-in account
|
| 67 |
+
# could `PUT` a grant on an id it had never seen. Because the 403 sat behind `if rec["owner"]`,
|
| 68 |
+
# an object with no grant record skipped the check entirely and the caller was stamped OWNER β
|
| 69 |
+
# sticky, so **the real creator was then refused on their own view, permanently.** Driven, not
|
| 70 |
+
# argued: `waves/wave32/sharing-audit.md` S-1 carries the four-step transcript.
|
| 71 |
+
#
|
| 72 |
+
# β AND IT WAS SILENT ON BOTH SIDES. The claimant does not even see the object in their own
|
| 73 |
+
# "Shared with me" (`shared_with` excludes what you own), so nothing appears anywhere until the
|
| 74 |
+
# victim next opens the dialog.
|
| 75 |
+
|
| 76 |
+
#: The built-in grid topics. A view or folder lives in `{topic}_table_workspace`, and the share
|
| 77 |
+
#: route is not told which topic β so resolving one means asking each.
|
| 78 |
+
_BUILTIN_TOPICS = ("customer", "product")
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
def _topics(session):
|
| 82 |
+
"""Every topic whose workspace could hold a view or folder for this tenant.
|
| 83 |
+
|
| 84 |
+
β `all_defs`, never `all_tables` β the latter is the whole 28.6 MB row payload (~703 ms on
|
| 85 |
+
tenant #0) to answer a question about KEYS (D-185).
|
| 86 |
+
"""
|
| 87 |
+
try:
|
| 88 |
+
import core.user_tables as ut
|
| 89 |
+
return (*_BUILTIN_TOPICS, *(ut.all_defs(st=session.runtime) or {}))
|
| 90 |
+
except Exception: # noqa: BLE001
|
| 91 |
+
return _BUILTIN_TOPICS
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
def _owns_object(session, kind, oid):
|
| 95 |
+
"""May this caller CLAIM an object that has no grant record yet β i.e. do they own it?
|
| 96 |
+
|
| 97 |
+
β THIS GUARDS THE CLAIM, NOT THE READ, AND THAT IS DELIBERATE. Resolving a view means asking
|
| 98 |
+
each topic's workspace in turn, which is N store reads; making every share call pay that
|
| 99 |
+
would put a loop on a route the manage-access dialog opens. The dangerous path is the one
|
| 100 |
+
where a caller is about to be stamped OWNER of something nobody owns β so the resolution runs
|
| 101 |
+
exactly there, and the common path (a record exists, `may_administer` decides) is untouched.
|
| 102 |
+
"""
|
| 103 |
+
if session.admin:
|
| 104 |
+
return True
|
| 105 |
+
if kind == "database":
|
| 106 |
+
# β `may_open` is THE resolver for a user table (its own docstring says so) and already
|
| 107 |
+
# admits creator, admin, or a `database` grantee. Re-implementing "who owns a table"
|
| 108 |
+
# here would be the second definition this wave keeps finding.
|
| 109 |
+
try:
|
| 110 |
+
import core.user_tables as ut
|
| 111 |
+
return bool(ut.may_open(oid, session.uname, is_admin=session.admin,
|
| 112 |
+
st=session.runtime))
|
| 113 |
+
except Exception: # noqa: BLE001
|
| 114 |
+
return False
|
| 115 |
+
try:
|
| 116 |
+
import core.table_store as table_store
|
| 117 |
+
except Exception: # noqa: BLE001
|
| 118 |
+
return False
|
| 119 |
+
for topic in _topics(session):
|
| 120 |
+
try:
|
| 121 |
+
ops = table_store.make(f"{topic}_table_workspace", st=session.runtime)
|
| 122 |
+
hit = ops.find_view(oid) if kind == "view" else ops.find_folder(oid)
|
| 123 |
+
except Exception: # noqa: BLE001
|
| 124 |
+
continue
|
| 125 |
+
if hit:
|
| 126 |
+
# `find_view`/`find_folder` answer `(owner_username, β¦)`. The claim belongs to the
|
| 127 |
+
# person whose personal stratum holds it β anybody else reaching this line is
|
| 128 |
+
# exactly the case S-1 describes.
|
| 129 |
+
return str(hit[0]) == str(session.uname)
|
| 130 |
+
return False
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
def _can_see_object(session, kind, oid):
|
| 134 |
+
"""May this caller READ an object's grant list β i.e. can they reach the object at all?
|
| 135 |
+
|
| 136 |
+
ββ THIS IS DELIBERATELY WIDER THAN {@link _owns_object}, AND CONFLATING THE TWO IS A
|
| 137 |
+
REGRESSION I SHIPPED AND CAUGHT. The first version of T26 guarded BOTH doors with the
|
| 138 |
+
ownership test, which reads sensibly and is wrong for the read, because **`find_view` searches
|
| 139 |
+
PERSONAL STRATA ONLY** (its own docstring says so). So a view living in alice's stratum with
|
| 140 |
+
`permissions.edit = "collaborative"` and no grant record yet β a view bob **can open and edit
|
| 141 |
+
in the grid** β answered `404` when bob opened its manage-access dialog. Measured before
|
| 142 |
+
fixing: `table_store._may_see(view, "bob") is True` while `GET /share/view/vc` said
|
| 143 |
+
`404 no_object`.
|
| 144 |
+
β THAT IS THE AUDIT'S OWN S-4 BITING THE AUDIT'S OWN FIX: two systems answer "is this shared",
|
| 145 |
+
and the wall consulted the grant registry (system A) plus stratum ownership, never the view's
|
| 146 |
+
`permissions` (system B) β which is the one that actually decides who may OPEN it.
|
| 147 |
+
β And it hides the ANSWER, not just the editor. `ViewSidebar`'s Share row is deliberately not
|
| 148 |
+
gated on edit rights because *"hiding the row from everyone else would hide the ANSWER too β
|
| 149 |
+
'who has this?' is a fair question for anyone the view was shared with"*. A 404 there tells a
|
| 150 |
+
legitimate collaborator their view does not exist.
|
| 151 |
+
|
| 152 |
+
β THE CLAIM KEEPS THE NARROW TEST. Being able to SEE an object must not let you become its
|
| 153 |
+
owner β that is S-1, and widening this predicate onto `put_share` would re-open it.
|
| 154 |
+
"""
|
| 155 |
+
if _owns_object(session, kind, oid):
|
| 156 |
+
return True
|
| 157 |
+
if kind != "view":
|
| 158 |
+
# A folder carries no per-object visibility flag of its own, and a database's `may_open`
|
| 159 |
+
# (inside `_owns_object`) already admits grantees. Nothing wider to ask.
|
| 160 |
+
return False
|
| 161 |
+
try:
|
| 162 |
+
import core.table_store as table_store
|
| 163 |
+
for topic in _topics(session):
|
| 164 |
+
hit = table_store.make(f"{topic}_table_workspace", st=session.runtime).find_view(oid)
|
| 165 |
+
if hit:
|
| 166 |
+
return bool(table_store._may_see(hit[1] if len(hit) > 1 else {},
|
| 167 |
+
session.uname, is_admin=session.admin))
|
| 168 |
+
except Exception: # noqa: BLE001
|
| 169 |
+
return False
|
| 170 |
+
return False
|
| 171 |
+
|
| 172 |
+
|
| 173 |
+
def _entries_or_400(session, entries):
|
| 174 |
+
"""Validate a grant list against the tenant's REAL, ACTIVE accounts β and refuse BY NAME.
|
| 175 |
+
|
| 176 |
+
β `core.shares._clean_entries` silently drops junk, and its docstring argues that correctly:
|
| 177 |
+
a UI mid-save must not lose the whole list to one malformed row. **But it validates the SHAPE
|
| 178 |
+
of a string and the role word β never that the user EXISTS, is ACTIVE, or is in this tenant**,
|
| 179 |
+
so a typo'd name is stored, reported as a successful save, and never reaches anybody. The
|
| 180 |
+
sharer believes the person has access. That is item 18's plain reading.
|
| 181 |
+
β The correct population is computed THREE FUNCTIONS BELOW and served to the picker
|
| 182 |
+
(`_people`). One route, two populations, and the write door was the permissive one.
|
| 183 |
+
β `*` (everyone) is not a user and is admitted deliberately β it is R10's vocabulary for
|
| 184 |
+
"every account that can already open the surface".
|
| 185 |
+
"""
|
| 186 |
+
known = {p["username"].strip().lower() for p in _people(session.tenant)}
|
| 187 |
+
unknown = []
|
| 188 |
+
for e in entries or ():
|
| 189 |
+
if not isinstance(e, dict):
|
| 190 |
+
continue
|
| 191 |
+
user = str(e.get("user") or "").strip().lower()
|
| 192 |
+
if user and user != shares.EVERYONE and user not in known:
|
| 193 |
+
unknown.append(user)
|
| 194 |
+
if unknown:
|
| 195 |
+
raise err(400, "unknown_people",
|
| 196 |
+
"no active account in this workspace is named "
|
| 197 |
+
+ ", ".join(sorted(set(unknown)))
|
| 198 |
+
+ " β nothing was shared. Pick people from the list rather than typing a name.")
|
| 199 |
+
|
| 200 |
+
|
| 201 |
+
@router.get("/share/mine")
|
| 202 |
+
def my_shares(session: Session = Depends(require_session)):
|
| 203 |
+
"""Everything shared WITH me, by kind β the "Shared with me" rail section (R10).
|
| 204 |
+
|
| 205 |
+
Registered before `/share/{kind}/{oid}` so the literal path wins the match; FastAPI resolves
|
| 206 |
+
in declaration order and `mine` would otherwise be read as a `kind`, answering 400 for a URL
|
| 207 |
+
that is not malformed at all.
|
| 208 |
+
"""
|
| 209 |
+
return shares.shared_with(session.uname, st=session.runtime)
|
| 210 |
+
|
| 211 |
+
|
| 212 |
+
@router.get("/share/{kind}/{oid}")
|
| 213 |
+
def get_share(kind: str, oid: str, session: Session = Depends(require_session)):
|
| 214 |
+
kind = _kind_or_400(kind)
|
| 215 |
+
rec = shares.grants(kind, oid, st=session.runtime)
|
| 216 |
+
role = shares.role_for(kind, oid, session.uname, is_admin=session.admin, st=session.runtime)
|
| 217 |
+
may_admin = shares.may_administer(kind, oid, session.uname, is_admin=session.admin,
|
| 218 |
+
st=session.runtime)
|
| 219 |
+
# β W32-T26 (audit S-3) β A STRANGER LEARNS NOTHING. This route used to answer for ANY id:
|
| 220 |
+
# who owns it, everyone it is granted to, and the tenant's whole usernameβname directory β
|
| 221 |
+
# to any signed-in session, about objects it cannot open. Now a caller with no role on an
|
| 222 |
+
# object must prove they can reach it, and gets a 404 otherwise: the same answer a
|
| 223 |
+
# non-existent id gives, so the route cannot be used to probe which ids are real.
|
| 224 |
+
# β `role is None` is the cheap pre-test, so the N-topic resolution below runs only for a
|
| 225 |
+
# caller who has no relationship with the object at all.
|
| 226 |
+
if role is None and not _can_see_object(session, kind, oid):
|
| 227 |
+
raise err(404, "no_object", "no such item, or it is not shared with this account")
|
| 228 |
+
return {
|
| 229 |
+
**rec,
|
| 230 |
+
"role": role,
|
| 231 |
+
"mayAdminister": may_admin,
|
| 232 |
+
# β WAVE 21 (C1 identity fix): grant entries BIND on USERNAMES, so the picker must carry
|
| 233 |
+
# them. `assignable_people` serves bare display names because `user`-kind CELLS store
|
| 234 |
+
# display names β that list's shape cannot change without migrating cell values β so
|
| 235 |
+
# this route serves objects of its own. Existing grants that were written as lowercased
|
| 236 |
+
# display names are normalised by the wave-21 cleanup script.
|
| 237 |
+
# β W32-T26 (audit S-3) β the roster is the EDITOR's data, so it rides only for a caller
|
| 238 |
+
# who may open the editor. A read-only grantee gets the grant list (their fair question is
|
| 239 |
+
# "who else has this?") and not a directory of every account in the workspace.
|
| 240 |
+
"people": _people(session.tenant) if may_admin else [],
|
| 241 |
+
}
|
| 242 |
+
|
| 243 |
+
|
| 244 |
+
def _people(tenant):
|
| 245 |
+
"""[{username, name}] for this tenant β same population as `assignable_people`, with the
|
| 246 |
+
BINDING identity alongside the display one."""
|
| 247 |
+
try:
|
| 248 |
+
reg = users.registry() or {}
|
| 249 |
+
except Exception:
|
| 250 |
+
return []
|
| 251 |
+
want = str(tenant or '').strip().lower()
|
| 252 |
+
out = []
|
| 253 |
+
for uname, u in reg.items():
|
| 254 |
+
if not isinstance(u, dict) or u.get('active') is False:
|
| 255 |
+
continue
|
| 256 |
+
if want and str(u.get('tenant') or 'royal-imports').strip().lower() != want:
|
| 257 |
+
continue
|
| 258 |
+
out.append({"username": str(uname), "name": str(u.get('name') or uname)})
|
| 259 |
+
return sorted(out, key=lambda p: p["name"].lower())
|
| 260 |
+
|
| 261 |
+
|
| 262 |
+
@router.put("/share/{kind}/{oid}")
|
| 263 |
+
def put_share(kind: str, oid: str, body: dict = Body(default=None),
|
| 264 |
+
session: Session = Depends(require_session)):
|
| 265 |
+
kind = _kind_or_400(kind)
|
| 266 |
+
body = body or {}
|
| 267 |
+
rec = shares.grants(kind, oid, st=session.runtime)
|
| 268 |
+
# An object with NO grant record yet has no owner β the first person to share it claims it.
|
| 269 |
+
# That is safe because reaching this route at all means passing the surface's own wall, and
|
| 270 |
+
# the alternative (refusing until somebody seeds an owner) would make a brand-new folder
|
| 271 |
+
# unshareable by the person who just made it.
|
| 272 |
+
if rec["owner"]:
|
| 273 |
+
if not shares.may_administer(kind, oid, session.uname, is_admin=session.admin,
|
| 274 |
+
st=session.runtime):
|
| 275 |
+
raise err(403, "not_owner",
|
| 276 |
+
"only the owner of this item (or an administrator) can change who it is "
|
| 277 |
+
"shared with")
|
| 278 |
+
# ββ W32-T26 (audit S-1) β THE CLAIM NOW HAS A PRECONDITION. An object with no grant record
|
| 279 |
+
# is still claimed by the first person to share it β that rule is right, and refusing until
|
| 280 |
+
# somebody seeds an owner would make a brand-new folder unshareable by the person who just
|
| 281 |
+
# made it. What was missing is the half the old comment ASSERTED and the code never did: the
|
| 282 |
+
# claimant has to be able to reach the object. Without this, any signed-in account could
|
| 283 |
+
# stamp itself owner of an id it had never seen and lock the real creator out for good.
|
| 284 |
+
elif not _owns_object(session, kind, oid):
|
| 285 |
+
raise err(404, "no_object", "no such item, or it is not shared with this account")
|
| 286 |
+
entries = body.get("entries")
|
| 287 |
+
if not isinstance(entries, list):
|
| 288 |
+
raise err(400, "bad_entries",
|
| 289 |
+
"entries must be a list of {user, role} β send [] to un-share, which is how "
|
| 290 |
+
"revoking is expressed")
|
| 291 |
+
_entries_or_400(session, entries)
|
| 292 |
+
out = shares.set_grants(kind, oid, entries, owner=rec["owner"] or session.uname,
|
| 293 |
+
st=session.runtime)
|
| 294 |
+
_notify_new_grantees(session, kind, oid, before=rec["entries"], after=out.get("entries") or [])
|
| 295 |
+
return out
|
| 296 |
+
|
| 297 |
+
|
| 298 |
+
def _notify_new_grantees(session, kind, oid, before, after):
|
| 299 |
+
"""ββ W32-T28 (owner item 18's last clause, contract C3) οΏ½οΏ½ tell the RECEIVER, in their Inbox.
|
| 300 |
+
|
| 301 |
+
Owner item 18 ends *"being shared a database notifies the receiver"*. Until now sharing was
|
| 302 |
+
silent: the grant landed in a rail section the receiver had to notice on their own, which is
|
| 303 |
+
why "I shared it with you" and "I never saw it" were both true.
|
| 304 |
+
|
| 305 |
+
β WRITTEN ON THE SHARE, NEVER POLLED. `/notifications` re-evaluates view-ALERTS on read
|
| 306 |
+
because an alert is a live question about rows; a share is an EVENT that happened once, and
|
| 307 |
+
polling for it would mean re-deriving "was this new?" on every inbox open β the diff below
|
| 308 |
+
only exists here, at the moment the set changes.
|
| 309 |
+
|
| 310 |
+
β ONLY THE NEWLY ADDED. `PUT` REPLACES the whole entry set (revoking is expressed by absence),
|
| 311 |
+
so every save re-sends everyone who was already there. Diffing against `before` is what stops
|
| 312 |
+
a rename or a role change from ringing the bell for people whose access did not change.
|
| 313 |
+
β `*` IS NOT NOTIFIED: there is no user to name, and minting one notification per account in
|
| 314 |
+
the tenant on a single click is a broadcast nobody asked for. The rail still shows it.
|
| 315 |
+
β IT NEVER RAISES. A notification that fails must not fail the share that triggered it β the
|
| 316 |
+
grant is the user's actual intent, and `core.alerts.notify` writes with `flush='async'`.
|
| 317 |
+
"""
|
| 318 |
+
try:
|
| 319 |
+
was = {e.get("user") for e in (before or ()) if isinstance(e, dict)}
|
| 320 |
+
fresh = [str(e.get("user")) for e in (after or ())
|
| 321 |
+
if isinstance(e, dict) and e.get("user") not in was
|
| 322 |
+
and e.get("user") != shares.EVERYONE]
|
| 323 |
+
if not fresh:
|
| 324 |
+
return
|
| 325 |
+
import core.alerts as alerts
|
| 326 |
+
|
| 327 |
+
label, route, view_id = _object_ref(session, kind, oid)
|
| 328 |
+
if not route:
|
| 329 |
+
# β NO ROUTE, NO NOTIFICATION β the receiver would get a row that opens nothing, and
|
| 330 |
+
# `notification_view` would have to invent a target. Silence is the honest answer
|
| 331 |
+
# here; the rail still shows the grant under "Shared with me".
|
| 332 |
+
return
|
| 333 |
+
sharer = str(session.user.get("name") or session.uname)
|
| 334 |
+
for user in fresh:
|
| 335 |
+
# β THE SHAPE IS `routes_alerts.notification_view`'s SHARE BRANCH, and the two must
|
| 336 |
+
# agree or the Inbox row is unclickable: `topic` selects the branch and `key` becomes
|
| 337 |
+
# `alertId`, which that branch reads as the id to open. Both constants are IMPORTED
|
| 338 |
+
# from there rather than typed again β one vocabulary, one owner.
|
| 339 |
+
# ββ W33-T28 (`ASK C-14`, answered) β `actor` IS THE SENDER, AND IT IS THE ONLY WAY
|
| 340 |
+
# THE INBOX CAN NAME ONE. An alert and an automation have no person behind them and
|
| 341 |
+
# are honestly named by their machine; a SHARE has a real person, and only this call
|
| 342 |
+
# site knows who. β It is passed as its OWN field rather than recovered from the
|
| 343 |
+
# `detail` prose below: a sender parsed out of "<name> shared this with you" breaks
|
| 344 |
+
# the first time the sentence is reworded, silently, in the header
|
| 345 |
+
# [[grep-output-is-not-source]]. The prose stays as the body; this is the From.
|
| 346 |
+
alerts.notify(user, label, topic=_SHARE_TOPIC, key=route, row_id=view_id,
|
| 347 |
+
detail=f"{sharer} shared this with you", actor=sharer,
|
| 348 |
+
st=session.runtime)
|
| 349 |
+
except Exception: # noqa: BLE001
|
| 350 |
+
return
|
| 351 |
+
|
| 352 |
+
|
| 353 |
+
def _object_ref(session, kind, oid):
|
| 354 |
+
"""`(label, route, view_id)` β what to CALL the shared thing, and where it OPENS.
|
| 355 |
+
|
| 356 |
+
β THE ROUTE IS RESOLVED HERE, NOT SHAPED IN THE CONSUMER, AND THE FIRST VERSION GOT IT
|
| 357 |
+
WRONG: it put the raw `oid` in the notification's key, so a shared VIEW produced
|
| 358 |
+
`target: {module: "database", id: "view_42"}` β an instruction to open a database named
|
| 359 |
+
`view_42`. It read perfectly in the payload and would have opened nothing. **A view is not
|
| 360 |
+
addressable on its own; it is a SELECTION inside a topic's grid**, so the pair is what has to
|
| 361 |
+
travel. Caught by looking at the notification the driver actually produced, not by reading
|
| 362 |
+
the code back.
|
| 363 |
+
|
| 364 |
+
β `label` never falls back to a raw id. A notification headed `ut_leads_3f2a` tells the
|
| 365 |
+
receiver nothing they can act on, and the id is already in the target.
|
| 366 |
+
β An unresolvable object answers `route=None`, and the caller then sends NOTHING rather than
|
| 367 |
+
a row that opens nowhere.
|
| 368 |
+
"""
|
| 369 |
+
try:
|
| 370 |
+
if kind == "database":
|
| 371 |
+
import core.user_tables as ut
|
| 372 |
+
defn = (ut.all_defs(st=session.runtime) or {}).get(str(oid)) or {}
|
| 373 |
+
# A user table IS its own route key in both vocabularies (`route_for_topic`).
|
| 374 |
+
return (str(defn.get("label") or "").strip() or "A database", str(oid), "")
|
| 375 |
+
import core.table_store as table_store
|
| 376 |
+
from routes_alerts import route_for_topic
|
| 377 |
+
for topic in _topics(session):
|
| 378 |
+
ops = table_store.make(f"{topic}_table_workspace", st=session.runtime)
|
| 379 |
+
hit = ops.find_view(oid) if kind == "view" else ops.find_folder(oid)
|
| 380 |
+
if not hit:
|
| 381 |
+
continue
|
| 382 |
+
route = route_for_topic(topic)
|
| 383 |
+
if not route:
|
| 384 |
+
break
|
| 385 |
+
row = hit[1] if len(hit) > 1 else {}
|
| 386 |
+
name = str((row or {}).get("name") or "").strip()
|
| 387 |
+
# β Only a VIEW carries a selection. A folder is a rail grouping, so the target opens
|
| 388 |
+
# the grid and stops there rather than naming a view the receiver did not get.
|
| 389 |
+
return (name or ("A view" if kind == "view" else "A folder"),
|
| 390 |
+
route, str(oid) if kind == "view" else "")
|
| 391 |
+
except Exception: # noqa: BLE001
|
| 392 |
+
pass
|
| 393 |
+
return ({"view": "A view", "folder": "A folder"}.get(kind, "An item"), None, "")
|
api/routes_slack.py
CHANGED
|
@@ -229,7 +229,10 @@ def slack_creds(rt):
|
|
| 229 |
def _summary(agent, modules):
|
| 230 |
"""One sentence per agent for the list row β computed here, because the alternative is one
|
| 231 |
round trip per row to fill one cell (the same reason `AdminUser.access` exists)."""
|
| 232 |
-
|
|
|
|
|
|
|
|
|
|
| 233 |
perms = agent.get("perms") or {}
|
| 234 |
open_n = sum(1 for m in governed if (perms.get(m["key"]) or {}).get("access"))
|
| 235 |
if not governed:
|
|
@@ -304,16 +307,26 @@ def get_channel_agent(agent_id: str, session: Session = Depends(admin_gate)):
|
|
| 304 |
if not isinstance(a, dict):
|
| 305 |
raise err(404, "no_such_agent", "no agent with that id")
|
| 306 |
modules = routes_admin._perm_modules(session)
|
| 307 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 308 |
stored = a.get("perms") or {}
|
| 309 |
principal = agent_principal(a)
|
| 310 |
import core.perm_scope as perm_scope
|
| 311 |
perms_out = {}
|
| 312 |
-
for k in
|
| 313 |
e = stored.get(k)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 314 |
perms_out[k] = e if isinstance(e, dict) else {
|
| 315 |
-
"access": bool(perm_scope.
|
| 316 |
-
"hiddenFields": []}
|
| 317 |
return {"id": str(agent_id), "channel": a.get("channel") or "",
|
| 318 |
"channelName": a.get("channelName") or "", "label": a.get("label") or "",
|
| 319 |
"active": a.get("active", True) is not False,
|
|
@@ -326,7 +339,8 @@ def get_channel_agent(agent_id: str, session: Session = Depends(admin_gate)):
|
|
| 326 |
# the "Everything (admin)" state for a principal that cannot have it.
|
| 327 |
"is_admin": False,
|
| 328 |
"modules": modules,
|
| 329 |
-
"fields_by_module": {k: routes_admin._module_fields(k
|
|
|
|
| 330 |
|
| 331 |
|
| 332 |
@router.put("/agents/{agent_id}/perms")
|
|
@@ -341,14 +355,16 @@ def put_channel_agent_perms(agent_id: str, body: dict = Body(default=None),
|
|
| 341 |
raise err(400, "empty_patch", "no perms to save")
|
| 342 |
mods = routes_admin._perm_modules(session)
|
| 343 |
# β THE SAME `_clean_perms`, NOT A COPY OF IT. It refuses a filter this module cannot
|
| 344 |
-
# evaluate, refuses a hiddenFields key that names nothing, refuses
|
| 345 |
-
#
|
| 346 |
-
#
|
| 347 |
-
#
|
|
|
|
|
|
|
|
|
|
| 348 |
cleaned = routes_admin._clean_perms(
|
| 349 |
body.get("perms"),
|
| 350 |
-
|
| 351 |
-
listed_keys={m["key"] for m in mods}) or {}
|
| 352 |
|
| 353 |
import core.perm_scope as perm_scope
|
| 354 |
|
|
|
|
| 229 |
def _summary(agent, modules):
|
| 230 |
"""One sentence per agent for the list row β computed here, because the alternative is one
|
| 231 |
round trip per row to fill one cell (the same reason `AdminUser.access` exists)."""
|
| 232 |
+
# β W36-T22 / C2 β every listed database is governed now; the `enforced` flag it used to
|
| 233 |
+
# filter on is DELETED, and `m.get("enforced")` would have gone silently falsy here and
|
| 234 |
+
# summarised every agent as "" (no databases at all).
|
| 235 |
+
governed = list(modules)
|
| 236 |
perms = agent.get("perms") or {}
|
| 237 |
open_n = sum(1 for m in governed if (perms.get(m["key"]) or {}).get("access"))
|
| 238 |
if not governed:
|
|
|
|
| 307 |
if not isinstance(a, dict):
|
| 308 |
raise err(404, "no_such_agent", "no agent with that id")
|
| 309 |
modules = routes_admin._perm_modules(session)
|
| 310 |
+
# ββ W36-T22 / C2 β EVERY listed database, `ut_*` included. β THIS IS WHY THE FLAG'S
|
| 311 |
+
# DELETION IS NOT A ONE-FILE CHANGE: `m.get("enforced")` on a row that no longer carries the
|
| 312 |
+
# key is None, so this list would have been EMPTY and the channel perms editor would have
|
| 313 |
+
# governed ZERO databases while looking entirely correct. A Slack channel is a principal of
|
| 314 |
+
# the SAME wall (`verify_perm_scope` section H) and gets the same catalogue.
|
| 315 |
+
governed = [m["key"] for m in modules]
|
| 316 |
stored = a.get("perms") or {}
|
| 317 |
principal = agent_principal(a)
|
| 318 |
import core.perm_scope as perm_scope
|
| 319 |
perms_out = {}
|
| 320 |
+
for k in governed:
|
| 321 |
e = stored.get(k)
|
| 322 |
+
# β W36-T22 β `may_read`, the same evaluator the read door uses, for the reason spelled
|
| 323 |
+
# out at `routes_admin.get_perms`. β It answers DIFFERENTLY here and correctly so: a
|
| 324 |
+
# channel agent is not a person with a share, so a `ut_*` key it has not been granted
|
| 325 |
+
# defaults CLOSED β which is the fail-closed direction for a bot, and the same answer the
|
| 326 |
+
# table routes would give it.
|
| 327 |
perms_out[k] = e if isinstance(e, dict) else {
|
| 328 |
+
"access": bool(perm_scope.may_read(principal, k, st=session.runtime)),
|
| 329 |
+
"filter": None, "hiddenFields": []}
|
| 330 |
return {"id": str(agent_id), "channel": a.get("channel") or "",
|
| 331 |
"channelName": a.get("channelName") or "", "label": a.get("label") or "",
|
| 332 |
"active": a.get("active", True) is not False,
|
|
|
|
| 339 |
# the "Everything (admin)" state for a principal that cannot have it.
|
| 340 |
"is_admin": False,
|
| 341 |
"modules": modules,
|
| 342 |
+
"fields_by_module": {k: routes_admin._module_fields(k, session=session)
|
| 343 |
+
for k in governed}}
|
| 344 |
|
| 345 |
|
| 346 |
@router.put("/agents/{agent_id}/perms")
|
|
|
|
| 355 |
raise err(400, "empty_patch", "no perms to save")
|
| 356 |
mods = routes_admin._perm_modules(session)
|
| 357 |
# β THE SAME `_clean_perms`, NOT A COPY OF IT. It refuses a filter this module cannot
|
| 358 |
+
# evaluate, refuses a hiddenFields key that names nothing, and refuses a BU condition the
|
| 359 |
+
# pushdown cannot read. Every one of those is exactly as true for a channel as for a person,
|
| 360 |
+
# and a second validator here is a second place for one of them to go missing.
|
| 361 |
+
# β CORRECTED W36-T22: this list used to end "refuses an unenforced `ut_*` key". That refusal
|
| 362 |
+
# is DELETED β W36-T21 armed the wall over every database, so there is no unenforced key left
|
| 363 |
+
# to refuse, and a comment naming a rule that no longer exists is how the next wave re-derives
|
| 364 |
+
# it ([[two-gates-can-assert-opposite-things]]).
|
| 365 |
cleaned = routes_admin._clean_perms(
|
| 366 |
body.get("perms"),
|
| 367 |
+
governed_keys={m["key"] for m in mods}, session=session) or {}
|
|
|
|
| 368 |
|
| 369 |
import core.perm_scope as perm_scope
|
| 370 |
|
api/routes_statements.py
CHANGED
|
@@ -129,9 +129,14 @@ def preview(body: dict = Body(default=None), session: Session = Depends(_gate)):
|
|
| 129 |
subject = (t.get("subject") or cs.DEFAULT_SUBJECT)
|
| 130 |
try:
|
| 131 |
subject = subject.format(customer=row["Customer"], company=cs.COMPANY, month=month)
|
| 132 |
-
except
|
| 133 |
# An unknown placeholder is the user's typo, not a 500. Show the template verbatim so they
|
| 134 |
# can see what they typed rather than getting an opaque error.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 135 |
pass
|
| 136 |
return {
|
| 137 |
"html": cs.render_statement_html(row, t.get("intro") or cs.DEFAULT_INTRO,
|
|
@@ -153,7 +158,31 @@ def send(body: dict = Body(default=None), session: Session = Depends(_gate)):
|
|
| 153 |
names = [str(n) for n in (body.get("customers") or []) if str(n).strip()]
|
| 154 |
if not names:
|
| 155 |
raise err(400, "bad_request", "name at least one customer")
|
| 156 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 157 |
if override and len(names) != 1:
|
| 158 |
raise err(400, "bad_request", "a test send takes exactly one customer")
|
| 159 |
t = body.get("templates") or {}
|
|
|
|
| 129 |
subject = (t.get("subject") or cs.DEFAULT_SUBJECT)
|
| 130 |
try:
|
| 131 |
subject = subject.format(customer=row["Customer"], company=cs.COMPANY, month=month)
|
| 132 |
+
except Exception: # noqa: BLE001
|
| 133 |
# An unknown placeholder is the user's typo, not a 500. Show the template verbatim so they
|
| 134 |
# can see what they typed rather than getting an opaque error.
|
| 135 |
+
# β W36-T42: THIS CAUGHT THREE OF THE WAYS `str.format` FAILS AND THERE ARE MORE.
|
| 136 |
+
# `{customer` is a ValueError, `{customer.x}` an AttributeError, `{customer:%Y}` a
|
| 137 |
+
# TypeError, and every one of them is the same user mistake this branch was written for.
|
| 138 |
+
# Naming a subset of the exception types turns a typo in the OTHER half into a 500 on the
|
| 139 |
+
# one screen somebody opens to avoid mailing 200 people the wrong thing.
|
| 140 |
pass
|
| 141 |
return {
|
| 142 |
"html": cs.render_statement_html(row, t.get("intro") or cs.DEFAULT_INTRO,
|
|
|
|
| 158 |
names = [str(n) for n in (body.get("customers") or []) if str(n).strip()]
|
| 159 |
if not names:
|
| 160 |
raise err(400, "bad_request", "name at least one customer")
|
| 161 |
+
# ββ W36-T42 / D-299 β A DECLARED TEST SEND WITH NO ADDRESS REFUSES. IT DOES NOT BECOME A
|
| 162 |
+
# REAL ONE. This read `(body.get("overrideTo") or "").strip() or None`, so a caller who sent
|
| 163 |
+
# `overrideTo: ""` (the field left blank, the state not yet typed into, a trimmed-away space)
|
| 164 |
+
# got a REAL statement mailed to the debtor's own address, reported back as `test: false`. On
|
| 165 |
+
# the one route in this product that is allowed to write to Odoo, the difference between a
|
| 166 |
+
# rehearsal and mailing a live customer was one empty string.
|
| 167 |
+
#
|
| 168 |
+
# β THE DISCRIMINATOR IS THE CALLER'S DECLARATION, NOT THE VALUE. A body with no `overrideTo`
|
| 169 |
+
# key and no `test` flag is the PRODUCTION send, and refusing that would delete the feature
|
| 170 |
+
# D-299 exists to preserve. What can be refused is a caller who SAID this is a test: the key
|
| 171 |
+
# being present, or `test: true`, is that statement, and an empty address beside it is the
|
| 172 |
+
# mistake. So both spellings of "absent" are covered: the key present and blank, and `test`
|
| 173 |
+
# asserted with no key at all.
|
| 174 |
+
#
|
| 175 |
+
# β THIS IS THE SECOND OF THREE WALLS AND THE ONLY ONE A PAYLOAD CANNOT ROUTE AROUND.
|
| 176 |
+
# `automationApi.testSendStatement` refuses an empty address before the request is built, and
|
| 177 |
+
# SAFE_MODE refuses an address outside the allow-list inside `queue_statement`. The client
|
| 178 |
+
# wall is bypassable by construction (anything can POST); this one is not.
|
| 179 |
+
declared_test = ("overrideTo" in body) or bool(body.get("test"))
|
| 180 |
+
override = str(body.get("overrideTo") or "").strip()
|
| 181 |
+
if declared_test and not override:
|
| 182 |
+
raise err(400, "no_override_address",
|
| 183 |
+
"a test send needs the address to send the test to. Without one this would "
|
| 184 |
+
"mail the customer's own address, which is the opposite of a test")
|
| 185 |
+
override = override or None
|
| 186 |
if override and len(names) != 1:
|
| 187 |
raise err(400, "bad_request", "a test send takes exactly one customer")
|
| 188 |
t = body.get("templates") or {}
|
api/routes_tables.py
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
api/routes_web_agent.py
CHANGED
|
@@ -22,10 +22,11 @@ admin-gated, because it spends money and because D-51 Β§5's authorisation postur
|
|
| 22 |
the tenant is authorised to use, at their instruction") is not something an ordinary member
|
| 23 |
should be able to commit the tenant to.
|
| 24 |
|
| 25 |
-
|
| 26 |
-
`
|
| 27 |
-
|
| 28 |
-
|
|
|
|
| 29 |
"""
|
| 30 |
from fastapi import APIRouter, Body, Depends
|
| 31 |
|
|
|
|
| 22 |
the tenant is authorised to use, at their instruction") is not something an ordinary member
|
| 23 |
should be able to commit the tenant to.
|
| 24 |
|
| 25 |
+
β MOUNTED, and the gate is what says so rather than this sentence: `main.py:80` imports it and
|
| 26 |
+
`main.py:316` includes it, and `verify_web_agent.py` asserts the route answers rather than trusting
|
| 27 |
+
either line. This paragraph read "NOT MOUNTED YET" for a whole wave after the mount landed, which
|
| 28 |
+
is the same class of stale claim as a green gate on a router nobody wired: three finished routers
|
| 29 |
+
once shipped 404-dead behind entirely green gates, and prose is not the control that stops it.
|
| 30 |
"""
|
| 31 |
from fastapi import APIRouter, Body, Depends
|
| 32 |
|
platform/aios_grid.py
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
platform/core/perm_scope.py
CHANGED
|
@@ -122,6 +122,19 @@ def hidden_keys(user, module, fields):
|
|
| 122 |
β This runs on every assembly, so it is a fixpoint over a handful of custom fields, not a
|
| 123 |
graph library. `MAX_PASSES` bounds a reference cycle the client would refuse to evaluate
|
| 124 |
anyway; without it a self-referential pair would spin here.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 125 |
"""
|
| 126 |
e = entry(user, module)
|
| 127 |
if perms.is_admin(user) or not e:
|
|
@@ -137,6 +150,9 @@ def hidden_keys(user, module, fields):
|
|
| 137 |
expr = f.get('formula')
|
| 138 |
if isinstance(expr, str) and expr:
|
| 139 |
refs[f['key']] = {m.strip() for m in _FORMULA_REF.findall(expr) if m.strip()}
|
|
|
|
|
|
|
|
|
|
| 140 |
|
| 141 |
MAX_PASSES = 12
|
| 142 |
for _ in range(MAX_PASSES):
|
|
@@ -452,3 +468,336 @@ def _group_dba_team(group):
|
|
| 452 |
if vals <= allowed:
|
| 453 |
return tid
|
| 454 |
return None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 122 |
β This runs on every assembly, so it is a fixpoint over a handful of custom fields, not a
|
| 123 |
graph library. `MAX_PASSES` bounds a reference cycle the client would refuse to evaluate
|
| 124 |
anyway; without it a self-referential pair would spin here.
|
| 125 |
+
|
| 126 |
+
ββ W36-T21 β AND A ROLLUP IS THE SAME LEAK ONE MECHANISM OVER, which matters now that this
|
| 127 |
+
closure runs on the `ut_*` databases rather than only on the two registry topics. A rollup
|
| 128 |
+
names a LINK COLUMN OF THIS TABLE (`rollup.link`) and aggregates a field on the table that
|
| 129 |
+
link points at β so `ut_odoo_customers.ar_outstanding` is *"sum `residual` over the invoices
|
| 130 |
+
this row links to"*. Hide `invoices` and keep `ar_outstanding` and the reader still learns
|
| 131 |
+
what the hidden link contains, in aggregate; the three outcomes are exactly the three the
|
| 132 |
+
formula argument above enumerates, and only "strip both" is coherent. Verified against the
|
| 133 |
+
real declarations (`odoo_relational.customer_fields`) rather than assumed: every rollup there
|
| 134 |
+
is either `{'link': <a link column on THIS table>, 'field': <a column on the TARGET table>}`
|
| 135 |
+
or a `source` topic aggregate, so `rollup.link` is the ONE same-table reference a rollup makes
|
| 136 |
+
and `rollup.field` is deliberately not treated as one β it names another database's column,
|
| 137 |
+
which has its own wall.
|
| 138 |
"""
|
| 139 |
e = entry(user, module)
|
| 140 |
if perms.is_admin(user) or not e:
|
|
|
|
| 150 |
expr = f.get('formula')
|
| 151 |
if isinstance(expr, str) and expr:
|
| 152 |
refs[f['key']] = {m.strip() for m in _FORMULA_REF.findall(expr) if m.strip()}
|
| 153 |
+
link = (f.get('rollup') or {}).get('link') if isinstance(f.get('rollup'), dict) else None
|
| 154 |
+
if isinstance(link, str) and link.strip():
|
| 155 |
+
refs.setdefault(f['key'], set()).add(link.strip())
|
| 156 |
|
| 157 |
MAX_PASSES = 12
|
| 158 |
for _ in range(MAX_PASSES):
|
|
|
|
| 468 |
if vals <= allowed:
|
| 469 |
return tid
|
| 470 |
return None
|
| 471 |
+
|
| 472 |
+
|
| 473 |
+
# ββ C1: THE ONE DOOR TO ANY DATABASE'S ROWS (wave 36, W36-T20) ββββββββββββββββββββββββββββββββ
|
| 474 |
+
#: ββ OWNER RULING R6, AND IT IS WHY THIS SECTION EXISTS AT ALL: *"EVERY database gets the same
|
| 475 |
+
#: permission logic, always"* β per-user field visibility AND row filtration on every database
|
| 476 |
+
#: carrying a unique id, whatever created it, with a NEW database inheriting it by construction
|
| 477 |
+
#: rather than by a list somebody maintains.
|
| 478 |
+
#:
|
| 479 |
+
#: β THE PRODUCT HAD TWO PERMISSION SYSTEMS AND ONLY ONE WAS ARMED. Everything above this line
|
| 480 |
+
#: walls the REGISTRY topics (`customer_data`, `product_data`) and is called only from the topic
|
| 481 |
+
#: assemblies. Every OTHER database is a `ut_*` table walled by `user_tables.may_open` alone β
|
| 482 |
+
#: creator, admin, or a `core.shares` grant β which is a BINARY door: you see all 31,418 rows of
|
| 483 |
+
#: `ut_odoo_invoices` or none of them. `perms.tenant_governable_modules`' docstring booked this
|
| 484 |
+
#: work in as many words (*"Arming `perm_scope` over `ut_*` β¦ booked, not faked"*), and owner
|
| 485 |
+
#: item 11 is that booking coming due.
|
| 486 |
+
#:
|
| 487 |
+
#: β AND THE PREMISE THE GRILL GOT WRONG, because the fix depends on it: those databases are NOT
|
| 488 |
+
#: user-created. Ten of them (`ut_odoo_invoices`, `β¦_orders`, `β¦_agents`, `β¦_accounts`, `β¦_bills`,
|
| 489 |
+
#: `β¦_vendors`, `β¦_order_lines`, `β¦_gl_lines`, `β¦_customers`, `β¦_products`) are generated by the
|
| 490 |
+
#: KEYCHAIN connector (`aios-web/api/odoo_relational.py`). **`ut_` is a storage prefix, not a
|
| 491 |
+
#: statement about origin**, and a wall keyed off it was reading a naming artefact as a security
|
| 492 |
+
#: boundary.
|
| 493 |
+
#:
|
| 494 |
+
#: ββ THE TWO QUESTIONS STAY TWO QUESTIONS. `may_open` answers *"IF you see this database"* and
|
| 495 |
+
#: is untouched by this section; C1 answers *"WHICH rows and fields"*. `may_read` below COMPOSES
|
| 496 |
+
#: them β it calls `may_open`, it does not reimplement it β because merging them is how this
|
| 497 |
+
#: codebase got two ideas of who owns a table once already (`user_tables.may_open`'s own wave-20
|
| 498 |
+
#: note). One resolver per question, asked in order.
|
| 499 |
+
|
| 500 |
+
|
| 501 |
+
class UnknownTable(LookupError):
|
| 502 |
+
"""No database in this tenant answers to that key.
|
| 503 |
+
|
| 504 |
+
β RAISED, NEVER RETURNED AS AN EMPTY LIST (contract C1). An empty list reads as *"this
|
| 505 |
+
database is empty"* β indistinguishable from a real empty table, and the caller least able to
|
| 506 |
+
notice is the one that wanted rows. This repo has shipped that exact silent-empty answer
|
| 507 |
+
before (`user_tables.all_defs`' own correction note; [[empty-answer-vs-unfinished-answer]]).
|
| 508 |
+
"""
|
| 509 |
+
|
| 510 |
+
|
| 511 |
+
class Denied(PermissionError):
|
| 512 |
+
"""This principal may not read this database at all. The IF question, answered by `may_read`."""
|
| 513 |
+
|
| 514 |
+
|
| 515 |
+
class Unresolvable(RuntimeError):
|
| 516 |
+
"""The rows exist and cannot be served under this call's constraints β R6's SECOND SENTENCE.
|
| 517 |
+
|
| 518 |
+
β STANDING RULE 1 IS TWO SENTENCES AND THE SECOND IS THE HALF THAT GETS DROPPED: *"if there
|
| 519 |
+
is lag or it can't be done, you need to explicitly tell me why and recommend a fix"*. So a
|
| 520 |
+
limit that genuinely cannot be removed is REPORTED with its cause and a recommendation, never
|
| 521 |
+
silently enforced as a short answer. Carries the same four keys
|
| 522 |
+
`routes_tables._PID_SCOPE_LIMIT` already puts on the wire, so a route can hand this straight
|
| 523 |
+
to a client without a second vocabulary ([[one-question-two-normalizers]]).
|
| 524 |
+
"""
|
| 525 |
+
|
| 526 |
+
def __init__(self, subject, effect, cause, recommendation):
|
| 527 |
+
self.subject, self.effect = subject, effect
|
| 528 |
+
self.cause, self.recommendation = cause, recommendation
|
| 529 |
+
super().__init__(f"{subject}: {effect}. {cause}. {recommendation}")
|
| 530 |
+
|
| 531 |
+
def as_limit(self):
|
| 532 |
+
"""The dict shape `routes_tables` puts in an assembly's `limits` list."""
|
| 533 |
+
return {"subject": self.subject, "effect": self.effect,
|
| 534 |
+
"cause": self.cause, "recommendation": self.recommendation}
|
| 535 |
+
|
| 536 |
+
|
| 537 |
+
#: Row readers DECLARED by the app layer, keyed by EXACT database key.
|
| 538 |
+
#: `reader(table_key, user, st) -> (fields, rows)`.
|
| 539 |
+
#:
|
| 540 |
+
#: β WHY A REGISTRY AND NOT AN IMPORT. `core` never imports up (`platform/ARCHITECTURE.md`), and
|
| 541 |
+
#: a registry TOPIC's rows are built by `modules/` + `aios_grid` behind an API-layer pool cache
|
| 542 |
+
#: (`routes_customers._pool_for`), which is two layers above this file. Same idiom `user_tables`
|
| 543 |
+
#: already uses for exactly this reason β `register_connected`, `register_read_through`,
|
| 544 |
+
#: `ROW_HOOKS`: *"`core` never imports up, so the app tells this layer rather than being
|
| 545 |
+
#: interrogated by it."*
|
| 546 |
+
_ROW_SOURCES = {}
|
| 547 |
+
|
| 548 |
+
#: THE reader for a read-through `ut_*` grid β one reader, because there is one mirror.
|
| 549 |
+
#: `reader(table_key, field_keys, st) -> rows`.
|
| 550 |
+
_MIRROR_READER = None
|
| 551 |
+
|
| 552 |
+
|
| 553 |
+
def register_rows(reader, *table_keys):
|
| 554 |
+
"""Declare who reads a NAMED database's rows. Returns the registered key set.
|
| 555 |
+
|
| 556 |
+
β The return value is the registrar's own answer on purpose: a public function whose only
|
| 557 |
+
caller is a `verify_*.py` file is a feature no user can reach, and this repo has a gate that
|
| 558 |
+
says so ([[reachable-is-not-the-same-as-built]]). Routing the read door through the write
|
| 559 |
+
door's return keeps one construction site of the set instead of two.
|
| 560 |
+
"""
|
| 561 |
+
for key in table_keys:
|
| 562 |
+
k = str(key or '').strip()
|
| 563 |
+
if k:
|
| 564 |
+
_ROW_SOURCES[k] = reader
|
| 565 |
+
return frozenset(_ROW_SOURCES)
|
| 566 |
+
|
| 567 |
+
|
| 568 |
+
def register_mirror(reader):
|
| 569 |
+
"""Declare THE reader for read-through `ut_*` grids (`routes_tables._read_through_rows`)."""
|
| 570 |
+
global _MIRROR_READER
|
| 571 |
+
_MIRROR_READER = reader
|
| 572 |
+
return _MIRROR_READER is not None
|
| 573 |
+
|
| 574 |
+
|
| 575 |
+
#: β `row_sources()` IS DELETED (W36-T24 / owner item 13), AND THE REASON IS THE ONE THIS WAVE
|
| 576 |
+
#: KEEPS FINDING. It returned `frozenset(_ROW_SOURCES)` under a docstring calling itself *"The ONE
|
| 577 |
+
#: list to read"* β and `register_rows` ALREADY returns exactly that, which is the same idiom
|
| 578 |
+
#: `user_tables.register_connected_prefix` uses and the same reason: routing the read door through
|
| 579 |
+
#: the write door's return keeps ONE construction site of the set. A second accessor beside it is a
|
| 580 |
+
#: parallel path with nothing of its own to say, and it shipped with no caller outside `verify_*.py`
|
| 581 |
+
#: β the shape that is whole, correct and unreachable ([[artifact-with-no-importer]]; reported by
|
| 582 |
+
#: the integrator's `web_reachability` pass, `mailbox/A.md` A-43). The registrar's return is the
|
| 583 |
+
#: read: `routes_grid._C1_ROW_SOURCES` is that value, held where it is registered.
|
| 584 |
+
|
| 585 |
+
|
| 586 |
+
def _ut():
|
| 587 |
+
import core.user_tables as user_tables
|
| 588 |
+
return user_tables
|
| 589 |
+
|
| 590 |
+
|
| 591 |
+
def may_read(user, table_key, st=None):
|
| 592 |
+
"""May this principal read `table_key` AT ALL β the IF question, on EVERY database.
|
| 593 |
+
|
| 594 |
+
β COMPOSED, NOT RE-DERIVED, and the order is the whole rule:
|
| 595 |
+
|
| 596 |
+
1. an admin reads everything (break-glass β `deps._user_for` hands back a hardcoded master
|
| 597 |
+
dict on a store outage and it will never carry a `perms` block);
|
| 598 |
+
2. an EXPLICIT stored `access: false` DENIES, on any database. This is the toggle owner
|
| 599 |
+
item 11 asks for, and it is a **deny-only overlay**: it may revoke a database the wall
|
| 600 |
+
below would admit, and it may never grant one that wall refuses;
|
| 601 |
+
3. a `ut_*` database defers to `user_tables.may_open` β creator, admin, or a `core.shares`
|
| 602 |
+
grant β UNMODIFIED. W36-T21: *"`may_open` still decides IF the database is visible."*
|
| 603 |
+
4. anything else is a registry topic and defers to `may_access` above.
|
| 604 |
+
|
| 605 |
+
ββ WHY ABSENCE MUST NOT DENY ON A `ut_*` KEY, which is the opposite of what leg 4 does.
|
| 606 |
+
`may_access` reads migrated-and-undeclared as DENY β correct for a topic, because
|
| 607 |
+
`routes_admin` writes an entry for every governable topic on every save. β NO `ut_*` ENTRY WAS
|
| 608 |
+
STORABLE AT ALL UNTIL W36-T22 β `_clean_perms` refused the key with `unenforced_module` β so
|
| 609 |
+
every record migrated before this wave carries no entry for any of them, and reading that
|
| 610 |
+
absence as a decision would revoke all ten keychain databases from every migrated account the
|
| 611 |
+
moment this arms. That is not R6, it is an outage. Leg 3 therefore asks the wall that HAS been
|
| 612 |
+
answering rather than the marker that has not, and it keeps being right AFTER the flag's
|
| 613 |
+
deletion: an admin who has never opened the editor for a database has still not decided
|
| 614 |
+
anything about it.
|
| 615 |
+
|
| 616 |
+
ββ PASS THE **PUBLIC** RECORD, NOT THE ONE OUT OF `users.json`. Leg 3 needs a username, and a
|
| 617 |
+
stored record is keyed BY username in that bucket and does not carry one INSIDE it β only
|
| 618 |
+
`core.users._public(uname, rec)` puts it there, which is what `deps.Session.user` holds. Hand
|
| 619 |
+
this the raw record and `may_open` gets a `None` viewer and fail-closes, so every `ut_*`
|
| 620 |
+
database reads as DENIED for an account that can open all of them. It fails in the SAFE
|
| 621 |
+
direction and is silently wrong, which is the worst pair to debug β it cost two call sites in
|
| 622 |
+
one afternoon: a gate double, and `routes_admin.get_perms`' own fix for this very outage.
|
| 623 |
+
"""
|
| 624 |
+
if perms.is_admin(user):
|
| 625 |
+
return True
|
| 626 |
+
e = entry(user, table_key)
|
| 627 |
+
if e is not None and not bool(e.get('access', True)):
|
| 628 |
+
return False
|
| 629 |
+
key = str(table_key or '')
|
| 630 |
+
if key.startswith(_ut().KEY_PREFIX):
|
| 631 |
+
return bool(_ut().may_open(key, (user or {}).get('username'), False, st=st))
|
| 632 |
+
return may_access(user, table_key)
|
| 633 |
+
|
| 634 |
+
|
| 635 |
+
def wall_declared(user, table_key):
|
| 636 |
+
"""Is a ROW or FIELD narrowing declared for this principal on this database?
|
| 637 |
+
|
| 638 |
+
β THE QUESTION A DOOR ASKS BEFORE SERVING ROWS IT CANNOT SCOPE. `perms.py` warned that a
|
| 639 |
+
stored `ut_*` wall would be INERT β *"the editor would say DENY, the table routes would keep
|
| 640 |
+
serving, and nothing anywhere would say so"*. A route that cannot apply C1 must therefore
|
| 641 |
+
REFUSE for a principal this returns True for, rather than serve the whole database. False for
|
| 642 |
+
an admin (they bypass the wall entirely) and for any record with no entry, so a door asking
|
| 643 |
+
this pays nothing and changes nothing for everybody who has no wall.
|
| 644 |
+
"""
|
| 645 |
+
if perms.is_admin(user):
|
| 646 |
+
return False
|
| 647 |
+
e = entry(user, table_key)
|
| 648 |
+
if not e:
|
| 649 |
+
return False
|
| 650 |
+
return bool(e.get('filter')) or bool(e.get('hiddenFields'))
|
| 651 |
+
|
| 652 |
+
|
| 653 |
+
def row_scope_applies(user, table_key):
|
| 654 |
+
"""Does a permanent ROW filter narrow this principal on this database?
|
| 655 |
+
|
| 656 |
+
β `wall_declared`'s narrower half, and it exists so a rows-free caller can SKIP building rows
|
| 657 |
+
it would only need in order to filter them. `routes_tables.scoped_pids` is that caller: its
|
| 658 |
+
whole point is that the pid set costs no row pass, and paying for one on every database
|
| 659 |
+
switch β for every account, walled or not β would undo W30-T30 to enforce a rule that applies
|
| 660 |
+
to almost nobody. Asked here rather than spelled out at the call site, so there is ONE
|
| 661 |
+
statement of when the row wall bites ([[one-question-two-normalizers]]).
|
| 662 |
+
"""
|
| 663 |
+
if perms.is_admin(user):
|
| 664 |
+
return False
|
| 665 |
+
return bool((entry(user, table_key) or {}).get('filter'))
|
| 666 |
+
|
| 667 |
+
|
| 668 |
+
def scoped_table(user, table_key, st=None, ctx=None):
|
| 669 |
+
"""ββ CONTRACT C1 β the rows of ANY database, already field-stripped and row-filtered for
|
| 670 |
+
`user`. Registry topic or `ut_*`; there is no third kind and no per-database branch.
|
| 671 |
+
|
| 672 |
+
rows = scoped_table(user, 'ut_odoo_invoices') # a keychain database
|
| 673 |
+
rows = scoped_table(user, 'customer_data') # a registry topic
|
| 674 |
+
|
| 675 |
+
`user` is a user RECORD (the dict `deps.Session.user` carries), not a username β the whole
|
| 676 |
+
wall is a pure function of that record. BOTH arguments are positional and REQUIRED: a caller
|
| 677 |
+
that forgets the principal must not run, because the only thing a defaulted one could mean is
|
| 678 |
+
"unscoped", which is the widening direction.
|
| 679 |
+
|
| 680 |
+
β FAIL-CLOSED, THREE WAYS, AND EACH IS A DIFFERENT EXCEPTION so a caller can answer with the
|
| 681 |
+
right status instead of guessing: `UnknownTable` (no such database β never an empty list),
|
| 682 |
+
`Denied` (the IF question said no), `Unresolvable` (the rows cannot be served and here is
|
| 683 |
+
why β standing rule 1's second sentence).
|
| 684 |
+
|
| 685 |
+
β NO CAP. A connected source is read THROUGH the mirror in full (standing rule 1); the only
|
| 686 |
+
thing that stops it is a population that exceeds one materialisation window, and that arrives
|
| 687 |
+
as `Unresolvable` carrying its cause and a recommendation rather than as a short answer.
|
| 688 |
+
|
| 689 |
+
β E's SCRIPT SANDBOX HOLDS NO SECOND PATH TO THE STORE (wiring W1), which is why this is
|
| 690 |
+
THE door rather than A door: everything a sandboxed script may read, it reads here, under the
|
| 691 |
+
CALLING user's scope (R5).
|
| 692 |
+
"""
|
| 693 |
+
_fields, rows = _scoped(user, table_key, st=st, ctx=ctx)
|
| 694 |
+
return rows
|
| 695 |
+
|
| 696 |
+
|
| 697 |
+
def scoped_fields(user, table_key, st=None):
|
| 698 |
+
"""The COLUMNS of any database this principal may see β C1's other half.
|
| 699 |
+
|
| 700 |
+
β IT IS NOT A CONVENIENCE, IT IS THE SECOND WIRE. `strip_row`'s own note above says it: the
|
| 701 |
+
field list and the row payload are two different wires, and narrowing one without the other
|
| 702 |
+
leaves the value sitting where anything can read it. A caller that must render a scoped table
|
| 703 |
+
needs both, and E cannot read a `ut_*` definition to learn its columns β the sandbox has no
|
| 704 |
+
second path to the store (W1). So both come from here, off one wall.
|
| 705 |
+
|
| 706 |
+
β On a `ut_*` database this reads the DEFINITION only β the projection, no rows (D-213). On a
|
| 707 |
+
registry topic it goes through the registered reader, which builds that topic's pool; the
|
| 708 |
+
pool is cached per scope on the tenant runtime, so it is a cache hit next to `scoped_table`.
|
| 709 |
+
"""
|
| 710 |
+
fields, _rows = _scoped(user, table_key, st=st, ctx=None, want_rows=False)
|
| 711 |
+
return fields
|
| 712 |
+
|
| 713 |
+
|
| 714 |
+
def _scoped(user, table_key, st=None, ctx=None, want_rows=True):
|
| 715 |
+
"""`(fields, rows)` β ONE evaluator behind both public doors, so they cannot disagree."""
|
| 716 |
+
key = str(table_key or '').strip()
|
| 717 |
+
if not key:
|
| 718 |
+
raise UnknownTable('a database key is required. This door will not guess which database '
|
| 719 |
+
'was meant')
|
| 720 |
+
if not may_read(user, key, st=st):
|
| 721 |
+
raise Denied(f"this account may not read '{key}'")
|
| 722 |
+
fields, rows = _read(key, user, st, want_rows)
|
| 723 |
+
# THE FIELD WALL β a TRANSITIVE closure, so hiding a column also hides every formula computed
|
| 724 |
+
# FROM it. Resolved ONCE and used for both wires; see `hidden_keys` for why a set difference
|
| 725 |
+
# is the wrong shape here.
|
| 726 |
+
hide = hidden_keys(user, key, fields)
|
| 727 |
+
if not want_rows:
|
| 728 |
+
return (visible_fields(fields, user, key) if hide else fields), []
|
| 729 |
+
# THE ROW WALL β `permits()`, so a permanent filter this evaluator cannot answer DENIES
|
| 730 |
+
# rather than being ignored. Evaluated against the UNSTRIPPED contract on purpose: a
|
| 731 |
+
# permanent filter may name a column the reader is not allowed to SEE, and dropping the
|
| 732 |
+
# predicate would widen the read rather than narrow it.
|
| 733 |
+
rows = apply_row_scope(rows, user, key, fields, ctx)
|
| 734 |
+
if hide:
|
| 735 |
+
fields = visible_fields(fields, user, key)
|
| 736 |
+
rows = [strip_row(r, hide) for r in rows]
|
| 737 |
+
return fields, rows
|
| 738 |
+
|
| 739 |
+
|
| 740 |
+
def _read(table_key, user, st, want_rows=True):
|
| 741 |
+
"""`(fields, rows)` BEFORE the wall β the app layer's reader, or core's own for a `ut_*`."""
|
| 742 |
+
reader = _ROW_SOURCES.get(table_key)
|
| 743 |
+
if reader is not None:
|
| 744 |
+
fields, rows = reader(table_key, user, st)
|
| 745 |
+
return list(fields or ()), list(rows or ())
|
| 746 |
+
ut = _ut()
|
| 747 |
+
if not table_key.startswith(ut.KEY_PREFIX):
|
| 748 |
+
# β A TOPIC WITH NO REGISTERED READER IS UNKNOWN, NOT EMPTY. In a process that never
|
| 749 |
+
# imported the API layer this is the honest answer: nothing here can build that pool.
|
| 750 |
+
raise UnknownTable(f"no database named '{table_key}' in this workspace, and no reader "
|
| 751 |
+
f"is registered for it")
|
| 752 |
+
return _read_user_table(table_key, st, want_rows)
|
| 753 |
+
|
| 754 |
+
|
| 755 |
+
def _read_user_table(table_key, st, want_rows=True):
|
| 756 |
+
"""core's OWN reader for a `ut_*` database. Answers with NO registrar, deliberately.
|
| 757 |
+
|
| 758 |
+
β WHY IT LIVES IN `core` RATHER THAN BEING REGISTERED LIKE THE TOPICS, and it is the same
|
| 759 |
+
argument that seeds `user_tables._CONNECTED_PREFIXES` rather than registering it: a cold
|
| 760 |
+
process β E's sandbox subprocess, a worker, a gate β that never imported an API route still
|
| 761 |
+
owes the right answer for `ut_odoo_invoices`. A registrar-only design would raise there, and
|
| 762 |
+
the sandbox is exactly such a process.
|
| 763 |
+
|
| 764 |
+
β THE WALL IS ANSWERED ON A PROJECTION AND THE ROWS ARE NOT. `lend_defs` serves definitions
|
| 765 |
+
without the 28.6 MB of rows (D-213), which is every read this function makes when
|
| 766 |
+
`want_rows` is false; a projected document RAISES on `rows` rather than answering empty, so
|
| 767 |
+
the materialised arm below takes the whole read explicitly.
|
| 768 |
+
"""
|
| 769 |
+
ut = _ut()
|
| 770 |
+
lent = ut.lend_defs(st)
|
| 771 |
+
defn = ut.get(table_key, st=lent)
|
| 772 |
+
if not defn:
|
| 773 |
+
raise UnknownTable(f"no database named '{table_key}'")
|
| 774 |
+
fields = [dict(f) for f in (defn.get('fields') or [])]
|
| 775 |
+
if not want_rows:
|
| 776 |
+
return fields, []
|
| 777 |
+
if not ut.materialises(table_key, st=st, defn=defn):
|
| 778 |
+
# A read-through grid stores no rows here β they live in the mirror, and reading
|
| 779 |
+
# `defn['rows']` would find an empty dict and serve an EMPTY GRID: correct-looking,
|
| 780 |
+
# wrong, and silent.
|
| 781 |
+
if _MIRROR_READER is None:
|
| 782 |
+
raise Unresolvable(
|
| 783 |
+
subject='rows', effect='unreadable',
|
| 784 |
+
cause=(f"'{table_key}' is served read-through from the connector mirror and no "
|
| 785 |
+
f'mirror reader is registered in this process'),
|
| 786 |
+
recommendation=('call `perm_scope.register_mirror(...)` from the app layer before '
|
| 787 |
+
'reading a read-through database, or read it through the API'))
|
| 788 |
+
keys = {f['key'] for f in fields if f.get('key')}
|
| 789 |
+
return fields, list(_MIRROR_READER(table_key, keys, st) or ())
|
| 790 |
+
whole = ut.get(table_key, st=st)
|
| 791 |
+
if whole is None:
|
| 792 |
+
# Deleted between the wall and here. The same refusal, not an empty table.
|
| 793 |
+
raise UnknownTable(f"no database named '{table_key}'")
|
| 794 |
+
field_keys = {f['key'] for f in fields if f.get('key')}
|
| 795 |
+
rows = []
|
| 796 |
+
for rid, row in (whole.get('rows') or {}).items():
|
| 797 |
+
if not str(rid).isdigit():
|
| 798 |
+
continue
|
| 799 |
+
r = {k: v for k, v in (row or {}).items() if k in field_keys}
|
| 800 |
+
r['pid'] = int(rid)
|
| 801 |
+
rows.append(r)
|
| 802 |
+
rows.sort(key=lambda r: r['pid'])
|
| 803 |
+
return fields, rows
|
platform/core/perms.py
CHANGED
|
@@ -269,19 +269,29 @@ def tenant_governable_modules(runtime, topics, ut_entries=()):
|
|
| 269 |
in; this function is the only place the tenant is applied.
|
| 270 |
`ut_entries` β this tenant's own `ut_*` databases, in `user_tables.nav_entries` shape.
|
| 271 |
|
| 272 |
-
Returns `[{'key', 'label'
|
| 273 |
-
|
| 274 |
-
|
| 275 |
-
|
| 276 |
-
|
| 277 |
-
|
| 278 |
-
|
| 279 |
-
|
| 280 |
-
|
| 281 |
-
|
| 282 |
-
|
| 283 |
-
|
| 284 |
-
`
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 285 |
"""
|
| 286 |
out = []
|
| 287 |
for t in topics or ():
|
|
@@ -293,20 +303,23 @@ def tenant_governable_modules(runtime, topics, ut_entries=()):
|
|
| 293 |
# "Odoo products") would have moved one of them and left the permission editor showing the
|
| 294 |
# old name with nothing to notice.
|
| 295 |
label = (t.get('label') or registry.BY_KEY.get(key, {}).get('label') or key)
|
| 296 |
-
out.append({'key': key, 'label': label
|
| 297 |
seen = {r['key'] for r in out}
|
| 298 |
for e in ut_entries or ():
|
| 299 |
key = str((e or {}).get('key') or '').strip()
|
| 300 |
if not key or key in seen:
|
| 301 |
continue
|
| 302 |
seen.add(key)
|
| 303 |
-
out.append({'key': key, 'label': (e.get('label') or key)
|
| 304 |
return out
|
| 305 |
|
| 306 |
|
| 307 |
-
|
| 308 |
-
|
| 309 |
-
|
|
|
|
|
|
|
|
|
|
| 310 |
|
| 311 |
|
| 312 |
def landing_page(user):
|
|
|
|
| 269 |
in; this function is the only place the tenant is applied.
|
| 270 |
`ut_entries` β this tenant's own `ut_*` databases, in `user_tables.nav_entries` shape.
|
| 271 |
|
| 272 |
+
Returns `[{'key', 'label'}]`.
|
| 273 |
+
|
| 274 |
+
ββ WAVE 36 (W36-T22 / CONTRACT C2 / OWNER RULING R6) β **`enforced` IS DELETED, NOT DEFAULTED
|
| 275 |
+
TO `True`.** This docstring carried the paragraph that booked the work, and it read:
|
| 276 |
+
|
| 277 |
+
*"`perm_scope` walls the REGISTRY topics β¦ It is never consulted for a `ut_*` database β¦
|
| 278 |
+
So a stored `ut_*` wall would be INERT β the editor would say DENY, the table routes would
|
| 279 |
+
keep serving, and nothing anywhere would say so β¦ Arming `perm_scope` over `ut_*` is a
|
| 280 |
+
change to `routes_tables`/`user_tables` β booked, not faked."*
|
| 281 |
+
|
| 282 |
+
That change is W36-T21 and it has landed: `perm_scope.scoped_table` is the ONE door to any
|
| 283 |
+
database's rows, `routes_tables` applies the row filter and the hidden-field closure on every
|
| 284 |
+
`ut_*` read, and a door that cannot apply them REFUSES rather than serving the lot. So there
|
| 285 |
+
is no longer a second class of database for the flag to distinguish, and **a flag that is
|
| 286 |
+
always true is a lie with a green gate behind it** β which is why C2 says DELETE rather than
|
| 287 |
+
default. Owner item 11, verbatim: *"EVERY database should be able to be toggleable by admin β¦
|
| 288 |
+
Everything that is a database with Unique ID, is always going to be configurable with regards
|
| 289 |
+
to Fields, and Filtration per User. Make this infrastructure robust. And always true when we
|
| 290 |
+
want to onboard more databases into our App."*
|
| 291 |
+
|
| 292 |
+
β AND THE LIST STAYS DERIVED. Topics come from the registry, `ut_*` from the tenant's own
|
| 293 |
+
`nav_entries` β so a database created after this wave is governable with no code change and no
|
| 294 |
+
list edit. That is the half of R6 a flag could never have delivered.
|
| 295 |
"""
|
| 296 |
out = []
|
| 297 |
for t in topics or ():
|
|
|
|
| 303 |
# "Odoo products") would have moved one of them and left the permission editor showing the
|
| 304 |
# old name with nothing to notice.
|
| 305 |
label = (t.get('label') or registry.BY_KEY.get(key, {}).get('label') or key)
|
| 306 |
+
out.append({'key': key, 'label': label})
|
| 307 |
seen = {r['key'] for r in out}
|
| 308 |
for e in ut_entries or ():
|
| 309 |
key = str((e or {}).get('key') or '').strip()
|
| 310 |
if not key or key in seen:
|
| 311 |
continue
|
| 312 |
seen.add(key)
|
| 313 |
+
out.append({'key': key, 'label': (e.get('label') or key)})
|
| 314 |
return out
|
| 315 |
|
| 316 |
|
| 317 |
+
#: β `enforced_module_keys` IS DELETED (W36-T22 / C2), NOT NEUTERED TO "every governable key".
|
| 318 |
+
#: It read `{m['key'] for m in modules if m.get('enforced')}`, so with the flag gone it would
|
| 319 |
+
#: answer the EMPTY SET and `_clean_perms` would refuse every module β and "fix" it by returning
|
| 320 |
+
#: everything is a function whose answer no longer depends on its argument, i.e. the always-true
|
| 321 |
+
#: flag wearing a different costume. Its one caller (`routes_admin._enforced_keys`) goes with it;
|
| 322 |
+
#: what replaces both is the module list itself, which is the only list there is now.
|
| 323 |
|
| 324 |
|
| 325 |
def landing_page(user):
|
platform/core/script_sandbox.py
ADDED
|
@@ -0,0 +1,519 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""core/script_sandbox.py β WAVE 36 (R5 / R10, contract C1): running a tenant's OWN Python.
|
| 2 |
+
|
| 3 |
+
Owner item 6: *"Add code script as an interface (database View) so a user can build whatever they
|
| 4 |
+
want through the Agent chat interface."* Item 8: *"We need to really guardrail the reach of this
|
| 5 |
+
script. So let's really grill this down."* R10 ruled it SERVER-SIDE PYTHON after the trade was
|
| 6 |
+
stated, so this file is the guardrail, and one engine serves both items.
|
| 7 |
+
|
| 8 |
+
ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 9 |
+
ββ THE ONE PARAGRAPH TO READ BEFORE CHANGING ANYTHING HERE.
|
| 10 |
+
|
| 11 |
+
In-process CPython cannot deliver two of this ticket's clauses. An AST allow-list plus a curated
|
| 12 |
+
namespace stops import, file, network and environment access β but it **cannot cap memory and
|
| 13 |
+
cannot interrupt a runaway loop**, because a `while True:` in the same interpreter is not a slow
|
| 14 |
+
request, it is the tenant's ONE FastAPI process gone. So the script runs in a **SUBPROCESS**:
|
| 15 |
+
`resource.setrlimit` for address space and CPU, a hard wall-clock kill from the parent, and the
|
| 16 |
+
allow-list inside. Neither half is sufficient; both are load-bearing.
|
| 17 |
+
|
| 18 |
+
β AND THE SUBPROCESS RECEIVES **ROWS, NEVER A STORE**. The parent calls C1's `scoped_table` under
|
| 19 |
+
the CALLING user's record and serialises the result; the child imports nothing from this repo and
|
| 20 |
+
holds no credential, no runtime and no store handle. Wiring W1 ("the sandbox has no second store
|
| 21 |
+
path") is then true by CONSTRUCTION rather than by discipline, and it is checkable: the child
|
| 22 |
+
reports its own `sys.modules`, and no `core.*` name may appear in it.
|
| 23 |
+
|
| 24 |
+
β NEVER A BLACKLIST. Every rule below is an ALLOW-LIST β a set of node types, a set of attribute
|
| 25 |
+
names, a dict of builtins. A blacklist of dangerous spellings is bypassable by construction, and
|
| 26 |
+
the bypass is usually one string method away (`"{0.__class__}".format(x)` performs its attribute
|
| 27 |
+
lookup inside `format`, so there is no `ast.Attribute` node to refuse).
|
| 28 |
+
ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 29 |
+
|
| 30 |
+
The two layers, and they refuse DIFFERENT things on purpose:
|
| 31 |
+
|
| 32 |
+
1. `check_source()` β a pure function over source text. Refuses a construct the language offers
|
| 33 |
+
and this sandbox does not: `import`, `class`, `with`, `async`, `yield`, `global`, and every
|
| 34 |
+
attribute name outside `ALLOWED_ATTRS`.
|
| 35 |
+
2. `SANDBOX_BUILTINS` β the names that resolve at all. `__import__`, `open`, `eval`, `exec`,
|
| 36 |
+
`compile`, `getattr`, `globals`, `vars` and `type` are simply absent, so a source that gets
|
| 37 |
+
past layer 1 still finds nothing to call.
|
| 38 |
+
|
| 39 |
+
β THAT DUPLICATION IS DELIBERATE AND IT CHANGES HOW THE GATE MUST BE WRITTEN. `import os` is
|
| 40 |
+
refused twice, so a negative control that drops ONE layer sees the other refuse and reports
|
| 41 |
+
green β the shape that already cost this wave one missed control in `routes_agent_harness`. So
|
| 42 |
+
each layer is tested AT ITS OWN BOUNDARY: `check_source()` is called directly on source strings,
|
| 43 |
+
and `run()` is driven end to end. An NC drops one entry from one frozenset and the matching
|
| 44 |
+
boundary goes red.
|
| 45 |
+
"""
|
| 46 |
+
import ast
|
| 47 |
+
import json
|
| 48 |
+
import os
|
| 49 |
+
import subprocess
|
| 50 |
+
import sys
|
| 51 |
+
import tempfile
|
| 52 |
+
import time
|
| 53 |
+
from pathlib import Path
|
| 54 |
+
|
| 55 |
+
#: Wall clock, enforced by the PARENT with a kill. The one cap that works on every platform.
|
| 56 |
+
DEFAULT_TIMEOUT_S = 10.0
|
| 57 |
+
|
| 58 |
+
#: Address space for the child (`RLIMIT_AS`). POSIX only β see `run()`'s `caps` report.
|
| 59 |
+
DEFAULT_MEMORY_BYTES = 512 * 1024 * 1024
|
| 60 |
+
|
| 61 |
+
#: CPU seconds for the child (`RLIMIT_CPU`). POSIX only. Deliberately above the wall clock: the
|
| 62 |
+
#: wall-clock kill is the primary control and this is the backstop for a child that stops being
|
| 63 |
+
#: reachable. A CPU limit BELOW the timeout would make every slow script look like a CPU refusal.
|
| 64 |
+
DEFAULT_CPU_SECONDS = 15
|
| 65 |
+
|
| 66 |
+
#: What the script may print, in bytes. `print` is a curated builtin writing to a capped buffer,
|
| 67 |
+
#: and the child's real stdout goes to DEVNULL β so a script cannot fill a pipe, and anything
|
| 68 |
+
#: that escaped far enough to write to fd 1 has nowhere for it to land.
|
| 69 |
+
MAX_STDOUT_BYTES = 64 * 1024
|
| 70 |
+
|
| 71 |
+
#: The serialised ROW payload handed to the child. β A REFUSAL, NEVER A TRUNCATION (standing rule
|
| 72 |
+
#: 1): a short answer from a data tool is a wrong answer that looks right. Over this, `run()`
|
| 73 |
+
#: returns a named limit carrying its cause and a recommendation.
|
| 74 |
+
MAX_PAYLOAD_BYTES = 32 * 1024 * 1024
|
| 75 |
+
|
| 76 |
+
#: The emitted spec. A render spec is a description of a picture; one larger than this is data
|
| 77 |
+
#: pretending to be a description.
|
| 78 |
+
MAX_SPEC_BYTES = 2 * 1024 * 1024
|
| 79 |
+
|
| 80 |
+
MAX_SOURCE_BYTES = 128 * 1024
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
# βββββββοΏ½οΏ½οΏ½ββββββββββββββββββββββββββββββββββββββββββββββ LAYER 1 β the AST allow-list βββββββββββ
|
| 84 |
+
#: Every `ast` node class a script may contain. β THE ABSENCES ARE THE POLICY: `Import` /
|
| 85 |
+
#: `ImportFrom` (no module reaches the script), `ClassDef` (a class body is a namespace with its
|
| 86 |
+
#: own scoping rules and buys a data script nothing), `With` (a context manager is `__enter__`
|
| 87 |
+
#: by another spelling), `Global` / `Nonlocal` (rebinding the sandbox's own names), and every
|
| 88 |
+
#: `Async*` / `Await` / `Yield` form (this engine is synchronous; a coroutine that is never
|
| 89 |
+
#: awaited is a silent no-op that looks like a working script).
|
| 90 |
+
ALLOWED_NODES = frozenset("""
|
| 91 |
+
Module Expr Assign AugAssign AnnAssign NamedExpr Return Pass Break Continue Delete Assert Raise
|
| 92 |
+
If For While Try TryStar ExceptHandler FunctionDef Lambda arguments arg keyword
|
| 93 |
+
BoolOp BinOp UnaryOp IfExp Dict Set List Tuple Starred Subscript Slice Compare Call Attribute Name
|
| 94 |
+
Constant JoinedStr FormattedValue ListComp SetComp DictComp GeneratorExp comprehension
|
| 95 |
+
Load Store Del
|
| 96 |
+
And Or Not Invert UAdd USub
|
| 97 |
+
Add Sub Mult Div FloorDiv Mod Pow LShift RShift BitOr BitXor BitAnd MatMult
|
| 98 |
+
Eq NotEq Lt LtE Gt GtE Is IsNot In NotIn
|
| 99 |
+
""".split())
|
| 100 |
+
|
| 101 |
+
#: Every attribute name a script may READ or CALL. ββ THIS IS THE LOAD-BEARING SET, and it is
|
| 102 |
+
#: an allow-list of NAMES rather than a refusal of dunders, because the interesting escapes are
|
| 103 |
+
#: ordinary-looking: `f.__globals__` on any function reaches the runner's own module namespace,
|
| 104 |
+
#: `e.__traceback__.tb_frame.f_globals` reaches it from an exception handler, and `().__class__`
|
| 105 |
+
#: reaches `object.__subclasses__`. None of those names is here, and neither is any name this
|
| 106 |
+
#: sandbox has not been asked for.
|
| 107 |
+
#: β `format` IS ABSENT DELIBERATELY. `"{0.__class__}".format(x)` performs the attribute lookup
|
| 108 |
+
#: INSIDE `str.format`, where no `ast.Attribute` node exists for layer 1 to see. f-strings are
|
| 109 |
+
#: fine β `f"{x.__class__}"` compiles to a real `Attribute` node and is refused.
|
| 110 |
+
ALLOWED_ATTRS = frozenset("""
|
| 111 |
+
append extend insert pop remove clear sort reverse copy count index
|
| 112 |
+
keys values items get setdefault update
|
| 113 |
+
add discard union intersection difference issubset issuperset
|
| 114 |
+
join split rsplit splitlines strip lstrip rstrip lower upper title capitalize casefold
|
| 115 |
+
replace startswith endswith find rfind zfill ljust rjust center partition removeprefix removesuffix
|
| 116 |
+
isdigit isalpha isalnum isspace isupper islower isnumeric
|
| 117 |
+
real imag numerator denominator
|
| 118 |
+
""".split())
|
| 119 |
+
|
| 120 |
+
|
| 121 |
+
class Refused(Exception):
|
| 122 |
+
"""A named refusal: `code` for a caller to branch on, `message` for a person to read."""
|
| 123 |
+
|
| 124 |
+
def __init__(self, code, message):
|
| 125 |
+
self.code, self.message = code, message
|
| 126 |
+
super().__init__(f"{code}: {message}")
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
def _attr_ok(name):
|
| 130 |
+
"""An attribute name passes only if it is on the list AND is not private.
|
| 131 |
+
|
| 132 |
+
β THE SECOND TEST IS NOT A BLACKLIST β it narrows an allow-list that already excludes every
|
| 133 |
+
private name. It is here so that adding a name to `ALLOWED_ATTRS` cannot open a dunder by
|
| 134 |
+
accident, which is the one edit a future reader is most likely to make in a hurry.
|
| 135 |
+
"""
|
| 136 |
+
return name in ALLOWED_ATTRS and not name.startswith("_")
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
def check_source(source):
|
| 140 |
+
"""LAYER 1. Return a `Refused` for source this sandbox will not run, or `None`.
|
| 141 |
+
|
| 142 |
+
β PURE, AND THAT IS WHAT MAKES IT TESTABLE AT ITS OWN BOUNDARY. It reads no file, spawns no
|
| 143 |
+
process and touches no store, so a gate can hand it a hundred hostile strings for free and an
|
| 144 |
+
NC can drop one entry from one frozenset and watch exactly this function change its answer.
|
| 145 |
+
"""
|
| 146 |
+
text = str(source or "")
|
| 147 |
+
if len(text.encode("utf-8", "replace")) > MAX_SOURCE_BYTES:
|
| 148 |
+
return Refused("source_too_long",
|
| 149 |
+
f"a script view is at most {MAX_SOURCE_BYTES // 1024} KB of source")
|
| 150 |
+
try:
|
| 151 |
+
tree = ast.parse(text)
|
| 152 |
+
except SyntaxError as exc:
|
| 153 |
+
return Refused("syntax", f"line {exc.lineno or 0}: {exc.msg}")
|
| 154 |
+
|
| 155 |
+
for node in ast.walk(tree):
|
| 156 |
+
kind = type(node).__name__
|
| 157 |
+
if kind not in ALLOWED_NODES:
|
| 158 |
+
return Refused("refused_construct",
|
| 159 |
+
f"line {getattr(node, 'lineno', 0)}: this sandbox does not run "
|
| 160 |
+
f"{_english(kind)}")
|
| 161 |
+
if isinstance(node, ast.Attribute) and not _attr_ok(node.attr):
|
| 162 |
+
return Refused("refused_attribute",
|
| 163 |
+
f"line {getattr(node, 'lineno', 0)}: the attribute "
|
| 164 |
+
f"'{node.attr}' is not available inside a script view")
|
| 165 |
+
# β A NAME may not be private either. `_` prefixed names are the runner's own, and a
|
| 166 |
+
# script that could bind one could shadow the machinery it runs on top of.
|
| 167 |
+
if isinstance(node, ast.Name) and node.id.startswith("_"):
|
| 168 |
+
return Refused("reserved_name",
|
| 169 |
+
f"line {getattr(node, 'lineno', 0)}: names starting with an "
|
| 170 |
+
f"underscore are reserved by the sandbox")
|
| 171 |
+
if isinstance(node, (ast.FunctionDef, ast.arg, ast.ExceptHandler)) and str(
|
| 172 |
+
getattr(node, "name", None) or getattr(node, "arg", "") or "").startswith("_"):
|
| 173 |
+
return Refused("reserved_name",
|
| 174 |
+
f"line {getattr(node, 'lineno', 0)}: names starting with an "
|
| 175 |
+
f"underscore are reserved by the sandbox")
|
| 176 |
+
if isinstance(node, ast.keyword) and str(node.arg or "").startswith("_"):
|
| 177 |
+
return Refused("reserved_name",
|
| 178 |
+
f"line {getattr(node, 'lineno', 0)}: keyword arguments starting with "
|
| 179 |
+
f"an underscore are reserved by the sandbox")
|
| 180 |
+
return None
|
| 181 |
+
|
| 182 |
+
|
| 183 |
+
_ENGLISH = {
|
| 184 |
+
"Import": "an import", "ImportFrom": "an import", "ClassDef": "a class definition",
|
| 185 |
+
"With": "a with block", "AsyncWith": "a with block", "AsyncFor": "an async loop",
|
| 186 |
+
"AsyncFunctionDef": "an async function", "Await": "await", "Yield": "yield",
|
| 187 |
+
"YieldFrom": "yield from", "Global": "a global statement", "Nonlocal": "a nonlocal statement",
|
| 188 |
+
"Match": "a match statement",
|
| 189 |
+
}
|
| 190 |
+
|
| 191 |
+
|
| 192 |
+
def _english(kind):
|
| 193 |
+
return _ENGLISH.get(kind, f"a {kind} expression")
|
| 194 |
+
|
| 195 |
+
|
| 196 |
+
# ββββββββββββββββββββββββββββββββββββββββββ LAYER 2 β the namespace, and the child program βββββ
|
| 197 |
+
#: The builtins a script may reach, BY NAME. Everything else is a `NameError` in the child.
|
| 198 |
+
#: β THE ABSENCES, again, are the policy: `__import__` `open` `eval` `exec` `compile` `input`
|
| 199 |
+
#: `getattr` `setattr` `delattr` `globals` `locals` `vars` `dir` `type` `super` `object` `help`
|
| 200 |
+
#: `exit` `breakpoint` `memoryview` `id`. Several are harmless on their own; each one is a step
|
| 201 |
+
#: on a published escape, and none has ever been asked for by a script that shapes rows.
|
| 202 |
+
#: β THE EXCEPTION CLASSES ARE HERE BECAUSE `try:` IS, and a `try` block whose `except` clause
|
| 203 |
+
#: cannot name what it catches is a construct that reads as supported and is not. They are safe
|
| 204 |
+
#: for the same reason everything else is: `Exception.__subclasses__` needs an attribute this
|
| 205 |
+
#: sandbox does not allow, so a class object in the namespace is a leaf, not a doorway.
|
| 206 |
+
SANDBOX_BUILTIN_NAMES = (
|
| 207 |
+
"abs all any bool bytes callable chr dict divmod enumerate filter float frozenset hash hex "
|
| 208 |
+
"int isinstance issubclass iter len list map max min next oct ord pow range repr reversed "
|
| 209 |
+
"round set slice sorted str sum tuple zip True False None "
|
| 210 |
+
"Exception ValueError TypeError KeyError IndexError ZeroDivisionError ArithmeticError "
|
| 211 |
+
"AttributeError StopIteration OverflowError"
|
| 212 |
+
).split()
|
| 213 |
+
|
| 214 |
+
#: The literal program the child runs. It is TEXT rather than a module because the child must
|
| 215 |
+
#: import nothing from this repo: a module would be found on `sys.path` and would drag `core`
|
| 216 |
+
#: with it, which is exactly the second store path W1 forbids.
|
| 217 |
+
#: β Every name in here is underscore-prefixed and layer 1 refuses a script from binding one, so
|
| 218 |
+
#: the runner's own machinery cannot be shadowed by the source it executes.
|
| 219 |
+
_RUNNER = r'''
|
| 220 |
+
import json as _json, os as _os, sys as _sys
|
| 221 |
+
|
| 222 |
+
_pay = _json.loads(open(_sys.argv[1], "r", encoding="utf-8").read())
|
| 223 |
+
_out = {"ok": False, "code": "not_run", "message": "the script did not run",
|
| 224 |
+
"stdout": "", "spec": None, "caps": {"wallClock": True, "memory": False, "cpu": False}}
|
| 225 |
+
|
| 226 |
+
# ββ the caps this platform can actually apply, reported either way (standing rule 1) ββββββββββ
|
| 227 |
+
try:
|
| 228 |
+
import resource as _res
|
| 229 |
+
_mem = int(_pay["memoryBytes"])
|
| 230 |
+
_res.setrlimit(_res.RLIMIT_AS, (_mem, _mem))
|
| 231 |
+
_out["caps"]["memory"] = True
|
| 232 |
+
_cpu = int(_pay["cpuSeconds"])
|
| 233 |
+
_res.setrlimit(_res.RLIMIT_CPU, (_cpu, _cpu))
|
| 234 |
+
_out["caps"]["cpu"] = True
|
| 235 |
+
except Exception:
|
| 236 |
+
# `resource` is POSIX only. The wall-clock kill in the parent still applies, and `caps` says
|
| 237 |
+
# which of the three held, never a silent partial.
|
| 238 |
+
pass
|
| 239 |
+
|
| 240 |
+
_printed = []
|
| 241 |
+
_spent = [0]
|
| 242 |
+
_LIMIT = int(_pay["maxStdout"])
|
| 243 |
+
|
| 244 |
+
|
| 245 |
+
def _print(*_a, **_k):
|
| 246 |
+
_text = (_k.get("sep") or " ").join(str(_x) for _x in _a) + (_k.get("end") or "\n")
|
| 247 |
+
_room = _LIMIT - _spent[0]
|
| 248 |
+
if _room > 0:
|
| 249 |
+
_printed.append(_text[:_room])
|
| 250 |
+
_spent[0] += len(_text)
|
| 251 |
+
|
| 252 |
+
|
| 253 |
+
class _Refusal(Exception):
|
| 254 |
+
"""The SANDBOX refusing, as distinct from the SCRIPT failing.
|
| 255 |
+
|
| 256 |
+
Without its own class these arrive as `ValueError`, indistinguishable from a `ValueError` the
|
| 257 |
+
script raised itself, and the answer then says "refused" about an ordinary bug in the tenant's
|
| 258 |
+
own code. Two different facts, two different codes.
|
| 259 |
+
"""
|
| 260 |
+
|
| 261 |
+
|
| 262 |
+
_emitted = []
|
| 263 |
+
|
| 264 |
+
|
| 265 |
+
def _emit(_spec):
|
| 266 |
+
if not isinstance(_spec, dict):
|
| 267 |
+
raise _Refusal("emit() takes a view spec, which is a dictionary")
|
| 268 |
+
if _emitted:
|
| 269 |
+
raise _Refusal("emit() was already called; a script view emits exactly one view")
|
| 270 |
+
_emitted.append(_spec)
|
| 271 |
+
|
| 272 |
+
|
| 273 |
+
_rows = _pay["rows"]
|
| 274 |
+
_fields = _pay["fields"]
|
| 275 |
+
_bound = _pay["table"]
|
| 276 |
+
|
| 277 |
+
|
| 278 |
+
def _scoped_table(_table=None):
|
| 279 |
+
if _table is not None and str(_table) != _bound:
|
| 280 |
+
raise _Refusal(
|
| 281 |
+
"this script view is bound to the database '" + _bound + "' and asked for '"
|
| 282 |
+
+ str(_table) + "'. A script view reads its own database only")
|
| 283 |
+
return [dict(_r) for _r in _rows]
|
| 284 |
+
|
| 285 |
+
|
| 286 |
+
def _scoped_fields():
|
| 287 |
+
return [dict(_f) for _f in _fields]
|
| 288 |
+
|
| 289 |
+
|
| 290 |
+
_ns = {"__builtins__": {_n: __builtins__[_n] if isinstance(__builtins__, dict)
|
| 291 |
+
else getattr(__builtins__, _n)
|
| 292 |
+
for _n in _pay["builtins"]}}
|
| 293 |
+
_ns["__builtins__"]["print"] = _print
|
| 294 |
+
_ns["print"] = _print
|
| 295 |
+
_ns["emit"] = _emit
|
| 296 |
+
_ns["scoped_table"] = _scoped_table
|
| 297 |
+
_ns["scoped_fields"] = _scoped_fields
|
| 298 |
+
_ns["table"] = _bound
|
| 299 |
+
|
| 300 |
+
try:
|
| 301 |
+
exec(compile(_pay["source"], "<script view>", "exec"), _ns)
|
| 302 |
+
if not _emitted:
|
| 303 |
+
_out.update(ok=False, code="no_view",
|
| 304 |
+
message="the script finished without calling emit(spec)")
|
| 305 |
+
else:
|
| 306 |
+
_out.update(ok=True, code="", message="", spec=_emitted[0])
|
| 307 |
+
except _Refusal as _e:
|
| 308 |
+
_out.update(ok=False, code="refused", message=str(_e)[:400])
|
| 309 |
+
except MemoryError:
|
| 310 |
+
_out.update(ok=False, code="memory",
|
| 311 |
+
message="the script used more memory than a script view is allowed")
|
| 312 |
+
except NameError as _e:
|
| 313 |
+
_out.update(ok=False, code="refused_name",
|
| 314 |
+
message=str(_e)[:200] + ". A script view may use only the names the sandbox "
|
| 315 |
+
"provides")
|
| 316 |
+
except BaseException as _e:
|
| 317 |
+
_out.update(ok=False, code="error",
|
| 318 |
+
message=type(_e).__name__ + ": " + str(_e)[:400])
|
| 319 |
+
|
| 320 |
+
_out["stdout"] = "".join(_printed)
|
| 321 |
+
_out["truncated"] = _spent[0] > _LIMIT
|
| 322 |
+
# β THE PROBE: what this child actually had. The PARENT strips it unless it was asked for, so a
|
| 323 |
+
# production run never carries it and a gate can still prove that no `core.*` module and no
|
| 324 |
+
# secret-shaped environment key was ever inside this process.
|
| 325 |
+
# β SNAPSHOTTED AFTER `exec`, and `os` is imported at the TOP so this list does not depend on
|
| 326 |
+
# dict-literal evaluation order. The first draft called `__import__("os")` inside this very
|
| 327 |
+
# expression, so whether `os` appeared depended on which value Python built first: a probe whose
|
| 328 |
+
# contents move with an unrelated edit is a probe a gate cannot assert against.
|
| 329 |
+
_out["probe"] = {"modules": sorted(_sys.modules), "env": sorted(_os.environ)}
|
| 330 |
+
open(_sys.argv[2], "w", encoding="utf-8").write(_json.dumps(_out, default=str))
|
| 331 |
+
'''
|
| 332 |
+
|
| 333 |
+
|
| 334 |
+
def _child_env():
|
| 335 |
+
"""The child's WHOLE environment. An allow-list of two keys, and neither is a credential.
|
| 336 |
+
|
| 337 |
+
β NOT `os.environ.copy()` MINUS SOMETHING. A subtractive environment ships every key nobody
|
| 338 |
+
thought to name: `HF_TOKEN`, `ODOO_PASSWORD`, `ANTHROPIC_API_KEY` and whatever the next
|
| 339 |
+
connector adds. The three names below are here because Python will not start on Windows
|
| 340 |
+
without them; on Linux this returns `{}` and the child runs with no environment at all.
|
| 341 |
+
"""
|
| 342 |
+
env = {}
|
| 343 |
+
for name in ("SystemRoot", "SYSTEMROOT", "WINDIR"):
|
| 344 |
+
if os.environ.get(name):
|
| 345 |
+
env[name] = os.environ[name]
|
| 346 |
+
return env
|
| 347 |
+
|
| 348 |
+
|
| 349 |
+
def run(source, rows, fields, table_key, *, timeout_s=DEFAULT_TIMEOUT_S,
|
| 350 |
+
memory_bytes=DEFAULT_MEMORY_BYTES, cpu_seconds=DEFAULT_CPU_SECONDS, probe=False):
|
| 351 |
+
"""Run ONE script over rows that are ALREADY scoped. Returns C3's envelope plus `caps`.
|
| 352 |
+
|
| 353 |
+
{ok, code, message, spec, stdout, truncated, ms, caps: {wallClock, memory, cpu}}
|
| 354 |
+
|
| 355 |
+
β THIS FUNCTION NEVER TOUCHES A STORE, AND THAT IS THE POINT: it takes rows. `run_view()`
|
| 356 |
+
below is the door that fetches them through C1; keeping the two apart is what lets a gate
|
| 357 |
+
drive the sandbox with no tenant, no runtime and no credential anywhere in the process.
|
| 358 |
+
|
| 359 |
+
β `caps` IS PART OF THE ANSWER, NOT DEBUG OUTPUT. On Windows `resource` does not exist, so
|
| 360 |
+
the memory and CPU limits are NOT applied and this says so. A caller that reports `ok:true`
|
| 361 |
+
without reading `caps` is claiming an enforcement that did not happen (standing rule 1).
|
| 362 |
+
"""
|
| 363 |
+
started = time.monotonic()
|
| 364 |
+
refusal = check_source(source)
|
| 365 |
+
if refusal is not None:
|
| 366 |
+
return _refusal(refusal.code, refusal.message, started)
|
| 367 |
+
|
| 368 |
+
payload = {"source": str(source or ""), "rows": rows, "fields": fields,
|
| 369 |
+
"table": str(table_key or ""), "builtins": SANDBOX_BUILTIN_NAMES,
|
| 370 |
+
"maxStdout": MAX_STDOUT_BYTES, "memoryBytes": int(memory_bytes),
|
| 371 |
+
"cpuSeconds": int(cpu_seconds)}
|
| 372 |
+
try:
|
| 373 |
+
blob = json.dumps(payload, default=str)
|
| 374 |
+
except (TypeError, ValueError) as exc:
|
| 375 |
+
return _refusal("bad_rows", f"these rows cannot be handed to a script ({exc})", started)
|
| 376 |
+
if len(blob.encode("utf-8", "replace")) > MAX_PAYLOAD_BYTES:
|
| 377 |
+
# β REPORTED, NOT TRUNCATED (standing rule 1's second sentence): cause and recommendation,
|
| 378 |
+
# in the words the owner asked for, rather than a quietly short answer.
|
| 379 |
+
return _refusal(
|
| 380 |
+
"payload_too_large",
|
| 381 |
+
f"this database's rows are larger than the {MAX_PAYLOAD_BYTES // (1024 * 1024)} MB a "
|
| 382 |
+
f"script view can be handed at once. Narrow the view with a filter, or raise the "
|
| 383 |
+
f"sandbox payload limit for this deployment", started)
|
| 384 |
+
|
| 385 |
+
with tempfile.TemporaryDirectory(prefix="aios-script-") as work:
|
| 386 |
+
pay_path = Path(work) / "payload.json"
|
| 387 |
+
res_path = Path(work) / "result.json"
|
| 388 |
+
pay_path.write_text(blob, encoding="utf-8")
|
| 389 |
+
# `-I` isolates the interpreter (no PYTHON* env, no user site), `-S` skips site-packages,
|
| 390 |
+
# and the program arrives on STDIN so there is no file for anything to import it as.
|
| 391 |
+
# β `-X utf8` AND AN EXPLICIT `encoding` ARE NOT TIDINESS. Without them this pipe is
|
| 392 |
+
# encoded with the parent's locale codec, which on this Windows box is cp1252: the runner
|
| 393 |
+
# text below cannot be represented in it and `subprocess.run` died with
|
| 394 |
+
# `UnicodeEncodeError` before the child ever started. A sandbox whose behaviour depends on
|
| 395 |
+
# the operator's locale is a sandbox with two behaviours. `-I` implies `-E`, so
|
| 396 |
+
# `PYTHONUTF8` in the environment could not have carried this, it has to be a flag.
|
| 397 |
+
argv = [sys.executable, "-I", "-S", "-X", "utf8", "-", str(pay_path), str(res_path)]
|
| 398 |
+
try:
|
| 399 |
+
done = subprocess.run(
|
| 400 |
+
argv, input=_RUNNER, text=True, encoding="utf-8", errors="replace",
|
| 401 |
+
cwd=work, env=_child_env(),
|
| 402 |
+
stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, timeout=timeout_s)
|
| 403 |
+
except subprocess.TimeoutExpired:
|
| 404 |
+
return _refusal("timeout",
|
| 405 |
+
f"the script ran longer than {timeout_s:g} seconds and was stopped",
|
| 406 |
+
started)
|
| 407 |
+
except OSError as exc:
|
| 408 |
+
return _refusal("no_sandbox",
|
| 409 |
+
f"a script view could not be started on this deployment ({exc})",
|
| 410 |
+
started)
|
| 411 |
+
|
| 412 |
+
if not res_path.is_file():
|
| 413 |
+
# The child died without writing an answer: an rlimit signal, an OOM kill, or a crash.
|
| 414 |
+
# β NAMED BY ITS RETURN CODE rather than reported as a generic failure β a memory kill
|
| 415 |
+
# and a bug in this file must not read identically to an operator.
|
| 416 |
+
return _refusal(*_died(done), started)
|
| 417 |
+
try:
|
| 418 |
+
out = json.loads(res_path.read_text(encoding="utf-8"))
|
| 419 |
+
except (OSError, ValueError) as exc:
|
| 420 |
+
return _refusal("unreadable", f"the script's answer could not be read ({exc})",
|
| 421 |
+
started)
|
| 422 |
+
|
| 423 |
+
out["ms"] = int((time.monotonic() - started) * 1000)
|
| 424 |
+
if out.get("ok"):
|
| 425 |
+
spec_error = _check_spec(out.get("spec"))
|
| 426 |
+
if spec_error:
|
| 427 |
+
out.update(ok=False, code="bad_spec", message=spec_error, spec=None)
|
| 428 |
+
if not probe:
|
| 429 |
+
out.pop("probe", None)
|
| 430 |
+
return out
|
| 431 |
+
|
| 432 |
+
|
| 433 |
+
def _died(done):
|
| 434 |
+
"""`(code, message)` for a child that produced no answer."""
|
| 435 |
+
rc = done.returncode
|
| 436 |
+
tail = " ".join((done.stderr or "").split())[-300:]
|
| 437 |
+
if rc in (-9, 137):
|
| 438 |
+
return "memory", "the script was stopped for using too much memory"
|
| 439 |
+
if rc in (-24, 152):
|
| 440 |
+
return "timeout", "the script used more processor time than a script view is allowed"
|
| 441 |
+
return "crashed", f"the script view engine stopped without an answer{': ' + tail if tail else ''}"
|
| 442 |
+
|
| 443 |
+
|
| 444 |
+
def _refusal(code, message, started):
|
| 445 |
+
return {"ok": False, "code": code, "message": message, "spec": None, "stdout": "",
|
| 446 |
+
"truncated": False, "ms": int((time.monotonic() - started) * 1000),
|
| 447 |
+
"caps": {"wallClock": True, "memory": False, "cpu": False}}
|
| 448 |
+
|
| 449 |
+
|
| 450 |
+
def _check_spec(spec):
|
| 451 |
+
"""C3: a spec is a DESCRIPTION the client draws. Never HTML, never a script, never a URL.
|
| 452 |
+
|
| 453 |
+
β THE CHECK IS ON THE KEYS, NOT ON THE STRING CONTENTS. Scanning values for `<script>` is a
|
| 454 |
+
blacklist and would pass `<SCR` + `IPT>`; refusing a spec that carries an `html`, `script`,
|
| 455 |
+
`src` or `onclick` key refuses the SHAPE that would let a renderer be talked into executing
|
| 456 |
+
something. The vocabulary of legal `kind`s is the ROUTE's business (W36-T37) β this is the
|
| 457 |
+
floor every caller gets whether or not the route above it remembers.
|
| 458 |
+
"""
|
| 459 |
+
if not isinstance(spec, dict):
|
| 460 |
+
return "the script emitted something that is not a view spec"
|
| 461 |
+
try:
|
| 462 |
+
blob = json.dumps(spec)
|
| 463 |
+
except (TypeError, ValueError):
|
| 464 |
+
return "the emitted view spec is not something the client can be sent"
|
| 465 |
+
if len(blob.encode("utf-8", "replace")) > MAX_SPEC_BYTES:
|
| 466 |
+
return (f"the emitted view spec is over {MAX_SPEC_BYTES // (1024 * 1024)} MB. A view spec "
|
| 467 |
+
f"describes a picture; it is not where the rows go")
|
| 468 |
+
banned = {"html", "innerhtml", "script", "src", "srcdoc", "href", "style", "onclick", "onload"}
|
| 469 |
+
found = sorted(k for k in _keys_of(spec) if str(k).lower() in banned)
|
| 470 |
+
if found:
|
| 471 |
+
return (f"a view spec may not carry {', '.join(found)}. The client DRAWS a spec, so a "
|
| 472 |
+
f"markup or URL key would be a script by another name")
|
| 473 |
+
return None
|
| 474 |
+
|
| 475 |
+
|
| 476 |
+
def _keys_of(value, depth=0):
|
| 477 |
+
"""Every key anywhere in a nested spec. Bounded, so a deep structure cannot spin this."""
|
| 478 |
+
if depth > 12:
|
| 479 |
+
return
|
| 480 |
+
if isinstance(value, dict):
|
| 481 |
+
for key, sub in value.items():
|
| 482 |
+
yield key
|
| 483 |
+
yield from _keys_of(sub, depth + 1)
|
| 484 |
+
elif isinstance(value, (list, tuple)):
|
| 485 |
+
for sub in value:
|
| 486 |
+
yield from _keys_of(sub, depth + 1)
|
| 487 |
+
|
| 488 |
+
|
| 489 |
+
# ββββββββββββββββββββββββββββββββββββββββββββββ THE DOOR β C1 is the ONLY way to a row βββββββββ
|
| 490 |
+
def run_view(user, table_key, source, st=None, **kw):
|
| 491 |
+
"""Fetch through C1 under `user`'s scope, then run the script over what came back (R5).
|
| 492 |
+
|
| 493 |
+
ββ THE FETCH HAPPENS IN THE PARENT AND ONLY ROWS CROSS INTO THE CHILD. That is wiring W1
|
| 494 |
+
made structural: the child has no runtime to ask, no store handle to open and no credential
|
| 495 |
+
to use, so "a script cannot read what its caller cannot read" is not a rule anybody has to
|
| 496 |
+
keep β there is no second path for it to be broken through.
|
| 497 |
+
|
| 498 |
+
β C1'S THREE EXCEPTIONS ARE ANSWERED, NEVER SWALLOWED. `UnknownTable`, `Denied` and
|
| 499 |
+
`Unresolvable` mean three different things to a person; collapsing them into "no rows" is the
|
| 500 |
+
silent-empty answer C1 was written to make impossible. `Unresolvable.as_limit()` is handed
|
| 501 |
+
through in the words it was raised with β standing rule 1's second sentence, verbatim.
|
| 502 |
+
"""
|
| 503 |
+
import core.perm_scope as perm_scope
|
| 504 |
+
|
| 505 |
+
key = str(table_key or "")
|
| 506 |
+
try:
|
| 507 |
+
rows = perm_scope.scoped_table(user, key, st=st)
|
| 508 |
+
fields = perm_scope.scoped_fields(user, key, st=st)
|
| 509 |
+
except perm_scope.UnknownTable as exc:
|
| 510 |
+
return _refusal("unknown_table", str(exc) or f"there is no database '{key}'",
|
| 511 |
+
time.monotonic())
|
| 512 |
+
except perm_scope.Denied as exc:
|
| 513 |
+
return _refusal("denied", str(exc) or "this account may not read that database",
|
| 514 |
+
time.monotonic())
|
| 515 |
+
except perm_scope.Unresolvable as exc:
|
| 516 |
+
out = _refusal("unresolvable", str(exc), time.monotonic())
|
| 517 |
+
out["limit"] = exc.as_limit()
|
| 518 |
+
return out
|
| 519 |
+
return run(source, rows, fields, key, **kw)
|
platform/core/shares.py
CHANGED
|
@@ -57,18 +57,26 @@ KINDS = ('view', 'folder', 'database')
|
|
| 57 |
#: the sentence holds. `require_session` plus the topic's own gate run first, and the
|
| 58 |
#: receiver's row scope and hidden-field closure are applied BEFORE any foreign view is
|
| 59 |
#: merged, so a grant can only narrow-or-equal what that account could already reach.
|
| 60 |
-
#: * `kind='database'` on a `ut_*` table β **THE SENTENCE
|
| 61 |
-
#:
|
| 62 |
-
#: `_clean_perms`
|
| 63 |
-
#:
|
| 64 |
-
#:
|
| 65 |
-
#:
|
|
|
|
| 66 |
#:
|
| 67 |
-
#:
|
| 68 |
-
#:
|
| 69 |
-
#:
|
| 70 |
-
#:
|
| 71 |
-
#:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 72 |
EVERYONE = '*'
|
| 73 |
|
| 74 |
ROLES = ('view', 'edit')
|
|
|
|
| 57 |
#: the sentence holds. `require_session` plus the topic's own gate run first, and the
|
| 58 |
#: receiver's row scope and hidden-field closure are applied BEFORE any foreign view is
|
| 59 |
#: merged, so a grant can only narrow-or-equal what that account could already reach.
|
| 60 |
+
#: * `kind='database'` on a `ut_*` table β ββ **THE SENTENCE HOLDS HERE TOO NOW, AND THAT IS
|
| 61 |
+
#: W36-T21 / OWNER RULING R6 (audit S-8, CLOSED).** It did not until wave 36, and the reason
|
| 62 |
+
#: is worth keeping: `routes_admin._clean_perms` 400'd any key outside
|
| 63 |
+
#: `("customer_data", "product_data")`, so no row filter and no hidden field could even be
|
| 64 |
+
#: DECLARED for a user table, `routes_tables.py` made zero `perm_scope` calls, and it passed
|
| 65 |
+
#: `hidden_keys=frozenset()`. A `database` grant was therefore ALL-OR-NOTHING β every row,
|
| 66 |
+
#: every column β and this registry was the only wall behind it.
|
| 67 |
#:
|
| 68 |
+
#: β WHAT CHANGED: `perm_scope.scoped_table` (contract C1) is the ONE door to any database's
|
| 69 |
+
#: rows. `routes_tables` applies the permanent filter before `pids` is taken and the transitive
|
| 70 |
+
#: hidden-field closure after `workspace_wire`, on EVERY `ut_*` read β the same code that walls
|
| 71 |
+
#: `customer_data` β and a door that cannot apply them REFUSES rather than serving the lot. So a
|
| 72 |
+
#: `database` grant is once again bounded by a second wall: it decides WHETHER an account reaches
|
| 73 |
+
#: the database, and C1 decides WHICH rows and columns it then sees. An `*` grant still admits
|
| 74 |
+
#: every account in the tenant, and each of them still only sees what their own wall allows.
|
| 75 |
+
#:
|
| 76 |
+
#: β `routes_shares.py`'s module docstring carries the OLD sentence at the other door and is in
|
| 77 |
+
#: no wave-36 fence β the audit's own fix was "say so at both", so one of the two is now stale.
|
| 78 |
+
#: Booked in `mailbox/C.md` (C-14) rather than edited across a fence
|
| 79 |
+
#: [[two-gates-can-assert-opposite-things]].
|
| 80 |
EVERYONE = '*'
|
| 81 |
|
| 82 |
ROLES = ('view', 'edit')
|
platform/core/store.py
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
platform/core/table_store.py
CHANGED
|
@@ -1,618 +1,746 @@
|
|
| 1 |
-
"""The generic per-user TABLE WORKSPACE store β the persistence half of the table-page factory.
|
| 2 |
-
|
| 3 |
-
One durable store key holds one table OBJECT's per-user Airtable-style state:
|
| 4 |
-
|
| 5 |
-
{username: {'views': {view_id: SavedView},
|
| 6 |
-
'fields': {field_key: Field}, # notes + custom_ + measure_ strata
|
| 7 |
-
'overlays': {str(pid): {field_key: value}}}}
|
| 8 |
-
|
| 9 |
-
`make(table_key)` returns the six operations a table page's host loop needs, closed over that
|
| 10 |
-
key. The Customer table's ops (`modules/customer_data.py`, key 'customer_table_workspace') are
|
| 11 |
-
these exact functions β the logic MOVED here 2026-07-27 so that duplicating the Customer table
|
| 12 |
-
pattern to a new object is a registry row + a config, not a copy of the store plumbing
|
| 13 |
-
(owner directive: the table-page factory).
|
| 14 |
-
|
| 15 |
-
A LIST (membership/formula semantics) is deliberately a different store from a VIEW
|
| 16 |
-
(presentation/query state) β see modules/customer_data.py's customer_lists key.
|
| 17 |
-
"""
|
| 18 |
-
import core.store as store
|
| 19 |
-
|
| 20 |
-
#: Wave-9 I17 β the SHARED bucket. Views whose permissions make them visible to anyone but
|
| 21 |
-
#: their creator live here instead of in a personal workspace, under a key that cannot collide
|
| 22 |
-
#: with a username (usernames come from core/users.py and are never dunder-wrapped; `is_shared`
|
| 23 |
-
#: guards it anyway). ONE HOME PER VIEW, never both: a view moved to personal is REMOVED from
|
| 24 |
-
#: here, and a view shared is removed from its creator's workspace. Two homes would mean two
|
| 25 |
-
#: divergent copies the moment either was edited.
|
| 26 |
-
SHARED_KEY = '__shared__'
|
| 27 |
-
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
|
| 34 |
-
|
| 35 |
-
|
| 36 |
-
|
| 37 |
-
|
| 38 |
-
|
| 39 |
-
|
| 40 |
-
|
| 41 |
-
|
| 42 |
-
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
|
| 55 |
-
|
| 56 |
-
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
|
| 71 |
-
|
| 72 |
-
return
|
| 73 |
-
|
| 74 |
-
|
| 75 |
-
def
|
| 76 |
-
"""
|
| 77 |
-
|
| 78 |
-
|
| 79 |
-
|
| 80 |
-
|
| 81 |
-
|
| 82 |
-
|
| 83 |
-
|
| 84 |
-
|
| 85 |
-
|
| 86 |
-
|
| 87 |
-
|
| 88 |
-
|
| 89 |
-
|
| 90 |
-
|
| 91 |
-
|
| 92 |
-
|
| 93 |
-
|
| 94 |
-
|
| 95 |
-
"""
|
| 96 |
-
|
| 97 |
-
|
| 98 |
-
|
| 99 |
-
|
| 100 |
-
|
| 101 |
-
|
| 102 |
-
|
| 103 |
-
|
| 104 |
-
|
| 105 |
-
|
| 106 |
-
"""
|
| 107 |
-
|
| 108 |
-
|
| 109 |
-
|
| 110 |
-
|
| 111 |
-
|
| 112 |
-
|
| 113 |
-
|
| 114 |
-
|
| 115 |
-
|
| 116 |
-
|
| 117 |
-
|
| 118 |
-
|
| 119 |
-
|
| 120 |
-
|
| 121 |
-
|
| 122 |
-
|
| 123 |
-
|
| 124 |
-
|
| 125 |
-
|
| 126 |
-
|
| 127 |
-
|
| 128 |
-
|
| 129 |
-
|
| 130 |
-
|
| 131 |
-
|
| 132 |
-
|
| 133 |
-
|
| 134 |
-
|
| 135 |
-
|
| 136 |
-
|
| 137 |
-
|
| 138 |
-
|
| 139 |
-
|
| 140 |
-
|
| 141 |
-
|
| 142 |
-
|
| 143 |
-
|
| 144 |
-
|
| 145 |
-
|
| 146 |
-
|
| 147 |
-
|
| 148 |
-
|
| 149 |
-
|
| 150 |
-
|
| 151 |
-
|
| 152 |
-
|
| 153 |
-
|
| 154 |
-
|
| 155 |
-
|
| 156 |
-
|
| 157 |
-
|
| 158 |
-
|
| 159 |
-
|
| 160 |
-
|
| 161 |
-
|
| 162 |
-
|
| 163 |
-
|
| 164 |
-
|
| 165 |
-
|
| 166 |
-
|
| 167 |
-
|
| 168 |
-
|
| 169 |
-
|
| 170 |
-
|
| 171 |
-
|
| 172 |
-
|
| 173 |
-
|
| 174 |
-
|
| 175 |
-
|
| 176 |
-
|
| 177 |
-
|
| 178 |
-
|
| 179 |
-
|
| 180 |
-
|
| 181 |
-
|
| 182 |
-
|
| 183 |
-
|
| 184 |
-
|
| 185 |
-
|
| 186 |
-
|
| 187 |
-
|
| 188 |
-
|
| 189 |
-
|
| 190 |
-
|
| 191 |
-
|
| 192 |
-
|
| 193 |
-
|
| 194 |
-
|
| 195 |
-
|
| 196 |
-
|
| 197 |
-
|
| 198 |
-
|
| 199 |
-
|
| 200 |
-
|
| 201 |
-
|
| 202 |
-
|
| 203 |
-
|
| 204 |
-
|
| 205 |
-
|
| 206 |
-
|
| 207 |
-
|
| 208 |
-
|
| 209 |
-
|
| 210 |
-
|
| 211 |
-
|
| 212 |
-
|
| 213 |
-
|
| 214 |
-
|
| 215 |
-
|
| 216 |
-
|
| 217 |
-
|
| 218 |
-
|
| 219 |
-
|
| 220 |
-
|
| 221 |
-
|
| 222 |
-
|
| 223 |
-
|
| 224 |
-
|
| 225 |
-
|
| 226 |
-
|
| 227 |
-
|
| 228 |
-
|
| 229 |
-
|
| 230 |
-
|
| 231 |
-
|
| 232 |
-
|
| 233 |
-
|
| 234 |
-
|
| 235 |
-
|
| 236 |
-
|
| 237 |
-
|
| 238 |
-
|
| 239 |
-
|
| 240 |
-
|
| 241 |
-
|
| 242 |
-
|
| 243 |
-
|
| 244 |
-
|
| 245 |
-
|
| 246 |
-
|
| 247 |
-
|
| 248 |
-
|
| 249 |
-
|
| 250 |
-
|
| 251 |
-
|
| 252 |
-
|
| 253 |
-
|
| 254 |
-
|
| 255 |
-
|
| 256 |
-
|
| 257 |
-
|
| 258 |
-
|
| 259 |
-
|
| 260 |
-
|
| 261 |
-
|
| 262 |
-
|
| 263 |
-
|
| 264 |
-
|
| 265 |
-
|
| 266 |
-
|
| 267 |
-
|
| 268 |
-
|
| 269 |
-
|
| 270 |
-
|
| 271 |
-
|
| 272 |
-
|
| 273 |
-
|
| 274 |
-
|
| 275 |
-
|
| 276 |
-
|
| 277 |
-
|
| 278 |
-
|
| 279 |
-
|
| 280 |
-
|
| 281 |
-
|
| 282 |
-
|
| 283 |
-
|
| 284 |
-
|
| 285 |
-
|
| 286 |
-
|
| 287 |
-
|
| 288 |
-
|
| 289 |
-
|
| 290 |
-
|
| 291 |
-
|
| 292 |
-
|
| 293 |
-
|
| 294 |
-
|
| 295 |
-
|
| 296 |
-
|
| 297 |
-
|
| 298 |
-
|
| 299 |
-
|
| 300 |
-
|
| 301 |
-
|
| 302 |
-
|
| 303 |
-
|
| 304 |
-
|
| 305 |
-
|
| 306 |
-
|
| 307 |
-
|
| 308 |
-
|
| 309 |
-
|
| 310 |
-
|
| 311 |
-
|
| 312 |
-
|
| 313 |
-
|
| 314 |
-
|
| 315 |
-
|
| 316 |
-
|
| 317 |
-
|
| 318 |
-
|
| 319 |
-
|
| 320 |
-
|
| 321 |
-
|
| 322 |
-
|
| 323 |
-
|
| 324 |
-
|
| 325 |
-
|
| 326 |
-
|
| 327 |
-
|
| 328 |
-
|
| 329 |
-
|
| 330 |
-
|
| 331 |
-
|
| 332 |
-
|
| 333 |
-
|
| 334 |
-
|
| 335 |
-
|
| 336 |
-
|
| 337 |
-
|
| 338 |
-
|
| 339 |
-
|
| 340 |
-
|
| 341 |
-
|
| 342 |
-
|
| 343 |
-
|
| 344 |
-
|
| 345 |
-
|
| 346 |
-
|
| 347 |
-
|
| 348 |
-
|
| 349 |
-
|
| 350 |
-
|
| 351 |
-
|
| 352 |
-
|
| 353 |
-
|
| 354 |
-
|
| 355 |
-
|
| 356 |
-
|
| 357 |
-
|
| 358 |
-
|
| 359 |
-
|
| 360 |
-
|
| 361 |
-
|
| 362 |
-
|
| 363 |
-
|
| 364 |
-
|
| 365 |
-
|
| 366 |
-
|
| 367 |
-
|
| 368 |
-
ws
|
| 369 |
-
ws
|
| 370 |
-
|
| 371 |
-
|
| 372 |
-
|
| 373 |
-
|
| 374 |
-
|
| 375 |
-
|
| 376 |
-
|
| 377 |
-
|
| 378 |
-
|
| 379 |
-
|
| 380 |
-
|
| 381 |
-
|
| 382 |
-
|
| 383 |
-
|
| 384 |
-
|
| 385 |
-
|
| 386 |
-
|
| 387 |
-
|
| 388 |
-
|
| 389 |
-
|
| 390 |
-
|
| 391 |
-
|
| 392 |
-
|
| 393 |
-
|
| 394 |
-
|
| 395 |
-
|
| 396 |
-
|
| 397 |
-
|
| 398 |
-
|
| 399 |
-
|
| 400 |
-
|
| 401 |
-
|
| 402 |
-
|
| 403 |
-
|
| 404 |
-
|
| 405 |
-
|
| 406 |
-
|
| 407 |
-
|
| 408 |
-
|
| 409 |
-
|
| 410 |
-
|
| 411 |
-
|
| 412 |
-
|
| 413 |
-
|
| 414 |
-
|
| 415 |
-
|
| 416 |
-
|
| 417 |
-
|
| 418 |
-
|
| 419 |
-
|
| 420 |
-
|
| 421 |
-
|
| 422 |
-
|
| 423 |
-
|
| 424 |
-
|
| 425 |
-
|
| 426 |
-
|
| 427 |
-
|
| 428 |
-
|
| 429 |
-
|
| 430 |
-
|
| 431 |
-
|
| 432 |
-
|
| 433 |
-
|
| 434 |
-
|
| 435 |
-
|
| 436 |
-
|
| 437 |
-
|
| 438 |
-
|
| 439 |
-
|
| 440 |
-
|
| 441 |
-
|
| 442 |
-
|
| 443 |
-
|
| 444 |
-
|
| 445 |
-
|
| 446 |
-
def
|
| 447 |
-
|
| 448 |
-
|
| 449 |
-
|
| 450 |
-
|
| 451 |
-
|
| 452 |
-
|
| 453 |
-
|
| 454 |
-
|
| 455 |
-
|
| 456 |
-
|
| 457 |
-
|
| 458 |
-
|
| 459 |
-
|
| 460 |
-
|
| 461 |
-
|
| 462 |
-
|
| 463 |
-
|
| 464 |
-
|
| 465 |
-
|
| 466 |
-
|
| 467 |
-
|
| 468 |
-
|
| 469 |
-
|
| 470 |
-
|
| 471 |
-
|
| 472 |
-
|
| 473 |
-
|
| 474 |
-
|
| 475 |
-
|
| 476 |
-
|
| 477 |
-
|
| 478 |
-
|
| 479 |
-
|
| 480 |
-
|
| 481 |
-
|
| 482 |
-
|
| 483 |
-
|
| 484 |
-
|
| 485 |
-
|
| 486 |
-
|
| 487 |
-
|
| 488 |
-
|
| 489 |
-
|
| 490 |
-
|
| 491 |
-
|
| 492 |
-
|
| 493 |
-
|
| 494 |
-
|
| 495 |
-
|
| 496 |
-
|
| 497 |
-
|
| 498 |
-
|
| 499 |
-
|
| 500 |
-
|
| 501 |
-
|
| 502 |
-
|
| 503 |
-
|
| 504 |
-
|
| 505 |
-
|
| 506 |
-
|
| 507 |
-
|
| 508 |
-
|
| 509 |
-
|
| 510 |
-
|
| 511 |
-
|
| 512 |
-
|
| 513 |
-
|
| 514 |
-
|
| 515 |
-
|
| 516 |
-
|
| 517 |
-
|
| 518 |
-
|
| 519 |
-
|
| 520 |
-
|
| 521 |
-
|
| 522 |
-
|
| 523 |
-
|
| 524 |
-
|
| 525 |
-
|
| 526 |
-
|
| 527 |
-
|
| 528 |
-
|
| 529 |
-
|
| 530 |
-
|
| 531 |
-
|
| 532 |
-
|
| 533 |
-
|
| 534 |
-
|
| 535 |
-
|
| 536 |
-
|
| 537 |
-
|
| 538 |
-
|
| 539 |
-
|
| 540 |
-
|
| 541 |
-
|
| 542 |
-
|
| 543 |
-
|
| 544 |
-
|
| 545 |
-
|
| 546 |
-
|
| 547 |
-
|
| 548 |
-
|
| 549 |
-
|
| 550 |
-
|
| 551 |
-
|
| 552 |
-
|
| 553 |
-
|
| 554 |
-
|
| 555 |
-
|
| 556 |
-
|
| 557 |
-
|
| 558 |
-
|
| 559 |
-
|
| 560 |
-
|
| 561 |
-
|
| 562 |
-
|
| 563 |
-
|
| 564 |
-
|
| 565 |
-
|
| 566 |
-
|
| 567 |
-
|
| 568 |
-
|
| 569 |
-
|
| 570 |
-
|
| 571 |
-
|
| 572 |
-
|
| 573 |
-
|
| 574 |
-
|
| 575 |
-
|
| 576 |
-
|
| 577 |
-
|
| 578 |
-
|
| 579 |
-
|
| 580 |
-
|
| 581 |
-
|
| 582 |
-
|
| 583 |
-
|
| 584 |
-
|
| 585 |
-
|
| 586 |
-
|
| 587 |
-
|
| 588 |
-
|
| 589 |
-
|
| 590 |
-
|
| 591 |
-
|
| 592 |
-
|
| 593 |
-
|
| 594 |
-
|
| 595 |
-
|
| 596 |
-
|
| 597 |
-
|
| 598 |
-
|
| 599 |
-
|
| 600 |
-
|
| 601 |
-
|
| 602 |
-
|
| 603 |
-
|
| 604 |
-
|
| 605 |
-
|
| 606 |
-
|
| 607 |
-
|
| 608 |
-
|
| 609 |
-
|
| 610 |
-
|
| 611 |
-
|
| 612 |
-
|
| 613 |
-
|
| 614 |
-
|
| 615 |
-
|
| 616 |
-
|
| 617 |
-
|
| 618 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""The generic per-user TABLE WORKSPACE store β the persistence half of the table-page factory.
|
| 2 |
+
|
| 3 |
+
One durable store key holds one table OBJECT's per-user Airtable-style state:
|
| 4 |
+
|
| 5 |
+
{username: {'views': {view_id: SavedView},
|
| 6 |
+
'fields': {field_key: Field}, # notes + custom_ + measure_ strata
|
| 7 |
+
'overlays': {str(pid): {field_key: value}}}}
|
| 8 |
+
|
| 9 |
+
`make(table_key)` returns the six operations a table page's host loop needs, closed over that
|
| 10 |
+
key. The Customer table's ops (`modules/customer_data.py`, key 'customer_table_workspace') are
|
| 11 |
+
these exact functions β the logic MOVED here 2026-07-27 so that duplicating the Customer table
|
| 12 |
+
pattern to a new object is a registry row + a config, not a copy of the store plumbing
|
| 13 |
+
(owner directive: the table-page factory).
|
| 14 |
+
|
| 15 |
+
A LIST (membership/formula semantics) is deliberately a different store from a VIEW
|
| 16 |
+
(presentation/query state) β see modules/customer_data.py's customer_lists key.
|
| 17 |
+
"""
|
| 18 |
+
import core.store as store
|
| 19 |
+
|
| 20 |
+
#: Wave-9 I17 β the SHARED bucket. Views whose permissions make them visible to anyone but
|
| 21 |
+
#: their creator live here instead of in a personal workspace, under a key that cannot collide
|
| 22 |
+
#: with a username (usernames come from core/users.py and are never dunder-wrapped; `is_shared`
|
| 23 |
+
#: guards it anyway). ONE HOME PER VIEW, never both: a view moved to personal is REMOVED from
|
| 24 |
+
#: here, and a view shared is removed from its creator's workspace. Two homes would mean two
|
| 25 |
+
#: divergent copies the moment either was edited.
|
| 26 |
+
SHARED_KEY = '__shared__'
|
| 27 |
+
|
| 28 |
+
#: ββ W36-T25 β THE COLUMN SUMMARY IS THE DATABASE'S, NOT ONE ACCOUNT'S.
|
| 29 |
+
#:
|
| 30 |
+
#: Owner item 1, second half, verbatim: *"When i filter Avi on my computer sum works for the Sales
|
| 31 |
+
#: last 365 days column but not when my colleague filter it β The sum of the field is not showing
|
| 32 |
+
#: at the bottom even when i zoom out. It only works on my screen??"* Measured while scouting it:
|
| 33 |
+
#: `save_field` writes the WHOLE field payload into `data[username]`, so a column summary β which
|
| 34 |
+
#: is a statement about the COLUMN, identical for every reader by construction β was stored once
|
| 35 |
+
#: per account. Set it, and the totals row exists for you and for nobody else. `computeAggs` then
|
| 36 |
+
#: paints no totals row at all for the colleague (`showTotals` is false when no field carries an
|
| 37 |
+
#: `agg`), which is exactly "not showing at the bottom".
|
| 38 |
+
#:
|
| 39 |
+
#: β ONLY `agg` MOVES, AND THE LINE IS NOT ARBITRARY. Width, column order, the note, the display
|
| 40 |
+
#: format and every `custom_`/`measure_` definition stay per user, because each of those is a
|
| 41 |
+
#: statement about how ONE PERSON reads the column. "Sum this column" is a statement about what
|
| 42 |
+
#: the column MEANS, and two accounts disagreeing about it is the defect, not a preference.
|
| 43 |
+
#:
|
| 44 |
+
#: β IT LIVES IN THE `__shared__` MEMBER OF THIS SAME BUCKET rather than in
|
| 45 |
+
#: `core/shared_overlay.py`'s `<key>__shared` document, and the reason is transactional: a summary
|
| 46 |
+
#: is written by the same `save_field` call that writes the note beside it, and the two must land
|
| 47 |
+
#: or fail together. `__shared__` is already this store's tenant-wide member (shared VIEWS live
|
| 48 |
+
#: there) and is guarded against colliding with a username by `is_shared` and by `core/users.py`'s
|
| 49 |
+
#: never-dunder rule, so there is no second bucket, no second flush and no second failure mode.
|
| 50 |
+
SHARED_FIELD_KEYS = ('agg',)
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
def _may_see(view, viewer, is_admin=False):
|
| 54 |
+
"""Visibility for ONE shared view, fail-closed.
|
| 55 |
+
|
| 56 |
+
'collaborative' = everyone who can already open the module (the caller has gated that).
|
| 57 |
+
'users' = the named users, plus the creator, plus admins β an admin who could not
|
| 58 |
+
see a view could not administer it either.
|
| 59 |
+
Anything unrecognised returns False rather than defaulting open: an unreadable permission
|
| 60 |
+
must never widen access ([[aios-permissioning]] β no fail-open defaults).
|
| 61 |
+
"""
|
| 62 |
+
if not isinstance(view, dict):
|
| 63 |
+
return False
|
| 64 |
+
if view.get('createdBy') == viewer or is_admin:
|
| 65 |
+
return True
|
| 66 |
+
perms = view.get('permissions') or {}
|
| 67 |
+
edit = perms.get('edit')
|
| 68 |
+
if edit == 'collaborative':
|
| 69 |
+
return True
|
| 70 |
+
if edit == 'users':
|
| 71 |
+
return viewer in set(perms.get('users') or ())
|
| 72 |
+
return False # 'personal', absent, or junk
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
def _may_edit(view, viewer, is_admin=False):
|
| 76 |
+
"""Who may WRITE a shared view. Same set as visibility today β the owner's item asks 'who
|
| 77 |
+
can edit' and lists who 'can have access', i.e. seeing and editing are one grant. Kept as a
|
| 78 |
+
separate function so they can diverge (a future read-only share) without hunting callers."""
|
| 79 |
+
return _may_see(view, viewer, is_admin)
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
def _may_administer(view, viewer, is_admin=False):
|
| 83 |
+
"""Who may change a view's PERMISSIONS, or delete it: the creator or an admin ONLY.
|
| 84 |
+
|
| 85 |
+
Deliberately narrower than _may_edit. If a collaborator could rewrite `permissions` they
|
| 86 |
+
could grant themselves sole ownership of somebody else's view, or quietly widen a
|
| 87 |
+
users-scoped view to everyone β the classic privilege-escalation-by-edit hole.
|
| 88 |
+
"""
|
| 89 |
+
if not isinstance(view, dict):
|
| 90 |
+
return False
|
| 91 |
+
return bool(is_admin) or view.get('createdBy') == viewer
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
def is_shared(view):
|
| 95 |
+
"""A view belongs in the shared bucket when its permissions reach beyond its creator."""
|
| 96 |
+
return ((view or {}).get('permissions') or {}).get('edit') in ('collaborative', 'users')
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
def _shared_fields(data):
|
| 100 |
+
"""The tenant-wide field stratum of one workspace document β `{field_key: {'agg': β¦}}`.
|
| 101 |
+
|
| 102 |
+
β TOTAL AND FAIL-SOFT: a document written before W36-T25 has no `__shared__` member at all,
|
| 103 |
+
and one written by a future version may have something else in it. Either way this answers an
|
| 104 |
+
empty mapping rather than raising, because the caller is a READ that must still serve the
|
| 105 |
+
workspace β a column with no summary is the state every column was in yesterday.
|
| 106 |
+
"""
|
| 107 |
+
shared = (data or {}).get(SHARED_KEY)
|
| 108 |
+
fields = (shared or {}).get('fields') if isinstance(shared, dict) else None
|
| 109 |
+
return fields if isinstance(fields, dict) else {}
|
| 110 |
+
|
| 111 |
+
|
| 112 |
+
def split_shared(payload):
|
| 113 |
+
"""`(per_user, shared)` β one field payload divided at the strata boundary.
|
| 114 |
+
|
| 115 |
+
β W36-T25. `shared` carries only `SHARED_FIELD_KEYS` that are actually SET; `per_user` is the
|
| 116 |
+
payload without them. Split HERE rather than at each write door so the three doors that store
|
| 117 |
+
a field (`save_field`, `duplicate_field`, and the delete that must clear it) cannot come apart
|
| 118 |
+
about where a summary lives β which is the whole reason this module exists rather than being
|
| 119 |
+
copied per topic.
|
| 120 |
+
"""
|
| 121 |
+
payload = dict(payload or {})
|
| 122 |
+
shared = {}
|
| 123 |
+
for key in SHARED_FIELD_KEYS:
|
| 124 |
+
value = str(payload.pop(key, '') or '').strip()
|
| 125 |
+
if value:
|
| 126 |
+
shared[key] = value
|
| 127 |
+
return payload, shared
|
| 128 |
+
|
| 129 |
+
|
| 130 |
+
def source_override_is_empty(payload):
|
| 131 |
+
"""Does this stored definition of a SOURCE (non-custom) column carry any user state?
|
| 132 |
+
|
| 133 |
+
A cleared note on an immutable source field returns to the canonical schema instead of
|
| 134 |
+
leaving a meaningless override row. Custom fields remain even with an empty note β and so
|
| 135 |
+
does a PRESET field carrying a measure-window override (wave-2 item 8), a DISPLAY-format
|
| 136 |
+
override (wave-5 item 10), and, since W29-T83, a COLUMN SUMMARY.
|
| 137 |
+
|
| 138 |
+
β EVERY CLAUSE IS A SETTING A USER MADE, and each one omitted is a setting that silently
|
| 139 |
+
stops surviving a session. `agg` was missing: choosing Average on Customer's `Overdue days`
|
| 140 |
+
built an override whose only content was that summary, so this rule threw the whole row away
|
| 141 |
+
on write while the menu went on reading "Summary: Average" from the client's own optimistic
|
| 142 |
+
copy until the next login β a discarded WRITE wearing the face of a failed read
|
| 143 |
+
([[lost-write-looks-like-failed-read]]). Measured on `bac40c2`; a `ut_*` table, which stores
|
| 144 |
+
its definitions through another door entirely, kept it.
|
| 145 |
+
|
| 146 |
+
β ONE RULE, TWO CALLERS β here and `grid_events`' store-less fallback. Two copies of a
|
| 147 |
+
discard rule is how one of them keeps a write the other bins ([[one-evaluator-per-question]]).
|
| 148 |
+
β A CLEARED summary still drops the row, which is the intent: with nothing else set, the
|
| 149 |
+
column goes back to whatever the contract declares for it.
|
| 150 |
+
"""
|
| 151 |
+
payload = payload or {}
|
| 152 |
+
if payload.get('custom'):
|
| 153 |
+
return False
|
| 154 |
+
return (not str(payload.get('note') or '').strip()
|
| 155 |
+
and not isinstance(payload.get('measure'), dict)
|
| 156 |
+
and not isinstance(payload.get('format'), dict)
|
| 157 |
+
and not str(payload.get('agg') or '').strip())
|
| 158 |
+
|
| 159 |
+
|
| 160 |
+
def _unique_name(wanted, existing, *, fallback='Untitled', max_len=120):
|
| 161 |
+
"""Allocate one human-facing name inside a store.update transaction.
|
| 162 |
+
|
| 163 |
+
Keys/ids remain structural identity. Names compare case-insensitively after collapsing
|
| 164 |
+
whitespace, because those variants are indistinguishable in the UI. This helper belongs
|
| 165 |
+
in the store layer: allocating from a pre-write snapshot lets two concurrent requests both
|
| 166 |
+
choose the same free name before either write lands.
|
| 167 |
+
"""
|
| 168 |
+
limit = max(1, int(max_len))
|
| 169 |
+
|
| 170 |
+
def _clean(value):
|
| 171 |
+
return ' '.join(str(value or '').split())
|
| 172 |
+
|
| 173 |
+
base = (_clean(wanted) or _clean(fallback) or 'Untitled')[:limit].rstrip()
|
| 174 |
+
taken = {_clean(value).casefold() for value in existing if _clean(value)}
|
| 175 |
+
if base.casefold() not in taken:
|
| 176 |
+
return base
|
| 177 |
+
index = 2
|
| 178 |
+
while True:
|
| 179 |
+
suffix = f' {index}'
|
| 180 |
+
stem = base[:max(0, limit - len(suffix))].rstrip()
|
| 181 |
+
candidate = f'{stem}{suffix}' if stem else str(index)[-limit:]
|
| 182 |
+
if candidate.casefold() not in taken:
|
| 183 |
+
return candidate
|
| 184 |
+
index += 1
|
| 185 |
+
|
| 186 |
+
|
| 187 |
+
class TableStore:
|
| 188 |
+
"""The six store operations for one table object's workspace, closed over its store key.
|
| 189 |
+
|
| 190 |
+
`st` (wave 18, C3-UT) is the STORE HANDLE β anything exposing `get(name)` /
|
| 191 |
+
`update(name, fn, flush=)`. Default = `core.store` (tenant #0, every existing caller,
|
| 192 |
+
zero behaviour change). The API passes the session's `TenantRuntime`, whose accessors
|
| 193 |
+
apply the tenant prefix / repo binding β which is what makes a user table created by a
|
| 194 |
+
Nurilab admin land in Nurilab's store instead of Royal's.
|
| 195 |
+
"""
|
| 196 |
+
|
| 197 |
+
def __init__(self, table_key, st=None):
|
| 198 |
+
self.table_key = table_key
|
| 199 |
+
self._st = st if st is not None else store
|
| 200 |
+
|
| 201 |
+
@property
|
| 202 |
+
def st(self):
|
| 203 |
+
"""The bound store handle β for SIBLING registries (core/shares) that must read the
|
| 204 |
+
same tenant's buckets this workspace lives in (wave 21, C1)."""
|
| 205 |
+
return self._st
|
| 206 |
+
|
| 207 |
+
def find_view(self, view_id):
|
| 208 |
+
"""`(owner_username, view)` for a view living in ANY personal stratum, else None.
|
| 209 |
+
|
| 210 |
+
β Wave 21 (item 9, C1): the R10 grant registry names bare ids, so projecting a granted
|
| 211 |
+
view means locating the OWNER's record inside this topic's bucket. Personal strata
|
| 212 |
+
only β the `__shared__` bucket has its own read path (`shared_views`), and serving one
|
| 213 |
+
view from two finders is how two copies drift."""
|
| 214 |
+
vid = str(view_id or '').strip()
|
| 215 |
+
if not vid:
|
| 216 |
+
return None
|
| 217 |
+
try:
|
| 218 |
+
data = self._st.get(self.table_key) or {}
|
| 219 |
+
except Exception:
|
| 220 |
+
return None
|
| 221 |
+
for username, ws in data.items():
|
| 222 |
+
if username == SHARED_KEY or not isinstance(ws, dict):
|
| 223 |
+
continue
|
| 224 |
+
v = (ws.get('views') or {}).get(vid)
|
| 225 |
+
if isinstance(v, dict):
|
| 226 |
+
return str(username), dict(v)
|
| 227 |
+
return None
|
| 228 |
+
|
| 229 |
+
def find_folder(self, folder_id):
|
| 230 |
+
"""`(owner_username, folder_row, {view_id: view})` for a VIEWS folder living in any
|
| 231 |
+
personal stratum, else None. `find_view`'s sibling, and here for the same reason.
|
| 232 |
+
|
| 233 |
+
β D-37 (wave 20's R10 remainder, closed 2026-08-05): the grant registry accepts kind
|
| 234 |
+
`folder` and has since wave 20, but only the VIEW kind was ever projected β so "share
|
| 235 |
+
this folder with Karen" recorded a row, listed under Shared with me, and put nothing on
|
| 236 |
+
Karen's screen. Projecting a folder means two lookups the view path does not need: WHO
|
| 237 |
+
owns it, and WHICH views are filed in it. Folder membership lives in the owner's
|
| 238 |
+
`itemFolders` map (item id -> folder id), never on the view record, so the views are
|
| 239 |
+
found by asking that map rather than by reading a list off the folder.
|
| 240 |
+
|
| 241 |
+
Views only (`folders['views']`): the cohort surface has its own store and its own
|
| 242 |
+
sharing question, and answering both here would make one function mean two things.
|
| 243 |
+
"""
|
| 244 |
+
fid = str(folder_id or '').strip()
|
| 245 |
+
if not fid:
|
| 246 |
+
return None
|
| 247 |
+
try:
|
| 248 |
+
data = self._st.get(self.table_key) or {}
|
| 249 |
+
except Exception:
|
| 250 |
+
return None
|
| 251 |
+
for username, ws in data.items():
|
| 252 |
+
if username == SHARED_KEY or not isinstance(ws, dict):
|
| 253 |
+
continue
|
| 254 |
+
rows = (ws.get('folders') or {}).get('views') or []
|
| 255 |
+
hit = next((f for f in rows
|
| 256 |
+
if isinstance(f, dict) and str(f.get('id') or '') == fid), None)
|
| 257 |
+
if not hit:
|
| 258 |
+
continue
|
| 259 |
+
# β `itemFolders` IS KEYED BY SURFACE FIRST (`{'views': {itemId: folderId}, β¦}`) β
|
| 260 |
+
# reading item ids off the top level finds the surface names instead and matches
|
| 261 |
+
# nothing, so the projection silently returns an EMPTY folder and the feature looks
|
| 262 |
+
# exactly as broken as it was before the fix. Caught by this change's own gate,
|
| 263 |
+
# which is the entire argument for writing one.
|
| 264 |
+
placed = (ws.get('itemFolders') or {}).get('views') or {}
|
| 265 |
+
views = ws.get('views') or {}
|
| 266 |
+
inside = {str(vid): dict(v) for vid, v in views.items()
|
| 267 |
+
if isinstance(v, dict) and str(placed.get(str(vid)) or '') == fid}
|
| 268 |
+
return str(username), dict(hit), inside
|
| 269 |
+
return None
|
| 270 |
+
|
| 271 |
+
# ---------------------------------------------------------------- read
|
| 272 |
+
def workspace(self, username, consume_corrections=True):
|
| 273 |
+
"""One user's durable workspace: always the full three-strata shape."""
|
| 274 |
+
try:
|
| 275 |
+
data = self._st.get(self.table_key) or {}
|
| 276 |
+
ws = data.get(username, {}) or {}
|
| 277 |
+
# A collision acknowledgement is protocol state, not part of a field definition.
|
| 278 |
+
# Consume it with the first fresh workspace payload after the correcting write,
|
| 279 |
+
# then splice a bounded copy into that payload only. Keeping it out of `fields`
|
| 280 |
+
# prevents an old request id surviving forever and overriding a later rename.
|
| 281 |
+
corrections = {}
|
| 282 |
+
if consume_corrections and ws.get('fieldCorrections'):
|
| 283 |
+
def _take(current):
|
| 284 |
+
current_ws = current.get(username) or {}
|
| 285 |
+
pending = current_ws.get('fieldCorrections') or {}
|
| 286 |
+
corrections.update({
|
| 287 |
+
str(key)[:80]: dict(value)
|
| 288 |
+
for key, value in pending.items()
|
| 289 |
+
if isinstance(value, dict)
|
| 290 |
+
})
|
| 291 |
+
current_ws.pop('fieldCorrections', None)
|
| 292 |
+
return current
|
| 293 |
+
|
| 294 |
+
data = self._st.update(self.table_key, _take, flush='async')
|
| 295 |
+
ws = (data or {}).get(username, {}) or {}
|
| 296 |
+
except Exception:
|
| 297 |
+
data = {}
|
| 298 |
+
ws = {}
|
| 299 |
+
corrections = {}
|
| 300 |
+
fields = {
|
| 301 |
+
key: dict(value) if isinstance(value, dict) else value
|
| 302 |
+
for key, value in (ws.get('fields') or {}).items()
|
| 303 |
+
}
|
| 304 |
+
# ββ W36-T25 β THE TENANT-WIDE COLUMN SUMMARY, MERGED OVER THIS USER'S STRATUM.
|
| 305 |
+
# β IT MUST BE ABLE TO CREATE AN ENTRY, not only decorate one, and that is the whole
|
| 306 |
+
# reason this is a merge rather than a lookup: the colleague who never touched the column
|
| 307 |
+
# has NO per-user record for it, so a decorate-only pass would have left them with exactly
|
| 308 |
+
# the blank totals row the owner reported. `aios_grid.workspace_wire` reads `meta['agg']`
|
| 309 |
+
# off whatever is here, base column or custom one alike.
|
| 310 |
+
for key, shared in _shared_fields(data).items():
|
| 311 |
+
agg = str((shared or {}).get('agg') or '').strip()
|
| 312 |
+
if not agg:
|
| 313 |
+
continue
|
| 314 |
+
entry = fields.get(key)
|
| 315 |
+
fields[key] = {**entry, 'agg': agg} if isinstance(entry, dict) else {'agg': agg}
|
| 316 |
+
for key, ack in corrections.items():
|
| 317 |
+
field = fields.get(key)
|
| 318 |
+
accepted_label = str(ack.get('label') or '')[:120]
|
| 319 |
+
requested_label = str(ack.get('labelCorrectedFrom') or '')[:120]
|
| 320 |
+
correction_id = str(ack.get('labelCorrectionId') or '')[:180]
|
| 321 |
+
# A newer field write clears/replaces the pending ack in the SAME transaction.
|
| 322 |
+
# The label check is an extra belt against ever attaching a stale ack to a newer
|
| 323 |
+
# definition if a future store implementation weakens that ordering.
|
| 324 |
+
if (isinstance(field, dict) and accepted_label
|
| 325 |
+
and str(field.get('label') or '') == accepted_label
|
| 326 |
+
and requested_label and correction_id):
|
| 327 |
+
field['labelCorrectedFrom'] = requested_label
|
| 328 |
+
field['labelCorrectionId'] = correction_id
|
| 329 |
+
out = {
|
| 330 |
+
'views': dict(ws.get('views') or {}),
|
| 331 |
+
'fields': fields,
|
| 332 |
+
'overlays': dict(ws.get('overlays') or {}),
|
| 333 |
+
# wave-8 I11 (C4): folders over the saved views / cohorts sidebars. A FOURTH
|
| 334 |
+
# stratum rather than a key on each item β see aios_grid.clean_folders for why
|
| 335 |
+
# (a cohort lives in another store, and filing is an organising act, not part of
|
| 336 |
+
# what a view is). Absent for every workspace saved before this wave, which is
|
| 337 |
+
# exactly "no folders yet".
|
| 338 |
+
'folders': dict(ws.get('folders') or {}),
|
| 339 |
+
'itemFolders': dict(ws.get('itemFolders') or {}),
|
| 340 |
+
}
|
| 341 |
+
# 2026-07-31 (owner item 3): WHERE THE USER LEFT OFF survives a new browser. The
|
| 342 |
+
# client's localStorage copy wins when present; this is the server's answer for a
|
| 343 |
+
# fresh profile, which used to fall all the way to the system default view.
|
| 344 |
+
if ws.get('activeViewId'):
|
| 345 |
+
out['activeViewId'] = str(ws['activeViewId'])
|
| 346 |
+
# Wave 2026-08-02 (C-LAYOUT): the per-user record-detail field order. A fifth
|
| 347 |
+
# stratum, absent until the user first reorders β exactly "default order".
|
| 348 |
+
if isinstance(ws.get('recordLayout'), dict):
|
| 349 |
+
out['recordLayout'] = dict(ws['recordLayout'])
|
| 350 |
+
return out
|
| 351 |
+
|
| 352 |
+
# ---------------------------------------------------------------- write
|
| 353 |
+
def _update(self, username, change, shared=None):
|
| 354 |
+
"""Apply `change(ws)` to this user's stratum, and `shared(shared_fields)` to the
|
| 355 |
+
tenant-wide one, in ONE transaction.
|
| 356 |
+
|
| 357 |
+
β W36-T25 β `shared` IS A SECOND CALLBACK RATHER THAN A SECOND `update`, and that is the
|
| 358 |
+
whole reason it exists here instead of at the caller. `save_field` writes a note (per
|
| 359 |
+
user) and a column summary (tenant-wide) from ONE payload; two transactions would let the
|
| 360 |
+
summary land while the note did not, and the store's own commit is asynchronous, so the
|
| 361 |
+
window is real rather than theoretical. One `update`, one flush, one failure mode.
|
| 362 |
+
"""
|
| 363 |
+
def _up(data):
|
| 364 |
+
ws = data.setdefault(username, {})
|
| 365 |
+
ws.setdefault('views', {})
|
| 366 |
+
ws.setdefault('fields', {})
|
| 367 |
+
ws.setdefault('overlays', {})
|
| 368 |
+
ws.setdefault('folders', {})
|
| 369 |
+
ws.setdefault('itemFolders', {})
|
| 370 |
+
change(ws)
|
| 371 |
+
if shared is not None:
|
| 372 |
+
shared(data.setdefault(SHARED_KEY, {}).setdefault('fields', {}))
|
| 373 |
+
return data
|
| 374 |
+
# flush='async' (wave-7 W3): this is THE hot path β every autosaved filter tweak,
|
| 375 |
+
# column note and typed overlay cell lands here inside the component round-trip, and
|
| 376 |
+
# the historical synchronous hub commit cost seconds per edit. The mutation applies to
|
| 377 |
+
# the in-process cache (read-your-writes for every subsequent render); the hub write
|
| 378 |
+
# coalesces in the background. Registry/auth writes elsewhere stay flush='sync'.
|
| 379 |
+
return self._st.update(self.table_key, _up, flush='async')
|
| 380 |
+
|
| 381 |
+
def rename_choice_values(self, username, change):
|
| 382 |
+
"""Apply an arbitrary workspace rewrite (wave 20, item 15 / C-RENAME).
|
| 383 |
+
|
| 384 |
+
β NAMED FOR ITS ONE CALLER RATHER THAN EXPOSED AS A GENERIC `mutate`, deliberately. A
|
| 385 |
+
public "do anything to the workspace" method is an invitation to put write logic in
|
| 386 |
+
callers instead of here, and every OTHER method on this class exists precisely because
|
| 387 |
+
that logic belongs in one place. Renaming a choice is the one operation that must touch
|
| 388 |
+
three strata AT ONCE β the field's `choices`, the cells in `overlays`, and the views that
|
| 389 |
+
filter or colour by the old value β inside a SINGLE transaction, because a rename that
|
| 390 |
+
updated the cells and not the filters would leave a saved view matching nothing.
|
| 391 |
+
|
| 392 |
+
`change(ws)` receives the whole workspace with every stratum pre-created (see `_update`).
|
| 393 |
+
"""
|
| 394 |
+
return self._update(username, change)
|
| 395 |
+
|
| 396 |
+
def save_active_view(self, username, view_id):
|
| 397 |
+
"""Remember which view this user last opened (owner item 3, 2026-07-31).
|
| 398 |
+
|
| 399 |
+
Presentation state, not authorisation: the READ side re-validates the id against what
|
| 400 |
+
the caller may actually see, so a stale or foreign id degrades to the default view
|
| 401 |
+
rather than granting anything. Stored per user like every other stratum.
|
| 402 |
+
"""
|
| 403 |
+
vid = str(view_id or '').strip()[:120]
|
| 404 |
+
if not vid or username == SHARED_KEY:
|
| 405 |
+
return
|
| 406 |
+
|
| 407 |
+
def _set(ws):
|
| 408 |
+
ws['activeViewId'] = vid
|
| 409 |
+
self._update(username, _set)
|
| 410 |
+
|
| 411 |
+
def save_record_layout(self, username, order):
|
| 412 |
+
"""The per-user RECORD-DETAIL field order (wave 2026-08-02, C-LAYOUT).
|
| 413 |
+
|
| 414 |
+
Presentation state for ONE surface β the record modal. Deliberately not view config:
|
| 415 |
+
the owner's ask is per-user, not per-view, and it must never reorder grid columns.
|
| 416 |
+
The event handler validated keys against the live field set; the wire re-validates at
|
| 417 |
+
serve time (aios_grid.workspace_wire), so a deleted field cannot outlive itself here.
|
| 418 |
+
An empty order clears the stratum back to "default order".
|
| 419 |
+
"""
|
| 420 |
+
if username == SHARED_KEY:
|
| 421 |
+
return
|
| 422 |
+
clean, seen = [], set()
|
| 423 |
+
for key in (order or [])[:200]:
|
| 424 |
+
key = str(key or '').strip()[:80]
|
| 425 |
+
if key and key not in seen:
|
| 426 |
+
seen.add(key)
|
| 427 |
+
clean.append(key)
|
| 428 |
+
|
| 429 |
+
def _set(ws):
|
| 430 |
+
if clean:
|
| 431 |
+
ws['recordLayout'] = {'order': clean}
|
| 432 |
+
else:
|
| 433 |
+
ws.pop('recordLayout', None)
|
| 434 |
+
|
| 435 |
+
self._update(username, _set)
|
| 436 |
+
|
| 437 |
+
def save_folders(self, username, folders, item_folders):
|
| 438 |
+
"""Replace the folder stratum wholesale (wave-8 I11).
|
| 439 |
+
|
| 440 |
+
Wholesale rather than per-folder because the caller has ALREADY validated the complete
|
| 441 |
+
picture through aios_grid.clean_folders / clean_item_folders, and those two are
|
| 442 |
+
interdependent: a placement is only legal while its folder exists, so committing them
|
| 443 |
+
separately would leave a window where a reader sees an item filed into a folder that is
|
| 444 |
+
not there yet. One write, one consistent state.
|
| 445 |
+
"""
|
| 446 |
+
def _set(ws):
|
| 447 |
+
ws['folders'] = dict(folders or {})
|
| 448 |
+
ws['itemFolders'] = dict(item_folders or {})
|
| 449 |
+
self._update(username, _set)
|
| 450 |
+
|
| 451 |
+
def save_view_order(self, username, order):
|
| 452 |
+
"""β WAVE-27 item 5 (contract C7) β this user's own ORDER for the views rail.
|
| 453 |
+
|
| 454 |
+
Wholesale, like `save_folders` above and for the same reason: the client sends the full
|
| 455 |
+
list it is looking at, not a delta, because a partial order cannot say where an UNNAMED
|
| 456 |
+
view went.
|
| 457 |
+
|
| 458 |
+
β PER USER, and it belongs in this stratum rather than on the view records themselves.
|
| 459 |
+
`aios_grid`'s own folder note argues it out for placements and every word applies: an
|
| 460 |
+
arrangement is a per-user ORGANISING act, not part of what a view IS β so keeping it out
|
| 461 |
+
of the view config means duplicating, sharing or exporting a view does not drag one
|
| 462 |
+
person's rail position along with it. It also means a SHARED view can sit in a different
|
| 463 |
+
place for each person who can see it, which is the only coherent answer once two people
|
| 464 |
+
share one view.
|
| 465 |
+
|
| 466 |
+
An empty list CLEARS the arrangement (back to server order) rather than storing `[]`.
|
| 467 |
+
"""
|
| 468 |
+
def _set(ws):
|
| 469 |
+
clean = []
|
| 470 |
+
seen = set()
|
| 471 |
+
for vid in (order or []):
|
| 472 |
+
vid = str(vid).strip()[:120]
|
| 473 |
+
if vid and vid not in seen:
|
| 474 |
+
seen.add(vid)
|
| 475 |
+
clean.append(vid)
|
| 476 |
+
if clean:
|
| 477 |
+
ws['viewOrder'] = clean
|
| 478 |
+
else:
|
| 479 |
+
ws.pop('viewOrder', None)
|
| 480 |
+
self._update(username, _set)
|
| 481 |
+
|
| 482 |
+
def shared_views(self, viewer, is_admin=False):
|
| 483 |
+
"""Every SHARED view this viewer may see, by id (wave-9 I17).
|
| 484 |
+
|
| 485 |
+
Read-only and independent of the viewer's own workspace: the caller merges. Returns
|
| 486 |
+
only what `_may_see` allows, so a caller cannot accidentally render somebody else's
|
| 487 |
+
personal view by forgetting to filter.
|
| 488 |
+
"""
|
| 489 |
+
try:
|
| 490 |
+
bucket = ((self._st.get(self.table_key) or {}).get(SHARED_KEY) or {}).get('views') or {}
|
| 491 |
+
except Exception:
|
| 492 |
+
return {}
|
| 493 |
+
return {vid: dict(v) for vid, v in bucket.items()
|
| 494 |
+
if _may_see(v, viewer, is_admin)}
|
| 495 |
+
|
| 496 |
+
def shared_view(self, view_id):
|
| 497 |
+
"""One shared view RAW β no visibility filter. For authorisation decisions only: a
|
| 498 |
+
caller must know a view exists and who owns it before it can decide whether the actor
|
| 499 |
+
may touch it. Never hand the result to a renderer without checking `_may_see`."""
|
| 500 |
+
try:
|
| 501 |
+
return ((self._st.get(self.table_key) or {}).get(SHARED_KEY) or {}
|
| 502 |
+
).get('views', {}).get(str(view_id))
|
| 503 |
+
except Exception:
|
| 504 |
+
return None
|
| 505 |
+
|
| 506 |
+
def save_view(self, username, view, shared=None, reserved_names=(), is_admin=False):
|
| 507 |
+
"""Upsert a SavedView into its ONE home β personal workspace or the shared bucket.
|
| 508 |
+
|
| 509 |
+
`shared` defaults to reading the view's own permissions (`is_shared`). Whichever home
|
| 510 |
+
it lands in, the view is REMOVED from the other, so a view can never exist as two
|
| 511 |
+
copies that diverge on the next edit.
|
| 512 |
+
|
| 513 |
+
β AUTHORISATION IS THE CALLER'S JOB and must happen BEFORE this is called β this layer
|
| 514 |
+
moves data and does not know who is asking. `_cl_handle_one` is the wall.
|
| 515 |
+
"""
|
| 516 |
+
view_id = str((view or {}).get('id') or '').strip()
|
| 517 |
+
if not view_id:
|
| 518 |
+
raise ValueError('view id is required')
|
| 519 |
+
if username == SHARED_KEY:
|
| 520 |
+
raise ValueError('reserved username')
|
| 521 |
+
to_shared = is_shared(view) if shared is None else bool(shared)
|
| 522 |
+
requested = dict(view)
|
| 523 |
+
accepted = {}
|
| 524 |
+
|
| 525 |
+
def _up(data):
|
| 526 |
+
# View names are tenant-global: every personal workspace plus the shared bucket.
|
| 527 |
+
# This deliberately includes views the actor cannot see. The only disclosed fact
|
| 528 |
+
# is that a display name is already taken, while the categorical "no duplicate
|
| 529 |
+
# view names" contract remains true when a personal view is later shared.
|
| 530 |
+
names = list(reserved_names or ())
|
| 531 |
+
for workspace in data.values():
|
| 532 |
+
if not isinstance(workspace, dict):
|
| 533 |
+
continue
|
| 534 |
+
names.extend(
|
| 535 |
+
value.get('name')
|
| 536 |
+
for candidate_id, value in (workspace.get('views') or {}).items()
|
| 537 |
+
if candidate_id != view_id and isinstance(value, dict)
|
| 538 |
+
)
|
| 539 |
+
payload = dict(requested)
|
| 540 |
+
payload['name'] = _unique_name(payload.get('name'), names)
|
| 541 |
+
accepted.clear()
|
| 542 |
+
accepted.update(payload)
|
| 543 |
+
if to_shared:
|
| 544 |
+
bucket = data.setdefault(SHARED_KEY, {})
|
| 545 |
+
bucket.setdefault('views', {})[view_id] = payload
|
| 546 |
+
# it may have lived in the creator's workspace before being shared
|
| 547 |
+
owner = data.get(payload.get('createdBy') or username) or {}
|
| 548 |
+
(owner.get('views') or {}).pop(view_id, None)
|
| 549 |
+
else:
|
| 550 |
+
ws = data.setdefault(username, {})
|
| 551 |
+
ws.setdefault('views', {})[view_id] = payload
|
| 552 |
+
(data.get(SHARED_KEY, {}).get('views') or {}).pop(view_id, None)
|
| 553 |
+
return data
|
| 554 |
+
|
| 555 |
+
self._st.update(self.table_key, _up, flush='async')
|
| 556 |
+
return dict(accepted)
|
| 557 |
+
|
| 558 |
+
def delete_view(self, username, view_id):
|
| 559 |
+
"""Delete a custom/list view override. The system all-rows view is guarded by caller.
|
| 560 |
+
|
| 561 |
+
Removes from BOTH homes: the caller has already authorised the delete, and leaving a
|
| 562 |
+
stale copy in the other bucket would resurrect the view on the next read.
|
| 563 |
+
"""
|
| 564 |
+
vid = str(view_id)
|
| 565 |
+
|
| 566 |
+
def _up(data):
|
| 567 |
+
(data.get(username, {}).get('views') or {}).pop(vid, None)
|
| 568 |
+
(data.get(SHARED_KEY, {}).get('views') or {}).pop(vid, None)
|
| 569 |
+
return data
|
| 570 |
+
|
| 571 |
+
self._st.update(self.table_key, _up, flush='async')
|
| 572 |
+
|
| 573 |
+
def save_field(self, username, field, reserved_names=(), correction_id=None):
|
| 574 |
+
"""Persist a column note or a user-created (custom_/measure_) field definition.
|
| 575 |
+
|
| 576 |
+
ββ W36-T25 β THE COLUMN SUMMARY GOES TO THE TENANT-WIDE STRATUM, EVERYTHING ELSE STAYS
|
| 577 |
+
PER USER, in ONE transaction. See `SHARED_FIELD_KEYS` above for the owner's report and for
|
| 578 |
+
why the line falls where it does. The returned `accepted` still carries the summary: the
|
| 579 |
+
caller is echoing back the field it just stored, and dropping a key from that echo would
|
| 580 |
+
tell the client its write was refused ([[read-path-cannot-witness-write-path]]).
|
| 581 |
+
"""
|
| 582 |
+
key = str((field or {}).get('key') or '').strip()
|
| 583 |
+
if not key:
|
| 584 |
+
raise ValueError('field key is required')
|
| 585 |
+
requested = dict(field)
|
| 586 |
+
accepted = {}
|
| 587 |
+
|
| 588 |
+
def _save(ws):
|
| 589 |
+
names = list(reserved_names or ())
|
| 590 |
+
names.extend(
|
| 591 |
+
value.get('label')
|
| 592 |
+
for candidate_key, value in (ws.get('fields') or {}).items()
|
| 593 |
+
if candidate_key != key and isinstance(value, dict)
|
| 594 |
+
)
|
| 595 |
+
payload = dict(requested)
|
| 596 |
+
payload.pop('labelCorrectedFrom', None)
|
| 597 |
+
payload.pop('labelCorrectionId', None)
|
| 598 |
+
requested_label = ' '.join(
|
| 599 |
+
str(payload.get('label') or 'Untitled').split())[:120].rstrip()
|
| 600 |
+
payload['label'] = _unique_name(requested_label, names)
|
| 601 |
+
corrections = ws.setdefault('fieldCorrections', {})
|
| 602 |
+
corrections.pop(key, None)
|
| 603 |
+
if payload['label'] != requested_label and correction_id:
|
| 604 |
+
corrections[key] = {
|
| 605 |
+
'label': payload['label'],
|
| 606 |
+
'labelCorrectedFrom': requested_label,
|
| 607 |
+
'labelCorrectionId': str(correction_id)[:180],
|
| 608 |
+
}
|
| 609 |
+
if not corrections:
|
| 610 |
+
ws.pop('fieldCorrections', None)
|
| 611 |
+
accepted.clear()
|
| 612 |
+
accepted.update(payload)
|
| 613 |
+
# ββ W36-T25 β SPLIT AFTER the label allocation and the correction bookkeeping, so
|
| 614 |
+
# both still see the whole payload, and BEFORE the per-user write.
|
| 615 |
+
mine, shared_now = split_shared(payload)
|
| 616 |
+
shared_write.clear()
|
| 617 |
+
shared_write.update(shared_now)
|
| 618 |
+
# β EMPTINESS IS JUDGED ON THE PER-USER HALF. A source column whose ONLY state was a
|
| 619 |
+
# summary now has no per-user state at all, and leaving an `{}` override behind is the
|
| 620 |
+
# meaningless row `source_override_is_empty` exists to prevent.
|
| 621 |
+
if source_override_is_empty(mine):
|
| 622 |
+
ws['fields'].pop(key, None)
|
| 623 |
+
else:
|
| 624 |
+
ws['fields'][key] = mine
|
| 625 |
+
|
| 626 |
+
shared_write = {}
|
| 627 |
+
|
| 628 |
+
def _share(shared_fields):
|
| 629 |
+
# β A CLEARED SUMMARY MUST REMOVE THE ROW, not leave an empty one: `workspace` treats
|
| 630 |
+
# any entry it finds as a live tenant-wide summary, so an `{'agg': ''}` husk would be
|
| 631 |
+
# skipped today and become a resurrection hazard the moment the read grows a second
|
| 632 |
+
# shared key. Absence is the only honest spelling of "nobody set one".
|
| 633 |
+
if shared_write:
|
| 634 |
+
shared_fields[key] = {**(shared_fields.get(key) or {}), **shared_write}
|
| 635 |
+
else:
|
| 636 |
+
shared_fields.pop(key, None)
|
| 637 |
+
|
| 638 |
+
self._update(username, _save, shared=_share)
|
| 639 |
+
return dict(accepted)
|
| 640 |
+
|
| 641 |
+
def duplicate_field(self, username, source_key, new_key, field,
|
| 642 |
+
reserved_names=(), correction_id=None):
|
| 643 |
+
"""Clone a user-created field in ONE store transaction (wave-5 item 1): the new
|
| 644 |
+
definition plus β for `custom_` overlay sources only β every stored cell value under
|
| 645 |
+
the source key. One transaction, because a def without its values (or values without a
|
| 646 |
+
def) is exactly the orphan state delete_field exists to prevent, in reverse.
|
| 647 |
+
The caller validated both keys (same created stratum) and stamped the clone's
|
| 648 |
+
createdBy; this layer only moves data."""
|
| 649 |
+
source_key = str(source_key or '').strip()
|
| 650 |
+
new_key = str(new_key or '').strip()
|
| 651 |
+
if not source_key or not new_key or source_key == new_key:
|
| 652 |
+
raise ValueError('duplicate_field needs two distinct keys')
|
| 653 |
+
requested = dict(field)
|
| 654 |
+
accepted = {}
|
| 655 |
+
|
| 656 |
+
def _dup(ws):
|
| 657 |
+
names = list(reserved_names or ())
|
| 658 |
+
names.extend(
|
| 659 |
+
value.get('label')
|
| 660 |
+
for candidate_key, value in (ws.get('fields') or {}).items()
|
| 661 |
+
if candidate_key != new_key and isinstance(value, dict)
|
| 662 |
+
)
|
| 663 |
+
payload = dict(requested)
|
| 664 |
+
payload.pop('labelCorrectedFrom', None)
|
| 665 |
+
payload.pop('labelCorrectionId', None)
|
| 666 |
+
requested_label = ' '.join(
|
| 667 |
+
str(payload.get('label') or 'Untitled').split())[:120].rstrip()
|
| 668 |
+
payload['label'] = _unique_name(requested_label, names)
|
| 669 |
+
corrections = ws.setdefault('fieldCorrections', {})
|
| 670 |
+
corrections.pop(new_key, None)
|
| 671 |
+
if payload['label'] != requested_label and correction_id:
|
| 672 |
+
corrections[new_key] = {
|
| 673 |
+
'label': payload['label'],
|
| 674 |
+
'labelCorrectedFrom': requested_label,
|
| 675 |
+
'labelCorrectionId': str(correction_id)[:180],
|
| 676 |
+
}
|
| 677 |
+
if not corrections:
|
| 678 |
+
ws.pop('fieldCorrections', None)
|
| 679 |
+
accepted.clear()
|
| 680 |
+
accepted.update(payload)
|
| 681 |
+
# β W36-T25: a CLONE carries the original's summary, and a summary is the database's
|
| 682 |
+
# (see `SHARED_FIELD_KEYS`). Storing it per user here would give the clone a different
|
| 683 |
+
# residency from every other column β one door out of three disagreeing about where a
|
| 684 |
+
# thing lives is how `save_field` and this function drift.
|
| 685 |
+
mine, shared_now = split_shared(payload)
|
| 686 |
+
shared_write.clear()
|
| 687 |
+
shared_write.update(shared_now)
|
| 688 |
+
ws['fields'][new_key] = mine
|
| 689 |
+
if source_key.startswith('custom_'):
|
| 690 |
+
for row in ws['overlays'].values():
|
| 691 |
+
if isinstance(row, dict) and source_key in row:
|
| 692 |
+
row[new_key] = row[source_key]
|
| 693 |
+
|
| 694 |
+
shared_write = {}
|
| 695 |
+
|
| 696 |
+
def _share(shared_fields):
|
| 697 |
+
if shared_write:
|
| 698 |
+
shared_fields[new_key] = {**(shared_fields.get(new_key) or {}), **shared_write}
|
| 699 |
+
else:
|
| 700 |
+
shared_fields.pop(new_key, None)
|
| 701 |
+
|
| 702 |
+
self._update(username, _dup, shared=_share)
|
| 703 |
+
return dict(accepted)
|
| 704 |
+
|
| 705 |
+
def delete_field(self, username, key):
|
| 706 |
+
"""Delete a USER-CREATED field definition outright (owner gap closed 2026-07-27).
|
| 707 |
+
|
| 708 |
+
Only the created strata ever reach here (`custom_` overlay fields, `measure_` formula
|
| 709 |
+
columns β the caller enforces the prefix). The stored overlay VALUES for the key are
|
| 710 |
+
scrubbed with it: a deleted column's cells must not linger as orphan data that would
|
| 711 |
+
silently resurface if the key were ever reused. Views referencing the key self-heal on
|
| 712 |
+
their next autosave (an unknown colId is dropped) β the rule every stale key rides.
|
| 713 |
+
|
| 714 |
+
ββ W36-T25 β AND THE TENANT-WIDE SUMMARY GOES WITH IT, for exactly the reason the
|
| 715 |
+
paragraph above gives about cells: an orphan `agg` under a deleted key is state nobody can
|
| 716 |
+
see and nobody can clear, and it would attach itself to the next column that happens to
|
| 717 |
+
take the key back. β This is the ONE stratum a per-user delete may reach across accounts,
|
| 718 |
+
and it is safe because the summary was never this user's to begin with β deleting the
|
| 719 |
+
COLUMN is a tenant-wide act already.
|
| 720 |
+
"""
|
| 721 |
+
key = str(key or '').strip()
|
| 722 |
+
if not key:
|
| 723 |
+
return
|
| 724 |
+
|
| 725 |
+
def _drop(ws):
|
| 726 |
+
ws['fields'].pop(key, None)
|
| 727 |
+
for row in ws['overlays'].values():
|
| 728 |
+
if isinstance(row, dict):
|
| 729 |
+
row.pop(key, None)
|
| 730 |
+
|
| 731 |
+
self._update(username, _drop, shared=lambda shared_fields: shared_fields.pop(key, None))
|
| 732 |
+
|
| 733 |
+
def patch_overlay(self, username, pid, updates):
|
| 734 |
+
"""Patch only the external editable stratum; never writes to the source system."""
|
| 735 |
+
clean = dict(updates or {})
|
| 736 |
+
if not clean:
|
| 737 |
+
return
|
| 738 |
+
|
| 739 |
+
def _patch(ws):
|
| 740 |
+
ws['overlays'].setdefault(str(int(pid)), {}).update(clean)
|
| 741 |
+
|
| 742 |
+
self._update(username, _patch)
|
| 743 |
+
|
| 744 |
+
|
| 745 |
+
def make(table_key, st=None):
|
| 746 |
+
return TableStore(table_key, st=st)
|
platform/core/user_tables.py
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
platform/harness/meta_store.py
CHANGED
|
@@ -1,521 +1,521 @@
|
|
| 1 |
-
"""meta_store.py β the REST loader for a mirror whose only other loader is XML-RPC (W31-T48).
|
| 2 |
-
|
| 3 |
-
python platform/harness/meta_store.py --sync pull into tenant #0's mirror
|
| 4 |
-
python platform/harness/meta_store.py --sync --tenant gtmlab
|
| 5 |
-
python platform/harness/meta_store.py --status what is in the mirror now
|
| 6 |
-
|
| 7 |
-
β WHY THIS IS A SIBLING AND NOT A ROW IN `datastore.ENTITIES`. `datastore.sync_entity` is
|
| 8 |
-
hardwired to XML-RPC β it does `import core.odoo as O` and speaks `search_read` β so no amount of
|
| 9 |
-
spec data makes it fetch over HTTPS. What IS reusable is everything *below* the fetch: the
|
| 10 |
-
per-tenant file (`path_for`), the process-singleton connection (`connect`), and the
|
| 11 |
-
delete-then-insert upsert. This module borrows those and brings its own reader. **The mirror is one
|
| 12 |
-
store with two loaders, not two stores.**
|
| 13 |
-
|
| 14 |
-
β THE SHAPE OF THE WHOLE THING, because it is the owner's actual request: *"we just need to pull
|
| 15 |
-
the data into our template database for Meta"*, working *"just like Odoo"*. Odoo's path is
|
| 16 |
-
`XML-RPC -> DuckDB mirror -> odoo_relational -> ut_odoo_* locked grids`. Meta's is
|
| 17 |
-
`Graph -> DuckDB mirror -> meta_relational -> ut_meta_* locked grids`. Same middle, same end, one
|
| 18 |
-
different first hop.
|
| 19 |
-
|
| 20 |
-
β EVERY COLUMN NAME BELOW WAS MEASURED, NOT READ OFF A DOC (R8). `proto/meta-entity-fields.json`
|
| 21 |
-
carries the run: each level's list is what the API ACCEPTED when asked, on a real ad account.
|
| 22 |
-
Ad Account 45 Β· Campaign 35 Β· Ad Set 54 Β· Ad 36 Β· Creative 58 Β· Insights 57
|
| 23 |
-
β AND `ACCEPTED` IS NOT `RETURNED`. Graph omits null fields from a response, so a 20-field ask
|
| 24 |
-
came back with 14 keys. Building a schema from what came back would drop ~30% of the columns with
|
| 25 |
-
nothing going red β which is exactly the "do not drop any column" instruction, broken silently.
|
| 26 |
-
The lists below are therefore the ASKED-AND-ACCEPTED set, and a column with no value is NULL.
|
| 27 |
-
|
| 28 |
-
β `owner` IS ABSENT FROM THE AD ACCOUNT LIST AND THAT IS A REPORTED LIMIT, NOT AN OMISSION: it
|
| 29 |
-
answers `403 (#200) Requires business_management permission`, which this token does not carry. One
|
| 30 |
-
column of 285. R6's second sentence β a limit that cannot be removed gets stated with its cause.
|
| 31 |
-
|
| 32 |
-
β IDS ARE TEXT, ALWAYS. A Meta object id is a 17-digit decimal string; `120273975028650555` is
|
| 33 |
-
larger than 2^53, so any float or JS-number path silently corrupts it. Same ruling as `ig_id`
|
| 34 |
-
(W26/R3), for the same reason, and it is why every `id` column here is VARCHAR.
|
| 35 |
-
"""
|
| 36 |
-
import argparse
|
| 37 |
-
import json
|
| 38 |
-
import os
|
| 39 |
-
import sys
|
| 40 |
-
import time
|
| 41 |
-
import urllib.error
|
| 42 |
-
import urllib.parse
|
| 43 |
-
import urllib.request
|
| 44 |
-
from pathlib import Path
|
| 45 |
-
|
| 46 |
-
_HERE = Path(__file__).resolve().parent
|
| 47 |
-
if str(_HERE.parent) not in sys.path:
|
| 48 |
-
sys.path.insert(0, str(_HERE.parent))
|
| 49 |
-
|
| 50 |
-
from harness import datastore # noqa: E402
|
| 51 |
-
|
| 52 |
-
GRAPH_VERSION = os.environ.get("META_GRAPH_VERSION") or "v21.0"
|
| 53 |
-
GRAPH = f"https://graph.facebook.com/{GRAPH_VERSION}"
|
| 54 |
-
|
| 55 |
-
#: Numeric columns, by name. Everything else is VARCHAR β including ids (see the header) and
|
| 56 |
-
#: including anything Graph returns as a nested object, which is stored as compact JSON text.
|
| 57 |
-
_INT = {"impressions", "reach", "clicks", "unique_clicks", "inline_link_clicks",
|
| 58 |
-
"inline_post_engagement", "full_view_impressions", "estimated_ad_recallers",
|
| 59 |
-
"account_status", "age", "timezone_id", "timezone_offset_hours_utc", "io_number",
|
| 60 |
-
"min_daily_budget", "min_campaign_group_spend_cap"}
|
| 61 |
-
_DBL = {"spend", "social_spend", "ctr", "unique_ctr", "cpc", "cpm", "cpp", "frequency",
|
| 62 |
-
"inline_link_click_ctr", "outbound_clicks_ctr", "amount_spent", "balance", "spend_cap",
|
| 63 |
-
"daily_budget", "lifetime_budget", "budget_remaining", "bid_amount", "canvas_avg_view_time",
|
| 64 |
-
"daily_min_spend_target", "daily_spend_cap", "lifetime_min_spend_target",
|
| 65 |
-
"lifetime_spend_cap", "lifetime_imps"}
|
| 66 |
-
|
| 67 |
-
#: β THE MEASURED CATALOG. `edge` is the connection on the ad account; `None` means the account
|
| 68 |
-
#: itself. `parent` names the column that ties a row to its parent, used by nothing here and by
|
| 69 |
-
#: the grid links in `meta_relational`.
|
| 70 |
-
ACCOUNT_FIELDS = (
|
| 71 |
-
"account_id account_status age amount_spent balance business_city business_country_code "
|
| 72 |
-
"business_name business_state business_street business_street2 business_zip capabilities "
|
| 73 |
-
"created_time currency disable_reason end_advertiser end_advertiser_name funding_source "
|
| 74 |
-
"has_migrated_permissions id io_number is_attribution_spec_system_default "
|
| 75 |
-
"is_direct_deals_enabled is_notifications_enabled is_personal is_prepay_account "
|
| 76 |
-
"is_tax_id_required line_numbers media_agency min_campaign_group_spend_cap min_daily_budget "
|
| 77 |
-
"name offsite_pixels_tos_accepted partner spend_cap tax_id tax_id_status tax_id_type "
|
| 78 |
-
"timezone_id timezone_name timezone_offset_hours_utc tos_accepted user_tasks user_tos_accepted"
|
| 79 |
-
).split()
|
| 80 |
-
|
| 81 |
-
CAMPAIGN_FIELDS = (
|
| 82 |
-
"account_id bid_strategy boosted_object_id budget_rebalance_flag budget_remaining buying_type "
|
| 83 |
-
"campaign_group_active_time can_create_brand_lift_study can_use_spend_cap configured_status "
|
| 84 |
-
"created_time daily_budget effective_status id is_skadnetwork_attribution issues_info "
|
| 85 |
-
"last_budget_toggling_time lifetime_budget name objective pacing_type primary_attribution "
|
| 86 |
-
"promoted_object smart_promotion_type source_campaign source_campaign_id special_ad_categories "
|
| 87 |
-
"special_ad_category special_ad_category_country spend_cap start_time status stop_time "
|
| 88 |
-
"topline_id updated_time"
|
| 89 |
-
).split()
|
| 90 |
-
|
| 91 |
-
ADSET_FIELDS = (
|
| 92 |
-
"account_id adlabels adset_schedule asset_feed_id attribution_spec bid_adjustments bid_amount "
|
| 93 |
-
"bid_constraints bid_info bid_strategy billing_event budget_remaining campaign "
|
| 94 |
-
"campaign_active_time campaign_attribution campaign_id configured_status created_time "
|
| 95 |
-
"creative_sequence daily_budget daily_min_spend_target daily_spend_cap destination_type "
|
| 96 |
-
"effective_status end_time frequency_control_specs id instagram_actor_id is_dynamic_creative "
|
| 97 |
-
"issues_info learning_stage_info lifetime_budget lifetime_imps lifetime_min_spend_target "
|
| 98 |
-
"lifetime_spend_cap multi_optimization_goal_weight name optimization_goal "
|
| 99 |
-
"optimization_sub_event pacing_type promoted_object recurring_budget_semantics review_feedback "
|
| 100 |
-
"rf_prediction_id source_adset source_adset_id start_time status targeting "
|
| 101 |
-
"targeting_optimization_types time_based_ad_rotation_id_blocks "
|
| 102 |
-
"time_based_ad_rotation_intervals updated_time use_new_app_click"
|
| 103 |
-
).split()
|
| 104 |
-
|
| 105 |
-
AD_FIELDS = (
|
| 106 |
-
"account_id ad_active_time ad_review_feedback ad_schedule_end_time ad_schedule_start_time "
|
| 107 |
-
"adlabels adset adset_id bid_amount bid_info bid_type campaign campaign_id configured_status "
|
| 108 |
-
"conversion_domain created_time creative demolink_hash display_sequence effective_status "
|
| 109 |
-
"engagement_audience failed_delivery_checks id issues_info last_updated_by_app_id name "
|
| 110 |
-
"preview_shareable_link priority recommendations source_ad source_ad_id status targeting "
|
| 111 |
-
"tracking_and_conversion_with_defaults tracking_specs updated_time"
|
| 112 |
-
).split()
|
| 113 |
-
|
| 114 |
-
CREATIVE_FIELDS = (
|
| 115 |
-
"account_id actor_id adlabels applink_treatment asset_feed_spec authorization_category body "
|
| 116 |
-
"branded_content_sponsor_page_id bundle_folder_id call_to_action_type categorization_criteria "
|
| 117 |
-
"category_media_source collaborative_ads_lsb_image_bank_id degrees_of_freedom_spec "
|
| 118 |
-
"destination_set_id dynamic_ad_voice effective_authorization_category "
|
| 119 |
-
"effective_instagram_media_id effective_object_story_id enable_direct_install "
|
| 120 |
-
"enable_launch_instant_app id image_crops image_hash image_url instagram_actor_id "
|
| 121 |
-
"instagram_permalink_url instagram_story_id instagram_user_id interactive_components_spec "
|
| 122 |
-
"link_deep_link_url link_destination_display_url link_og_id link_url "
|
| 123 |
-
"messenger_sponsored_message name object_id object_store_url object_story_id object_story_spec "
|
| 124 |
-
"object_type object_url place_page_set_id platform_customizations playable_asset_id "
|
| 125 |
-
"portrait_customizations product_set_id recommender_settings source_instagram_media_id status "
|
| 126 |
-
"template_url template_url_spec thumbnail_id thumbnail_url title url_tags "
|
| 127 |
-
"use_page_actor_override video_id"
|
| 128 |
-
).split()
|
| 129 |
-
|
| 130 |
-
INSIGHT_FIELDS = (
|
| 131 |
-
"account_currency account_id account_name action_values actions ad_id ad_name adset_id "
|
| 132 |
-
"adset_name attribution_setting buying_type campaign_id campaign_name canvas_avg_view_time "
|
| 133 |
-
"clicks conversion_rate_ranking conversion_values conversions cost_per_action_type "
|
| 134 |
-
"cost_per_inline_link_click cost_per_thruplay cost_per_unique_click cpc cpm cpp ctr date_start "
|
| 135 |
-
"date_stop engagement_rate_ranking estimated_ad_recallers frequency full_view_impressions "
|
| 136 |
-
"impressions inline_link_click_ctr inline_link_clicks inline_post_engagement objective "
|
| 137 |
-
"optimization_goal outbound_clicks outbound_clicks_ctr purchase_roas quality_ranking reach "
|
| 138 |
-
"social_spend spend unique_clicks unique_ctr unique_outbound_clicks "
|
| 139 |
-
"video_avg_time_watched_actions video_p100_watched_actions video_p25_watched_actions "
|
| 140 |
-
"video_p50_watched_actions video_p75_watched_actions video_p95_watched_actions "
|
| 141 |
-
"video_play_actions video_thruplay_watched_actions website_purchase_roas"
|
| 142 |
-
).split()
|
| 143 |
-
|
| 144 |
-
#: table -> (graph edge on the account | None, measured fields, parent column)
|
| 145 |
-
SPECS = {
|
| 146 |
-
"meta_ad_accounts": (None, ACCOUNT_FIELDS, None),
|
| 147 |
-
"meta_campaigns": ("campaigns", CAMPAIGN_FIELDS, "account_id"),
|
| 148 |
-
"meta_adsets": ("adsets", ADSET_FIELDS, "campaign_id"),
|
| 149 |
-
"meta_ads": ("ads", AD_FIELDS, "adset_id"),
|
| 150 |
-
"meta_creatives": ("adcreatives", CREATIVE_FIELDS, "account_id"),
|
| 151 |
-
}
|
| 152 |
-
|
| 153 |
-
#: The daily Insights grain (R2). One row per (ad, day) β the id is synthesised because Insights
|
| 154 |
-
#: has no id of its own, and it must be STABLE so a re-run updates instead of appending.
|
| 155 |
-
INSIGHTS_TABLE = "meta_insights_daily"
|
| 156 |
-
INSIGHTS_LEVEL = os.environ.get("META_INSIGHTS_LEVEL") or "ad"
|
| 157 |
-
|
| 158 |
-
#: β INSIGHTS IS FETCHED IN TIME SLICES, AND THE REASON IS MEASURED, NOT DEFENSIVE. All 57 fields
|
| 159 |
-
#: at ad level over `last_90d` with a 100-row page answers **HTTP 500 "An unknown error occurred"**
|
| 160 |
-
#: β Graph's way of saying the synchronous query is too heavy (the async report-run API is the
|
| 161 |
-
#: other answer, and it costs a poll loop this does not need). The SAME 57 fields over 7 days at
|
| 162 |
-
#: page 25 answer 200. So the window is walked in slices with every column intact:
|
| 163 |
-
#: 57 fields Β· ad level Β· 7d Β· limit 25 -> 200, 25 rows
|
| 164 |
-
#: 57 fields Β· account level Β· 7d -> 200, 7 rows
|
| 165 |
-
#: β Narrowing the FIELD list would also have "fixed" it, and that is the wrong fix twice over β
|
| 166 |
-
#: it drops columns R2 requires, and it does so invisibly.
|
| 167 |
-
INSIGHTS_DAYS = int(os.environ.get("META_INSIGHTS_DAYS") or 90)
|
| 168 |
-
INSIGHTS_SLICE = int(os.environ.get("META_INSIGHTS_SLICE_DAYS") or 7)
|
| 169 |
-
INSIGHTS_PAGE = int(os.environ.get("META_INSIGHTS_PAGE") or 25)
|
| 170 |
-
|
| 171 |
-
|
| 172 |
-
def _slices(days, size):
|
| 173 |
-
"""[(since, until)] covering the last `days`, oldest first, in `size`-day windows."""
|
| 174 |
-
from datetime import date, timedelta
|
| 175 |
-
end = date.today()
|
| 176 |
-
start = end - timedelta(days=max(1, days) - 1)
|
| 177 |
-
out, cur = [], start
|
| 178 |
-
while cur <= end:
|
| 179 |
-
stop = min(cur + timedelta(days=max(1, size) - 1), end)
|
| 180 |
-
out.append((cur.isoformat(), stop.isoformat()))
|
| 181 |
-
cur = stop + timedelta(days=1)
|
| 182 |
-
return out
|
| 183 |
-
|
| 184 |
-
#: β A REAL CEILING, AND R6's SECOND SENTENCE APPLIES TO IT. Graph pages at 25-100 rows; this is
|
| 185 |
-
#: the number of PAGES a single edge may walk before the loader stops and SAYS it stopped. It is
|
| 186 |
-
#: not a row cap on a connected source (R6 forbids that) β it is a runaway guard, and reaching it
|
| 187 |
-
#: is reported as a problem, never absorbed.
|
| 188 |
-
MAX_PAGES = int(os.environ.get("META_MAX_PAGES") or 200)
|
| 189 |
-
PAGE = int(os.environ.get("META_PAGE_SIZE") or 100)
|
| 190 |
-
|
| 191 |
-
|
| 192 |
-
class MetaError(RuntimeError):
|
| 193 |
-
"""A Graph refusal carrying Meta's own words. Safe to print: no token ever reaches it."""
|
| 194 |
-
|
| 195 |
-
|
| 196 |
-
def token():
|
| 197 |
-
"""The Meta token from the environment or gitignored `platform/.env`; "" when absent.
|
| 198 |
-
|
| 199 |
-
β Same resolver `aios-web/api/connectors_meta.py` uses. Duplicated deliberately and minimally:
|
| 200 |
-
`platform/` must not import from `aios-web/api/`, which is the layering rule this repo keeps
|
| 201 |
-
(`core` never imports up). Fifteen lines is the price of that boundary.
|
| 202 |
-
"""
|
| 203 |
-
tok = os.environ.get("META_ADS_ACCESS_TOKEN") or ""
|
| 204 |
-
if tok:
|
| 205 |
-
return tok.strip()
|
| 206 |
-
env = _HERE.parent / ".env"
|
| 207 |
-
if env.exists():
|
| 208 |
-
for line in env.read_text(encoding="utf-8", errors="replace").splitlines():
|
| 209 |
-
if line.strip().startswith("META_ADS_ACCESS_TOKEN"):
|
| 210 |
-
_, _, v = line.partition("=")
|
| 211 |
-
return v.strip().strip('"').strip("'")
|
| 212 |
-
return ""
|
| 213 |
-
|
| 214 |
-
|
| 215 |
-
def _get(path, tok, **params):
|
| 216 |
-
params["access_token"] = tok
|
| 217 |
-
url = f"{GRAPH}/{path.lstrip('/')}?" + urllib.parse.urlencode(params)
|
| 218 |
-
req = urllib.request.Request(url, headers={"User-Agent": "aios-meta-store/1"})
|
| 219 |
-
try:
|
| 220 |
-
with urllib.request.urlopen(req, timeout=120) as r:
|
| 221 |
-
return json.loads(r.read().decode("utf-8", "replace"))
|
| 222 |
-
except urllib.error.HTTPError as e:
|
| 223 |
-
try:
|
| 224 |
-
msg = (json.loads(e.read().decode("utf-8", "replace")).get("error") or {}
|
| 225 |
-
).get("message") or ""
|
| 226 |
-
except Exception:
|
| 227 |
-
msg = ""
|
| 228 |
-
raise MetaError(f"HTTP {e.code} on /{path.lstrip('/')}: {msg[:200]}") from None
|
| 229 |
-
except Exception as e:
|
| 230 |
-
raise MetaError(f"{type(e).__name__} on /{path.lstrip('/')}") from None
|
| 231 |
-
|
| 232 |
-
|
| 233 |
-
#: Graph's own words when a page is too heavy. It arrives as an HTTP **500**, not a 4xx, which is
|
| 234 |
-
#: why it cannot be treated as "the server is broken, give up".
|
| 235 |
-
_TOO_MUCH = "reduce the amount of data"
|
| 236 |
-
|
| 237 |
-
#: β THE ADS-MANAGEMENT RATE LIMIT, WHICH IS PER AD ACCOUNT AND NOT PER TOKEN. Measured: after a
|
| 238 |
-
#: heavy backfill Graph answers **HTTP 400 "There have been too many calls to this ad-account.
|
| 239 |
-
#: Wait a bit and try again."** It is a 4xx, so nothing about the status code says "retry" β the
|
| 240 |
-
#: MESSAGE is the only signal, which is why it is matched here rather than inferred from a code.
|
| 241 |
-
#: β It persists for minutes, so the backoff is measured in minutes and BOUNDED: after
|
| 242 |
-
#: `_RATE_TRIES` waits the loader STOPS and reports what it got, rather than sitting in a retry
|
| 243 |
-
#: loop nobody can see. A partial mirror that says it is partial beats a hung sync.
|
| 244 |
-
_RATE_LIMITED = "too many calls"
|
| 245 |
-
_RATE_TRIES = int(os.environ.get("META_RATE_TRIES") or 3)
|
| 246 |
-
_RATE_WAIT = int(os.environ.get("META_RATE_WAIT_S") or 90)
|
| 247 |
-
|
| 248 |
-
|
| 249 |
-
def _walk(path, tok, log, **params):
|
| 250 |
-
"""Every page of an edge, paged by CURSOR under our own parameters.
|
| 251 |
-
|
| 252 |
-
β IT DOES NOT FOLLOW GRAPH'S `paging.next` URL, AND THAT IS THE WHOLE POINT. Measured: the
|
| 253 |
-
first call to `/adcreatives` at limit=100 answers **HTTP 500 "Please reduce the amount of data
|
| 254 |
-
you're asking for"**, the retry at limit=25 succeeds β and then `next` carries the ORIGINAL
|
| 255 |
-
limit=100 and fails again on page 2. A backoff that cannot reach every page is not a backoff.
|
| 256 |
-
Re-issuing each page ourselves with `after=<cursor>` keeps the reduced limit for the whole walk.
|
| 257 |
-
|
| 258 |
-
β It shrinks the PAGE, never the FIELD LIST. Dropping columns to make a request fit is the
|
| 259 |
-
silent omission R2 forbids, and nothing downstream could see it. Fewer rows per call, always
|
| 260 |
-
every column per row.
|
| 261 |
-
|
| 262 |
-
-> (rows, hit_page_cap)
|
| 263 |
-
"""
|
| 264 |
-
out, pages = [], 0
|
| 265 |
-
limit = int(params.pop("limit", None) or PAGE)
|
| 266 |
-
after, waited = None, 0
|
| 267 |
-
while True:
|
| 268 |
-
call = dict(params, limit=limit)
|
| 269 |
-
if after:
|
| 270 |
-
call["after"] = after
|
| 271 |
-
try:
|
| 272 |
-
body = _get(path, tok, **call)
|
| 273 |
-
except MetaError as e:
|
| 274 |
-
if _TOO_MUCH in str(e) and limit > 5:
|
| 275 |
-
limit = max(5, limit // 4)
|
| 276 |
-
log(f" page too heavy for /{path} - retrying at limit={limit} "
|
| 277 |
-
f"(a payload ceiling, not a row cap; every column is still asked for)")
|
| 278 |
-
continue
|
| 279 |
-
if _RATE_LIMITED in str(e).lower() and waited < _RATE_TRIES:
|
| 280 |
-
waited += 1
|
| 281 |
-
log(f" rate-limited on /{path} (per AD ACCOUNT, not per token) - waiting "
|
| 282 |
-
f"{_RATE_WAIT}s, attempt {waited}/{_RATE_TRIES}")
|
| 283 |
-
time.sleep(_RATE_WAIT)
|
| 284 |
-
continue
|
| 285 |
-
if _RATE_LIMITED in str(e).lower():
|
| 286 |
-
log(f" !! GIVING UP on /{path} after {waited} waits: still rate-limited. "
|
| 287 |
-
f"{len(out)} row(s) collected so far are kept. Cause: the ads-management "
|
| 288 |
-
f"limit is per ad account and persists for minutes. Fix: re-run "
|
| 289 |
-
f"`--sync --only <table>` later; the upsert is idempotent.")
|
| 290 |
-
return out, True
|
| 291 |
-
raise
|
| 292 |
-
out.extend(body.get("data") or [])
|
| 293 |
-
pages += 1
|
| 294 |
-
after = ((body.get("paging") or {}).get("cursors") or {}).get("after")
|
| 295 |
-
has_next = bool((body.get("paging") or {}).get("next")) and bool(after)
|
| 296 |
-
if not has_next:
|
| 297 |
-
return out, False
|
| 298 |
-
if pages >= MAX_PAGES:
|
| 299 |
-
log(f" !! STOPPED at MAX_PAGES={MAX_PAGES} on /{path} with more pages left. "
|
| 300 |
-
f"Cause: a runaway guard, not a row cap. Fix: raise META_MAX_PAGES, or narrow the "
|
| 301 |
-
f"window with META_INSIGHTS_DAYS.")
|
| 302 |
-
return out, True
|
| 303 |
-
|
| 304 |
-
|
| 305 |
-
def _cell(value):
|
| 306 |
-
"""One Graph value -> one DuckDB cell. Nested structures become compact JSON TEXT rather than
|
| 307 |
-
being dropped: R2 says every field the API returns, and `targeting` is a field."""
|
| 308 |
-
if value is None or isinstance(value, (str, int, float, bool)):
|
| 309 |
-
return value
|
| 310 |
-
return json.dumps(value, separators=(",", ":"), ensure_ascii=False)
|
| 311 |
-
|
| 312 |
-
|
| 313 |
-
def _coltype(name, rows=None):
|
| 314 |
-
"""The DuckDB type for one column β decided by the DATA when there is data, by the name only
|
| 315 |
-
as a fallback.
|
| 316 |
-
|
| 317 |
-
β THE NAME LIST WAS WRONG AND ONLY THE API COULD SAY SO. `cost_per_unique_click`,
|
| 318 |
-
`cost_per_action_type`, `purchase_roas` and friends READ like money and are **arrays of
|
| 319 |
-
action-type objects** on the Insights edge:
|
| 320 |
-
[{"action_type":"outbound_click","value":"1.459854"}]
|
| 321 |
-
Typed DOUBLE from `_DBL`, the insert died with `Conversion Error: Could not convert string
|
| 322 |
-
'[{...}]' to DOUBLE` β after five entity tables had already been written, so the sync looked
|
| 323 |
-
like it worked and then blew up on the last table.
|
| 324 |
-
β Same principle the field CATALOG is built on, applied one layer down: **ask the response,
|
| 325 |
-
do not assert from a list.** A value that ever arrives as a list or dict is JSON TEXT, because
|
| 326 |
-
that is what `_cell` stores; anything else falls back to the measured name hints.
|
| 327 |
-
"""
|
| 328 |
-
if rows:
|
| 329 |
-
seen, kinds = 0, set()
|
| 330 |
-
for r in rows:
|
| 331 |
-
v = r.get(name)
|
| 332 |
-
if v is None or v == "":
|
| 333 |
-
continue
|
| 334 |
-
kinds.add("json" if isinstance(v, (list, dict)) else
|
| 335 |
-
"bool" if isinstance(v, bool) else
|
| 336 |
-
"int" if isinstance(v, int) else
|
| 337 |
-
"float" if isinstance(v, float) else "str")
|
| 338 |
-
seen += 1
|
| 339 |
-
if seen >= 200:
|
| 340 |
-
break
|
| 341 |
-
if kinds:
|
| 342 |
-
if "json" in kinds or "str" in kinds:
|
| 343 |
-
return "VARCHAR" # a JSON blob, or a numeric STRING
|
| 344 |
-
if kinds <= {"int", "bool"}:
|
| 345 |
-
return "BIGINT" if name in _INT else ("VARCHAR" if "bool" in kinds else "BIGINT")
|
| 346 |
-
return "DOUBLE"
|
| 347 |
-
if name in _INT:
|
| 348 |
-
return "BIGINT"
|
| 349 |
-
if name in _DBL:
|
| 350 |
-
return "DOUBLE"
|
| 351 |
-
return "VARCHAR" # ids included β see the header
|
| 352 |
-
|
| 353 |
-
|
| 354 |
-
def _ensure(con, table, fields, rows=None):
|
| 355 |
-
"""Create or widen the table, with every column typed from `rows` when they are available.
|
| 356 |
-
|
| 357 |
-
β AN EXISTING COLUMN WHOSE TYPE IS NOW WRONG IS REBUILT, NOT PATCHED. DuckDB cannot retype a
|
| 358 |
-
column in place, and this table is DERIVED data that can be re-pulled in minutes β so a type
|
| 359 |
-
disagreement drops and recreates rather than limping on with a column that refuses every
|
| 360 |
-
insert. The alternative is a mirror that is permanently unwritable for one bad guess.
|
| 361 |
-
"""
|
| 362 |
-
want = {f: _coltype(f, rows) for f in fields}
|
| 363 |
-
have = {r[1]: str(r[2]).upper() for r in con.execute(f"PRAGMA table_info('{table}')").fetchall()}
|
| 364 |
-
if have:
|
| 365 |
-
clash = [f for f, ty in want.items() if f in have and have[f] != ty
|
| 366 |
-
and not (have[f].startswith("VARCHAR") and ty == "VARCHAR")]
|
| 367 |
-
if clash:
|
| 368 |
-
con.execute(f"DROP TABLE {table}")
|
| 369 |
-
have = {}
|
| 370 |
-
if not have:
|
| 371 |
-
cols = ", ".join(f"{f} {want[f]}" for f in fields)
|
| 372 |
-
con.execute(f"CREATE TABLE IF NOT EXISTS {table} (id VARCHAR PRIMARY KEY, {cols})"
|
| 373 |
-
if "id" not in fields else
|
| 374 |
-
f"CREATE TABLE IF NOT EXISTS {table} ({cols})")
|
| 375 |
-
return
|
| 376 |
-
for f in fields:
|
| 377 |
-
if f not in have:
|
| 378 |
-
con.execute(f"ALTER TABLE {table} ADD COLUMN {f} {want[f]}")
|
| 379 |
-
|
| 380 |
-
|
| 381 |
-
def _upsert(con, table, fields, rows):
|
| 382 |
-
"""Delete-then-insert by id β the same idempotence `datastore._upsert` gives the Odoo half, so
|
| 383 |
-
a re-sync updates in place and can never append a second copy of the same object."""
|
| 384 |
-
if not rows:
|
| 385 |
-
return 0
|
| 386 |
-
cols = list(fields)
|
| 387 |
-
ids = [str(r.get("id") or "") for r in rows]
|
| 388 |
-
q = ",".join("?" for _ in ids)
|
| 389 |
-
con.execute(f"DELETE FROM {table} WHERE id IN ({q})", ids)
|
| 390 |
-
con.executemany(
|
| 391 |
-
f"INSERT INTO {table} ({', '.join(cols)}) VALUES ({', '.join('?' for _ in cols)})",
|
| 392 |
-
[[_cell(r.get(c)) for c in cols] for r in rows])
|
| 393 |
-
return len(rows)
|
| 394 |
-
|
| 395 |
-
|
| 396 |
-
def sync(tenant_key="royal-imports", tok=None, log=print, insights=True, insights_days=None):
|
| 397 |
-
"""Pull every level into the tenant's mirror. -> a report dict; raises only on a bad token.
|
| 398 |
-
|
| 399 |
-
Idempotent: re-running updates rows in place. Safe to call at boot and on the resync loop,
|
| 400 |
-
exactly as `odoo_relational.refresh` is.
|
| 401 |
-
"""
|
| 402 |
-
tok = tok or token()
|
| 403 |
-
# β THE WINDOW IS A PARAMETER, NOT AN ENVIRONMENT READ AT CALL TIME β and the difference cost a
|
| 404 |
-
# live deploy. `INSIGHTS_DAYS` binds at IMPORT (module scope), so `main._pull_meta`'s
|
| 405 |
-
# `os.environ.setdefault("META_INSIGHTS_DAYS", "7")` executed AFTER this module was already
|
| 406 |
-
# imported and changed nothing: every boot pulled **90 days**, not 7, which is the slow path
|
| 407 |
-
# that trips the per-ad-account rate limit and never finishes. The comment beside that call
|
| 408 |
-
# claimed a short window the code could not deliver.
|
| 409 |
-
# β The shape: **a knob read at import cannot be turned by a caller at runtime.** Passing it
|
| 410 |
-
# makes the caller's intent effective instead of aspirational; the env var stays the DEFAULT.
|
| 411 |
-
days = int(insights_days or INSIGHTS_DAYS)
|
| 412 |
-
report = {"tenant": tenant_key, "tables": {}, "problems": [], "accounts": []}
|
| 413 |
-
if not tok:
|
| 414 |
-
report["problems"].append(
|
| 415 |
-
"META_ADS_ACCESS_TOKEN is not set in the environment or in platform/.env, so nothing "
|
| 416 |
-
"was pulled. This is a missing CREDENTIAL, not a missing capability.")
|
| 417 |
-
return report
|
| 418 |
-
|
| 419 |
-
path = datastore.path_for(tenant_key)
|
| 420 |
-
if Path(datastore.DB_PATH) != Path(path):
|
| 421 |
-
datastore.use_path(path)
|
| 422 |
-
con = datastore.connect()
|
| 423 |
-
log(f" mirror: {Path(path).name}")
|
| 424 |
-
|
| 425 |
-
accts, _ = _walk("me/adaccounts", tok, log, fields="id,name", limit=PAGE)
|
| 426 |
-
report["accounts"] = [a.get("id") for a in accts]
|
| 427 |
-
if not accts:
|
| 428 |
-
report["problems"].append("the token reaches no ad accounts")
|
| 429 |
-
return report
|
| 430 |
-
|
| 431 |
-
# ββ the five entity levels ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 432 |
-
for table, (edge, fields, _parent) in SPECS.items():
|
| 433 |
-
rows, capped = [], False
|
| 434 |
-
for acct in accts:
|
| 435 |
-
aid = acct["id"]
|
| 436 |
-
if edge is None:
|
| 437 |
-
rows.append(_get(aid, tok, fields=",".join(fields)))
|
| 438 |
-
else:
|
| 439 |
-
got, hit = _walk(f"{aid}/{edge}", tok, log, fields=",".join(fields), limit=PAGE)
|
| 440 |
-
rows.extend(got)
|
| 441 |
-
capped = capped or hit
|
| 442 |
-
_ensure(con, table, fields, rows)
|
| 443 |
-
n = _upsert(con, table, fields, rows)
|
| 444 |
-
total = con.execute(f"SELECT count(*) FROM {table}").fetchone()[0]
|
| 445 |
-
report["tables"][table] = {"pulled": n, "in_mirror": total, "capped": capped}
|
| 446 |
-
log(f" {table:<20} pulled {n:>6} mirror total {total:>6}"
|
| 447 |
-
+ (" !! PAGE CAP HIT" if capped else ""))
|
| 448 |
-
if capped:
|
| 449 |
-
report["problems"].append(f"{table}: stopped at MAX_PAGES={MAX_PAGES}")
|
| 450 |
-
|
| 451 |
-
# ββ the daily Insights grain ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 452 |
-
if insights:
|
| 453 |
-
rows, capped = [], False
|
| 454 |
-
for acct in accts:
|
| 455 |
-
for since, until in _slices(days, INSIGHTS_SLICE):
|
| 456 |
-
got, hit = _walk(f"{acct['id']}/insights", tok, log,
|
| 457 |
-
fields=",".join(INSIGHT_FIELDS), level=INSIGHTS_LEVEL,
|
| 458 |
-
time_increment="1", limit=INSIGHTS_PAGE,
|
| 459 |
-
time_range=json.dumps({"since": since, "until": until}))
|
| 460 |
-
rows.extend(got)
|
| 461 |
-
capped = capped or hit
|
| 462 |
-
# β Insights rows carry no id. The key must be STABLE across runs or every re-sync
|
| 463 |
-
# appends a second copy of the same day β so it is composed from the grain itself.
|
| 464 |
-
for r in rows:
|
| 465 |
-
r["id"] = ":".join(str(r.get(k) or "") for k in
|
| 466 |
-
(f"{INSIGHTS_LEVEL}_id", "date_start", "date_stop"))
|
| 467 |
-
fields = ["id"] + INSIGHT_FIELDS
|
| 468 |
-
_ensure(con, INSIGHTS_TABLE, fields, rows)
|
| 469 |
-
n = _upsert(con, INSIGHTS_TABLE, fields, rows)
|
| 470 |
-
total = con.execute(f"SELECT count(*) FROM {INSIGHTS_TABLE}").fetchone()[0]
|
| 471 |
-
report["tables"][INSIGHTS_TABLE] = {"pulled": n, "in_mirror": total, "capped": capped}
|
| 472 |
-
log(f" {INSIGHTS_TABLE:<20} pulled {n:>6} mirror total {total:>6}"
|
| 473 |
-
+ (" !! PAGE CAP HIT" if capped else ""))
|
| 474 |
-
|
| 475 |
-
con.execute("INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)",
|
| 476 |
-
["meta", "done", 0, f"{days}d/{INSIGHTS_SLICE}d@{INSIGHTS_LEVEL}",
|
| 477 |
-
sum(t["in_mirror"] for t in report["tables"].values()),
|
| 478 |
-
time.strftime("%Y-%m-%d %H:%M:%S")])
|
| 479 |
-
return report
|
| 480 |
-
|
| 481 |
-
|
| 482 |
-
def status(tenant_key="royal-imports"):
|
| 483 |
-
"""What is in the mirror right now, per table. Never fetches."""
|
| 484 |
-
path = datastore.path_for(tenant_key)
|
| 485 |
-
if Path(datastore.DB_PATH) != Path(path):
|
| 486 |
-
datastore.use_path(path)
|
| 487 |
-
out = {}
|
| 488 |
-
con = datastore.connect()
|
| 489 |
-
for table in list(SPECS) + [INSIGHTS_TABLE]:
|
| 490 |
-
try:
|
| 491 |
-
out[table] = con.execute(f"SELECT count(*) FROM {table}").fetchone()[0]
|
| 492 |
-
except Exception:
|
| 493 |
-
out[table] = None # table absent = never synced
|
| 494 |
-
return out
|
| 495 |
-
|
| 496 |
-
|
| 497 |
-
def main(argv=None):
|
| 498 |
-
ap = argparse.ArgumentParser(description="Pull Meta Ads into the tenant's DuckDB mirror.")
|
| 499 |
-
ap.add_argument("--sync", action="store_true")
|
| 500 |
-
ap.add_argument("--status", action="store_true")
|
| 501 |
-
ap.add_argument("--tenant", default="royal-imports")
|
| 502 |
-
ap.add_argument("--no-insights", action="store_true")
|
| 503 |
-
ap.add_argument("--insights-days", type=int, default=None,
|
| 504 |
-
help="override the Insights window for THIS run (default INSIGHTS_DAYS); the env var is the default, this is the caller's say")
|
| 505 |
-
a = ap.parse_args(argv)
|
| 506 |
-
if a.status:
|
| 507 |
-
for k, v in status(a.tenant).items():
|
| 508 |
-
print(f" {k:<20} {'(never synced)' if v is None else v}")
|
| 509 |
-
return 0
|
| 510 |
-
if not a.sync:
|
| 511 |
-
ap.print_help()
|
| 512 |
-
return 2
|
| 513 |
-
rep = sync(a.tenant, insights=not a.no_insights, insights_days=a.insights_days)
|
| 514 |
-
for p in rep["problems"]:
|
| 515 |
-
print(" PROBLEM:", p)
|
| 516 |
-
print(f" accounts: {len(rep['accounts'])} tables: {len(rep['tables'])}")
|
| 517 |
-
return 1 if rep["problems"] else 0
|
| 518 |
-
|
| 519 |
-
|
| 520 |
-
if __name__ == "__main__":
|
| 521 |
-
sys.exit(main())
|
|
|
|
| 1 |
+
"""meta_store.py β the REST loader for a mirror whose only other loader is XML-RPC (W31-T48).
|
| 2 |
+
|
| 3 |
+
python platform/harness/meta_store.py --sync pull into tenant #0's mirror
|
| 4 |
+
python platform/harness/meta_store.py --sync --tenant gtmlab
|
| 5 |
+
python platform/harness/meta_store.py --status what is in the mirror now
|
| 6 |
+
|
| 7 |
+
β WHY THIS IS A SIBLING AND NOT A ROW IN `datastore.ENTITIES`. `datastore.sync_entity` is
|
| 8 |
+
hardwired to XML-RPC β it does `import core.odoo as O` and speaks `search_read` β so no amount of
|
| 9 |
+
spec data makes it fetch over HTTPS. What IS reusable is everything *below* the fetch: the
|
| 10 |
+
per-tenant file (`path_for`), the process-singleton connection (`connect`), and the
|
| 11 |
+
delete-then-insert upsert. This module borrows those and brings its own reader. **The mirror is one
|
| 12 |
+
store with two loaders, not two stores.**
|
| 13 |
+
|
| 14 |
+
β THE SHAPE OF THE WHOLE THING, because it is the owner's actual request: *"we just need to pull
|
| 15 |
+
the data into our template database for Meta"*, working *"just like Odoo"*. Odoo's path is
|
| 16 |
+
`XML-RPC -> DuckDB mirror -> odoo_relational -> ut_odoo_* locked grids`. Meta's is
|
| 17 |
+
`Graph -> DuckDB mirror -> meta_relational -> ut_meta_* locked grids`. Same middle, same end, one
|
| 18 |
+
different first hop.
|
| 19 |
+
|
| 20 |
+
β EVERY COLUMN NAME BELOW WAS MEASURED, NOT READ OFF A DOC (R8). `proto/meta-entity-fields.json`
|
| 21 |
+
carries the run: each level's list is what the API ACCEPTED when asked, on a real ad account.
|
| 22 |
+
Ad Account 45 Β· Campaign 35 Β· Ad Set 54 Β· Ad 36 Β· Creative 58 Β· Insights 57
|
| 23 |
+
β AND `ACCEPTED` IS NOT `RETURNED`. Graph omits null fields from a response, so a 20-field ask
|
| 24 |
+
came back with 14 keys. Building a schema from what came back would drop ~30% of the columns with
|
| 25 |
+
nothing going red β which is exactly the "do not drop any column" instruction, broken silently.
|
| 26 |
+
The lists below are therefore the ASKED-AND-ACCEPTED set, and a column with no value is NULL.
|
| 27 |
+
|
| 28 |
+
β `owner` IS ABSENT FROM THE AD ACCOUNT LIST AND THAT IS A REPORTED LIMIT, NOT AN OMISSION: it
|
| 29 |
+
answers `403 (#200) Requires business_management permission`, which this token does not carry. One
|
| 30 |
+
column of 285. R6's second sentence β a limit that cannot be removed gets stated with its cause.
|
| 31 |
+
|
| 32 |
+
β IDS ARE TEXT, ALWAYS. A Meta object id is a 17-digit decimal string; `120273975028650555` is
|
| 33 |
+
larger than 2^53, so any float or JS-number path silently corrupts it. Same ruling as `ig_id`
|
| 34 |
+
(W26/R3), for the same reason, and it is why every `id` column here is VARCHAR.
|
| 35 |
+
"""
|
| 36 |
+
import argparse
|
| 37 |
+
import json
|
| 38 |
+
import os
|
| 39 |
+
import sys
|
| 40 |
+
import time
|
| 41 |
+
import urllib.error
|
| 42 |
+
import urllib.parse
|
| 43 |
+
import urllib.request
|
| 44 |
+
from pathlib import Path
|
| 45 |
+
|
| 46 |
+
_HERE = Path(__file__).resolve().parent
|
| 47 |
+
if str(_HERE.parent) not in sys.path:
|
| 48 |
+
sys.path.insert(0, str(_HERE.parent))
|
| 49 |
+
|
| 50 |
+
from harness import datastore # noqa: E402
|
| 51 |
+
|
| 52 |
+
GRAPH_VERSION = os.environ.get("META_GRAPH_VERSION") or "v21.0"
|
| 53 |
+
GRAPH = f"https://graph.facebook.com/{GRAPH_VERSION}"
|
| 54 |
+
|
| 55 |
+
#: Numeric columns, by name. Everything else is VARCHAR β including ids (see the header) and
|
| 56 |
+
#: including anything Graph returns as a nested object, which is stored as compact JSON text.
|
| 57 |
+
_INT = {"impressions", "reach", "clicks", "unique_clicks", "inline_link_clicks",
|
| 58 |
+
"inline_post_engagement", "full_view_impressions", "estimated_ad_recallers",
|
| 59 |
+
"account_status", "age", "timezone_id", "timezone_offset_hours_utc", "io_number",
|
| 60 |
+
"min_daily_budget", "min_campaign_group_spend_cap"}
|
| 61 |
+
_DBL = {"spend", "social_spend", "ctr", "unique_ctr", "cpc", "cpm", "cpp", "frequency",
|
| 62 |
+
"inline_link_click_ctr", "outbound_clicks_ctr", "amount_spent", "balance", "spend_cap",
|
| 63 |
+
"daily_budget", "lifetime_budget", "budget_remaining", "bid_amount", "canvas_avg_view_time",
|
| 64 |
+
"daily_min_spend_target", "daily_spend_cap", "lifetime_min_spend_target",
|
| 65 |
+
"lifetime_spend_cap", "lifetime_imps"}
|
| 66 |
+
|
| 67 |
+
#: β THE MEASURED CATALOG. `edge` is the connection on the ad account; `None` means the account
|
| 68 |
+
#: itself. `parent` names the column that ties a row to its parent, used by nothing here and by
|
| 69 |
+
#: the grid links in `meta_relational`.
|
| 70 |
+
ACCOUNT_FIELDS = (
|
| 71 |
+
"account_id account_status age amount_spent balance business_city business_country_code "
|
| 72 |
+
"business_name business_state business_street business_street2 business_zip capabilities "
|
| 73 |
+
"created_time currency disable_reason end_advertiser end_advertiser_name funding_source "
|
| 74 |
+
"has_migrated_permissions id io_number is_attribution_spec_system_default "
|
| 75 |
+
"is_direct_deals_enabled is_notifications_enabled is_personal is_prepay_account "
|
| 76 |
+
"is_tax_id_required line_numbers media_agency min_campaign_group_spend_cap min_daily_budget "
|
| 77 |
+
"name offsite_pixels_tos_accepted partner spend_cap tax_id tax_id_status tax_id_type "
|
| 78 |
+
"timezone_id timezone_name timezone_offset_hours_utc tos_accepted user_tasks user_tos_accepted"
|
| 79 |
+
).split()
|
| 80 |
+
|
| 81 |
+
CAMPAIGN_FIELDS = (
|
| 82 |
+
"account_id bid_strategy boosted_object_id budget_rebalance_flag budget_remaining buying_type "
|
| 83 |
+
"campaign_group_active_time can_create_brand_lift_study can_use_spend_cap configured_status "
|
| 84 |
+
"created_time daily_budget effective_status id is_skadnetwork_attribution issues_info "
|
| 85 |
+
"last_budget_toggling_time lifetime_budget name objective pacing_type primary_attribution "
|
| 86 |
+
"promoted_object smart_promotion_type source_campaign source_campaign_id special_ad_categories "
|
| 87 |
+
"special_ad_category special_ad_category_country spend_cap start_time status stop_time "
|
| 88 |
+
"topline_id updated_time"
|
| 89 |
+
).split()
|
| 90 |
+
|
| 91 |
+
ADSET_FIELDS = (
|
| 92 |
+
"account_id adlabels adset_schedule asset_feed_id attribution_spec bid_adjustments bid_amount "
|
| 93 |
+
"bid_constraints bid_info bid_strategy billing_event budget_remaining campaign "
|
| 94 |
+
"campaign_active_time campaign_attribution campaign_id configured_status created_time "
|
| 95 |
+
"creative_sequence daily_budget daily_min_spend_target daily_spend_cap destination_type "
|
| 96 |
+
"effective_status end_time frequency_control_specs id instagram_actor_id is_dynamic_creative "
|
| 97 |
+
"issues_info learning_stage_info lifetime_budget lifetime_imps lifetime_min_spend_target "
|
| 98 |
+
"lifetime_spend_cap multi_optimization_goal_weight name optimization_goal "
|
| 99 |
+
"optimization_sub_event pacing_type promoted_object recurring_budget_semantics review_feedback "
|
| 100 |
+
"rf_prediction_id source_adset source_adset_id start_time status targeting "
|
| 101 |
+
"targeting_optimization_types time_based_ad_rotation_id_blocks "
|
| 102 |
+
"time_based_ad_rotation_intervals updated_time use_new_app_click"
|
| 103 |
+
).split()
|
| 104 |
+
|
| 105 |
+
AD_FIELDS = (
|
| 106 |
+
"account_id ad_active_time ad_review_feedback ad_schedule_end_time ad_schedule_start_time "
|
| 107 |
+
"adlabels adset adset_id bid_amount bid_info bid_type campaign campaign_id configured_status "
|
| 108 |
+
"conversion_domain created_time creative demolink_hash display_sequence effective_status "
|
| 109 |
+
"engagement_audience failed_delivery_checks id issues_info last_updated_by_app_id name "
|
| 110 |
+
"preview_shareable_link priority recommendations source_ad source_ad_id status targeting "
|
| 111 |
+
"tracking_and_conversion_with_defaults tracking_specs updated_time"
|
| 112 |
+
).split()
|
| 113 |
+
|
| 114 |
+
CREATIVE_FIELDS = (
|
| 115 |
+
"account_id actor_id adlabels applink_treatment asset_feed_spec authorization_category body "
|
| 116 |
+
"branded_content_sponsor_page_id bundle_folder_id call_to_action_type categorization_criteria "
|
| 117 |
+
"category_media_source collaborative_ads_lsb_image_bank_id degrees_of_freedom_spec "
|
| 118 |
+
"destination_set_id dynamic_ad_voice effective_authorization_category "
|
| 119 |
+
"effective_instagram_media_id effective_object_story_id enable_direct_install "
|
| 120 |
+
"enable_launch_instant_app id image_crops image_hash image_url instagram_actor_id "
|
| 121 |
+
"instagram_permalink_url instagram_story_id instagram_user_id interactive_components_spec "
|
| 122 |
+
"link_deep_link_url link_destination_display_url link_og_id link_url "
|
| 123 |
+
"messenger_sponsored_message name object_id object_store_url object_story_id object_story_spec "
|
| 124 |
+
"object_type object_url place_page_set_id platform_customizations playable_asset_id "
|
| 125 |
+
"portrait_customizations product_set_id recommender_settings source_instagram_media_id status "
|
| 126 |
+
"template_url template_url_spec thumbnail_id thumbnail_url title url_tags "
|
| 127 |
+
"use_page_actor_override video_id"
|
| 128 |
+
).split()
|
| 129 |
+
|
| 130 |
+
INSIGHT_FIELDS = (
|
| 131 |
+
"account_currency account_id account_name action_values actions ad_id ad_name adset_id "
|
| 132 |
+
"adset_name attribution_setting buying_type campaign_id campaign_name canvas_avg_view_time "
|
| 133 |
+
"clicks conversion_rate_ranking conversion_values conversions cost_per_action_type "
|
| 134 |
+
"cost_per_inline_link_click cost_per_thruplay cost_per_unique_click cpc cpm cpp ctr date_start "
|
| 135 |
+
"date_stop engagement_rate_ranking estimated_ad_recallers frequency full_view_impressions "
|
| 136 |
+
"impressions inline_link_click_ctr inline_link_clicks inline_post_engagement objective "
|
| 137 |
+
"optimization_goal outbound_clicks outbound_clicks_ctr purchase_roas quality_ranking reach "
|
| 138 |
+
"social_spend spend unique_clicks unique_ctr unique_outbound_clicks "
|
| 139 |
+
"video_avg_time_watched_actions video_p100_watched_actions video_p25_watched_actions "
|
| 140 |
+
"video_p50_watched_actions video_p75_watched_actions video_p95_watched_actions "
|
| 141 |
+
"video_play_actions video_thruplay_watched_actions website_purchase_roas"
|
| 142 |
+
).split()
|
| 143 |
+
|
| 144 |
+
#: table -> (graph edge on the account | None, measured fields, parent column)
|
| 145 |
+
SPECS = {
|
| 146 |
+
"meta_ad_accounts": (None, ACCOUNT_FIELDS, None),
|
| 147 |
+
"meta_campaigns": ("campaigns", CAMPAIGN_FIELDS, "account_id"),
|
| 148 |
+
"meta_adsets": ("adsets", ADSET_FIELDS, "campaign_id"),
|
| 149 |
+
"meta_ads": ("ads", AD_FIELDS, "adset_id"),
|
| 150 |
+
"meta_creatives": ("adcreatives", CREATIVE_FIELDS, "account_id"),
|
| 151 |
+
}
|
| 152 |
+
|
| 153 |
+
#: The daily Insights grain (R2). One row per (ad, day) β the id is synthesised because Insights
|
| 154 |
+
#: has no id of its own, and it must be STABLE so a re-run updates instead of appending.
|
| 155 |
+
INSIGHTS_TABLE = "meta_insights_daily"
|
| 156 |
+
INSIGHTS_LEVEL = os.environ.get("META_INSIGHTS_LEVEL") or "ad"
|
| 157 |
+
|
| 158 |
+
#: β INSIGHTS IS FETCHED IN TIME SLICES, AND THE REASON IS MEASURED, NOT DEFENSIVE. All 57 fields
|
| 159 |
+
#: at ad level over `last_90d` with a 100-row page answers **HTTP 500 "An unknown error occurred"**
|
| 160 |
+
#: β Graph's way of saying the synchronous query is too heavy (the async report-run API is the
|
| 161 |
+
#: other answer, and it costs a poll loop this does not need). The SAME 57 fields over 7 days at
|
| 162 |
+
#: page 25 answer 200. So the window is walked in slices with every column intact:
|
| 163 |
+
#: 57 fields Β· ad level Β· 7d Β· limit 25 -> 200, 25 rows
|
| 164 |
+
#: 57 fields Β· account level Β· 7d -> 200, 7 rows
|
| 165 |
+
#: β Narrowing the FIELD list would also have "fixed" it, and that is the wrong fix twice over β
|
| 166 |
+
#: it drops columns R2 requires, and it does so invisibly.
|
| 167 |
+
INSIGHTS_DAYS = int(os.environ.get("META_INSIGHTS_DAYS") or 90)
|
| 168 |
+
INSIGHTS_SLICE = int(os.environ.get("META_INSIGHTS_SLICE_DAYS") or 7)
|
| 169 |
+
INSIGHTS_PAGE = int(os.environ.get("META_INSIGHTS_PAGE") or 25)
|
| 170 |
+
|
| 171 |
+
|
| 172 |
+
def _slices(days, size):
|
| 173 |
+
"""[(since, until)] covering the last `days`, oldest first, in `size`-day windows."""
|
| 174 |
+
from datetime import date, timedelta
|
| 175 |
+
end = date.today()
|
| 176 |
+
start = end - timedelta(days=max(1, days) - 1)
|
| 177 |
+
out, cur = [], start
|
| 178 |
+
while cur <= end:
|
| 179 |
+
stop = min(cur + timedelta(days=max(1, size) - 1), end)
|
| 180 |
+
out.append((cur.isoformat(), stop.isoformat()))
|
| 181 |
+
cur = stop + timedelta(days=1)
|
| 182 |
+
return out
|
| 183 |
+
|
| 184 |
+
#: β A REAL CEILING, AND R6's SECOND SENTENCE APPLIES TO IT. Graph pages at 25-100 rows; this is
|
| 185 |
+
#: the number of PAGES a single edge may walk before the loader stops and SAYS it stopped. It is
|
| 186 |
+
#: not a row cap on a connected source (R6 forbids that) β it is a runaway guard, and reaching it
|
| 187 |
+
#: is reported as a problem, never absorbed.
|
| 188 |
+
MAX_PAGES = int(os.environ.get("META_MAX_PAGES") or 200)
|
| 189 |
+
PAGE = int(os.environ.get("META_PAGE_SIZE") or 100)
|
| 190 |
+
|
| 191 |
+
|
| 192 |
+
class MetaError(RuntimeError):
|
| 193 |
+
"""A Graph refusal carrying Meta's own words. Safe to print: no token ever reaches it."""
|
| 194 |
+
|
| 195 |
+
|
| 196 |
+
def token():
|
| 197 |
+
"""The Meta token from the environment or gitignored `platform/.env`; "" when absent.
|
| 198 |
+
|
| 199 |
+
β Same resolver `aios-web/api/connectors_meta.py` uses. Duplicated deliberately and minimally:
|
| 200 |
+
`platform/` must not import from `aios-web/api/`, which is the layering rule this repo keeps
|
| 201 |
+
(`core` never imports up). Fifteen lines is the price of that boundary.
|
| 202 |
+
"""
|
| 203 |
+
tok = os.environ.get("META_ADS_ACCESS_TOKEN") or ""
|
| 204 |
+
if tok:
|
| 205 |
+
return tok.strip()
|
| 206 |
+
env = _HERE.parent / ".env"
|
| 207 |
+
if env.exists():
|
| 208 |
+
for line in env.read_text(encoding="utf-8", errors="replace").splitlines():
|
| 209 |
+
if line.strip().startswith("META_ADS_ACCESS_TOKEN"):
|
| 210 |
+
_, _, v = line.partition("=")
|
| 211 |
+
return v.strip().strip('"').strip("'")
|
| 212 |
+
return ""
|
| 213 |
+
|
| 214 |
+
|
| 215 |
+
def _get(path, tok, **params):
|
| 216 |
+
params["access_token"] = tok
|
| 217 |
+
url = f"{GRAPH}/{path.lstrip('/')}?" + urllib.parse.urlencode(params)
|
| 218 |
+
req = urllib.request.Request(url, headers={"User-Agent": "aios-meta-store/1"})
|
| 219 |
+
try:
|
| 220 |
+
with urllib.request.urlopen(req, timeout=120) as r:
|
| 221 |
+
return json.loads(r.read().decode("utf-8", "replace"))
|
| 222 |
+
except urllib.error.HTTPError as e:
|
| 223 |
+
try:
|
| 224 |
+
msg = (json.loads(e.read().decode("utf-8", "replace")).get("error") or {}
|
| 225 |
+
).get("message") or ""
|
| 226 |
+
except Exception:
|
| 227 |
+
msg = ""
|
| 228 |
+
raise MetaError(f"HTTP {e.code} on /{path.lstrip('/')}: {msg[:200]}") from None
|
| 229 |
+
except Exception as e:
|
| 230 |
+
raise MetaError(f"{type(e).__name__} on /{path.lstrip('/')}") from None
|
| 231 |
+
|
| 232 |
+
|
| 233 |
+
#: Graph's own words when a page is too heavy. It arrives as an HTTP **500**, not a 4xx, which is
|
| 234 |
+
#: why it cannot be treated as "the server is broken, give up".
|
| 235 |
+
_TOO_MUCH = "reduce the amount of data"
|
| 236 |
+
|
| 237 |
+
#: β THE ADS-MANAGEMENT RATE LIMIT, WHICH IS PER AD ACCOUNT AND NOT PER TOKEN. Measured: after a
|
| 238 |
+
#: heavy backfill Graph answers **HTTP 400 "There have been too many calls to this ad-account.
|
| 239 |
+
#: Wait a bit and try again."** It is a 4xx, so nothing about the status code says "retry" β the
|
| 240 |
+
#: MESSAGE is the only signal, which is why it is matched here rather than inferred from a code.
|
| 241 |
+
#: β It persists for minutes, so the backoff is measured in minutes and BOUNDED: after
|
| 242 |
+
#: `_RATE_TRIES` waits the loader STOPS and reports what it got, rather than sitting in a retry
|
| 243 |
+
#: loop nobody can see. A partial mirror that says it is partial beats a hung sync.
|
| 244 |
+
_RATE_LIMITED = "too many calls"
|
| 245 |
+
_RATE_TRIES = int(os.environ.get("META_RATE_TRIES") or 3)
|
| 246 |
+
_RATE_WAIT = int(os.environ.get("META_RATE_WAIT_S") or 90)
|
| 247 |
+
|
| 248 |
+
|
| 249 |
+
def _walk(path, tok, log, **params):
|
| 250 |
+
"""Every page of an edge, paged by CURSOR under our own parameters.
|
| 251 |
+
|
| 252 |
+
β IT DOES NOT FOLLOW GRAPH'S `paging.next` URL, AND THAT IS THE WHOLE POINT. Measured: the
|
| 253 |
+
first call to `/adcreatives` at limit=100 answers **HTTP 500 "Please reduce the amount of data
|
| 254 |
+
you're asking for"**, the retry at limit=25 succeeds β and then `next` carries the ORIGINAL
|
| 255 |
+
limit=100 and fails again on page 2. A backoff that cannot reach every page is not a backoff.
|
| 256 |
+
Re-issuing each page ourselves with `after=<cursor>` keeps the reduced limit for the whole walk.
|
| 257 |
+
|
| 258 |
+
β It shrinks the PAGE, never the FIELD LIST. Dropping columns to make a request fit is the
|
| 259 |
+
silent omission R2 forbids, and nothing downstream could see it. Fewer rows per call, always
|
| 260 |
+
every column per row.
|
| 261 |
+
|
| 262 |
+
-> (rows, hit_page_cap)
|
| 263 |
+
"""
|
| 264 |
+
out, pages = [], 0
|
| 265 |
+
limit = int(params.pop("limit", None) or PAGE)
|
| 266 |
+
after, waited = None, 0
|
| 267 |
+
while True:
|
| 268 |
+
call = dict(params, limit=limit)
|
| 269 |
+
if after:
|
| 270 |
+
call["after"] = after
|
| 271 |
+
try:
|
| 272 |
+
body = _get(path, tok, **call)
|
| 273 |
+
except MetaError as e:
|
| 274 |
+
if _TOO_MUCH in str(e) and limit > 5:
|
| 275 |
+
limit = max(5, limit // 4)
|
| 276 |
+
log(f" page too heavy for /{path} - retrying at limit={limit} "
|
| 277 |
+
f"(a payload ceiling, not a row cap; every column is still asked for)")
|
| 278 |
+
continue
|
| 279 |
+
if _RATE_LIMITED in str(e).lower() and waited < _RATE_TRIES:
|
| 280 |
+
waited += 1
|
| 281 |
+
log(f" rate-limited on /{path} (per AD ACCOUNT, not per token) - waiting "
|
| 282 |
+
f"{_RATE_WAIT}s, attempt {waited}/{_RATE_TRIES}")
|
| 283 |
+
time.sleep(_RATE_WAIT)
|
| 284 |
+
continue
|
| 285 |
+
if _RATE_LIMITED in str(e).lower():
|
| 286 |
+
log(f" !! GIVING UP on /{path} after {waited} waits: still rate-limited. "
|
| 287 |
+
f"{len(out)} row(s) collected so far are kept. Cause: the ads-management "
|
| 288 |
+
f"limit is per ad account and persists for minutes. Fix: re-run "
|
| 289 |
+
f"`--sync --only <table>` later; the upsert is idempotent.")
|
| 290 |
+
return out, True
|
| 291 |
+
raise
|
| 292 |
+
out.extend(body.get("data") or [])
|
| 293 |
+
pages += 1
|
| 294 |
+
after = ((body.get("paging") or {}).get("cursors") or {}).get("after")
|
| 295 |
+
has_next = bool((body.get("paging") or {}).get("next")) and bool(after)
|
| 296 |
+
if not has_next:
|
| 297 |
+
return out, False
|
| 298 |
+
if pages >= MAX_PAGES:
|
| 299 |
+
log(f" !! STOPPED at MAX_PAGES={MAX_PAGES} on /{path} with more pages left. "
|
| 300 |
+
f"Cause: a runaway guard, not a row cap. Fix: raise META_MAX_PAGES, or narrow the "
|
| 301 |
+
f"window with META_INSIGHTS_DAYS.")
|
| 302 |
+
return out, True
|
| 303 |
+
|
| 304 |
+
|
| 305 |
+
def _cell(value):
|
| 306 |
+
"""One Graph value -> one DuckDB cell. Nested structures become compact JSON TEXT rather than
|
| 307 |
+
being dropped: R2 says every field the API returns, and `targeting` is a field."""
|
| 308 |
+
if value is None or isinstance(value, (str, int, float, bool)):
|
| 309 |
+
return value
|
| 310 |
+
return json.dumps(value, separators=(",", ":"), ensure_ascii=False)
|
| 311 |
+
|
| 312 |
+
|
| 313 |
+
def _coltype(name, rows=None):
|
| 314 |
+
"""The DuckDB type for one column β decided by the DATA when there is data, by the name only
|
| 315 |
+
as a fallback.
|
| 316 |
+
|
| 317 |
+
β THE NAME LIST WAS WRONG AND ONLY THE API COULD SAY SO. `cost_per_unique_click`,
|
| 318 |
+
`cost_per_action_type`, `purchase_roas` and friends READ like money and are **arrays of
|
| 319 |
+
action-type objects** on the Insights edge:
|
| 320 |
+
[{"action_type":"outbound_click","value":"1.459854"}]
|
| 321 |
+
Typed DOUBLE from `_DBL`, the insert died with `Conversion Error: Could not convert string
|
| 322 |
+
'[{...}]' to DOUBLE` β after five entity tables had already been written, so the sync looked
|
| 323 |
+
like it worked and then blew up on the last table.
|
| 324 |
+
β Same principle the field CATALOG is built on, applied one layer down: **ask the response,
|
| 325 |
+
do not assert from a list.** A value that ever arrives as a list or dict is JSON TEXT, because
|
| 326 |
+
that is what `_cell` stores; anything else falls back to the measured name hints.
|
| 327 |
+
"""
|
| 328 |
+
if rows:
|
| 329 |
+
seen, kinds = 0, set()
|
| 330 |
+
for r in rows:
|
| 331 |
+
v = r.get(name)
|
| 332 |
+
if v is None or v == "":
|
| 333 |
+
continue
|
| 334 |
+
kinds.add("json" if isinstance(v, (list, dict)) else
|
| 335 |
+
"bool" if isinstance(v, bool) else
|
| 336 |
+
"int" if isinstance(v, int) else
|
| 337 |
+
"float" if isinstance(v, float) else "str")
|
| 338 |
+
seen += 1
|
| 339 |
+
if seen >= 200:
|
| 340 |
+
break
|
| 341 |
+
if kinds:
|
| 342 |
+
if "json" in kinds or "str" in kinds:
|
| 343 |
+
return "VARCHAR" # a JSON blob, or a numeric STRING
|
| 344 |
+
if kinds <= {"int", "bool"}:
|
| 345 |
+
return "BIGINT" if name in _INT else ("VARCHAR" if "bool" in kinds else "BIGINT")
|
| 346 |
+
return "DOUBLE"
|
| 347 |
+
if name in _INT:
|
| 348 |
+
return "BIGINT"
|
| 349 |
+
if name in _DBL:
|
| 350 |
+
return "DOUBLE"
|
| 351 |
+
return "VARCHAR" # ids included β see the header
|
| 352 |
+
|
| 353 |
+
|
| 354 |
+
def _ensure(con, table, fields, rows=None):
|
| 355 |
+
"""Create or widen the table, with every column typed from `rows` when they are available.
|
| 356 |
+
|
| 357 |
+
β AN EXISTING COLUMN WHOSE TYPE IS NOW WRONG IS REBUILT, NOT PATCHED. DuckDB cannot retype a
|
| 358 |
+
column in place, and this table is DERIVED data that can be re-pulled in minutes β so a type
|
| 359 |
+
disagreement drops and recreates rather than limping on with a column that refuses every
|
| 360 |
+
insert. The alternative is a mirror that is permanently unwritable for one bad guess.
|
| 361 |
+
"""
|
| 362 |
+
want = {f: _coltype(f, rows) for f in fields}
|
| 363 |
+
have = {r[1]: str(r[2]).upper() for r in con.execute(f"PRAGMA table_info('{table}')").fetchall()}
|
| 364 |
+
if have:
|
| 365 |
+
clash = [f for f, ty in want.items() if f in have and have[f] != ty
|
| 366 |
+
and not (have[f].startswith("VARCHAR") and ty == "VARCHAR")]
|
| 367 |
+
if clash:
|
| 368 |
+
con.execute(f"DROP TABLE {table}")
|
| 369 |
+
have = {}
|
| 370 |
+
if not have:
|
| 371 |
+
cols = ", ".join(f"{f} {want[f]}" for f in fields)
|
| 372 |
+
con.execute(f"CREATE TABLE IF NOT EXISTS {table} (id VARCHAR PRIMARY KEY, {cols})"
|
| 373 |
+
if "id" not in fields else
|
| 374 |
+
f"CREATE TABLE IF NOT EXISTS {table} ({cols})")
|
| 375 |
+
return
|
| 376 |
+
for f in fields:
|
| 377 |
+
if f not in have:
|
| 378 |
+
con.execute(f"ALTER TABLE {table} ADD COLUMN {f} {want[f]}")
|
| 379 |
+
|
| 380 |
+
|
| 381 |
+
def _upsert(con, table, fields, rows):
|
| 382 |
+
"""Delete-then-insert by id β the same idempotence `datastore._upsert` gives the Odoo half, so
|
| 383 |
+
a re-sync updates in place and can never append a second copy of the same object."""
|
| 384 |
+
if not rows:
|
| 385 |
+
return 0
|
| 386 |
+
cols = list(fields)
|
| 387 |
+
ids = [str(r.get("id") or "") for r in rows]
|
| 388 |
+
q = ",".join("?" for _ in ids)
|
| 389 |
+
con.execute(f"DELETE FROM {table} WHERE id IN ({q})", ids)
|
| 390 |
+
con.executemany(
|
| 391 |
+
f"INSERT INTO {table} ({', '.join(cols)}) VALUES ({', '.join('?' for _ in cols)})",
|
| 392 |
+
[[_cell(r.get(c)) for c in cols] for r in rows])
|
| 393 |
+
return len(rows)
|
| 394 |
+
|
| 395 |
+
|
| 396 |
+
def sync(tenant_key="royal-imports", tok=None, log=print, insights=True, insights_days=None):
|
| 397 |
+
"""Pull every level into the tenant's mirror. -> a report dict; raises only on a bad token.
|
| 398 |
+
|
| 399 |
+
Idempotent: re-running updates rows in place. Safe to call at boot and on the resync loop,
|
| 400 |
+
exactly as `odoo_relational.refresh` is.
|
| 401 |
+
"""
|
| 402 |
+
tok = tok or token()
|
| 403 |
+
# β THE WINDOW IS A PARAMETER, NOT AN ENVIRONMENT READ AT CALL TIME β and the difference cost a
|
| 404 |
+
# live deploy. `INSIGHTS_DAYS` binds at IMPORT (module scope), so `main._pull_meta`'s
|
| 405 |
+
# `os.environ.setdefault("META_INSIGHTS_DAYS", "7")` executed AFTER this module was already
|
| 406 |
+
# imported and changed nothing: every boot pulled **90 days**, not 7, which is the slow path
|
| 407 |
+
# that trips the per-ad-account rate limit and never finishes. The comment beside that call
|
| 408 |
+
# claimed a short window the code could not deliver.
|
| 409 |
+
# β The shape: **a knob read at import cannot be turned by a caller at runtime.** Passing it
|
| 410 |
+
# makes the caller's intent effective instead of aspirational; the env var stays the DEFAULT.
|
| 411 |
+
days = int(insights_days or INSIGHTS_DAYS)
|
| 412 |
+
report = {"tenant": tenant_key, "tables": {}, "problems": [], "accounts": []}
|
| 413 |
+
if not tok:
|
| 414 |
+
report["problems"].append(
|
| 415 |
+
"META_ADS_ACCESS_TOKEN is not set in the environment or in platform/.env, so nothing "
|
| 416 |
+
"was pulled. This is a missing CREDENTIAL, not a missing capability.")
|
| 417 |
+
return report
|
| 418 |
+
|
| 419 |
+
path = datastore.path_for(tenant_key)
|
| 420 |
+
if Path(datastore.DB_PATH) != Path(path):
|
| 421 |
+
datastore.use_path(path)
|
| 422 |
+
con = datastore.connect()
|
| 423 |
+
log(f" mirror: {Path(path).name}")
|
| 424 |
+
|
| 425 |
+
accts, _ = _walk("me/adaccounts", tok, log, fields="id,name", limit=PAGE)
|
| 426 |
+
report["accounts"] = [a.get("id") for a in accts]
|
| 427 |
+
if not accts:
|
| 428 |
+
report["problems"].append("the token reaches no ad accounts")
|
| 429 |
+
return report
|
| 430 |
+
|
| 431 |
+
# ββ the five entity levels ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 432 |
+
for table, (edge, fields, _parent) in SPECS.items():
|
| 433 |
+
rows, capped = [], False
|
| 434 |
+
for acct in accts:
|
| 435 |
+
aid = acct["id"]
|
| 436 |
+
if edge is None:
|
| 437 |
+
rows.append(_get(aid, tok, fields=",".join(fields)))
|
| 438 |
+
else:
|
| 439 |
+
got, hit = _walk(f"{aid}/{edge}", tok, log, fields=",".join(fields), limit=PAGE)
|
| 440 |
+
rows.extend(got)
|
| 441 |
+
capped = capped or hit
|
| 442 |
+
_ensure(con, table, fields, rows)
|
| 443 |
+
n = _upsert(con, table, fields, rows)
|
| 444 |
+
total = con.execute(f"SELECT count(*) FROM {table}").fetchone()[0]
|
| 445 |
+
report["tables"][table] = {"pulled": n, "in_mirror": total, "capped": capped}
|
| 446 |
+
log(f" {table:<20} pulled {n:>6} mirror total {total:>6}"
|
| 447 |
+
+ (" !! PAGE CAP HIT" if capped else ""))
|
| 448 |
+
if capped:
|
| 449 |
+
report["problems"].append(f"{table}: stopped at MAX_PAGES={MAX_PAGES}")
|
| 450 |
+
|
| 451 |
+
# ββ the daily Insights grain ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 452 |
+
if insights:
|
| 453 |
+
rows, capped = [], False
|
| 454 |
+
for acct in accts:
|
| 455 |
+
for since, until in _slices(days, INSIGHTS_SLICE):
|
| 456 |
+
got, hit = _walk(f"{acct['id']}/insights", tok, log,
|
| 457 |
+
fields=",".join(INSIGHT_FIELDS), level=INSIGHTS_LEVEL,
|
| 458 |
+
time_increment="1", limit=INSIGHTS_PAGE,
|
| 459 |
+
time_range=json.dumps({"since": since, "until": until}))
|
| 460 |
+
rows.extend(got)
|
| 461 |
+
capped = capped or hit
|
| 462 |
+
# β Insights rows carry no id. The key must be STABLE across runs or every re-sync
|
| 463 |
+
# appends a second copy of the same day β so it is composed from the grain itself.
|
| 464 |
+
for r in rows:
|
| 465 |
+
r["id"] = ":".join(str(r.get(k) or "") for k in
|
| 466 |
+
(f"{INSIGHTS_LEVEL}_id", "date_start", "date_stop"))
|
| 467 |
+
fields = ["id"] + INSIGHT_FIELDS
|
| 468 |
+
_ensure(con, INSIGHTS_TABLE, fields, rows)
|
| 469 |
+
n = _upsert(con, INSIGHTS_TABLE, fields, rows)
|
| 470 |
+
total = con.execute(f"SELECT count(*) FROM {INSIGHTS_TABLE}").fetchone()[0]
|
| 471 |
+
report["tables"][INSIGHTS_TABLE] = {"pulled": n, "in_mirror": total, "capped": capped}
|
| 472 |
+
log(f" {INSIGHTS_TABLE:<20} pulled {n:>6} mirror total {total:>6}"
|
| 473 |
+
+ (" !! PAGE CAP HIT" if capped else ""))
|
| 474 |
+
|
| 475 |
+
con.execute("INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)",
|
| 476 |
+
["meta", "done", 0, f"{days}d/{INSIGHTS_SLICE}d@{INSIGHTS_LEVEL}",
|
| 477 |
+
sum(t["in_mirror"] for t in report["tables"].values()),
|
| 478 |
+
time.strftime("%Y-%m-%d %H:%M:%S")])
|
| 479 |
+
return report
|
| 480 |
+
|
| 481 |
+
|
| 482 |
+
def status(tenant_key="royal-imports"):
|
| 483 |
+
"""What is in the mirror right now, per table. Never fetches."""
|
| 484 |
+
path = datastore.path_for(tenant_key)
|
| 485 |
+
if Path(datastore.DB_PATH) != Path(path):
|
| 486 |
+
datastore.use_path(path)
|
| 487 |
+
out = {}
|
| 488 |
+
con = datastore.connect()
|
| 489 |
+
for table in list(SPECS) + [INSIGHTS_TABLE]:
|
| 490 |
+
try:
|
| 491 |
+
out[table] = con.execute(f"SELECT count(*) FROM {table}").fetchone()[0]
|
| 492 |
+
except Exception:
|
| 493 |
+
out[table] = None # table absent = never synced
|
| 494 |
+
return out
|
| 495 |
+
|
| 496 |
+
|
| 497 |
+
def main(argv=None):
|
| 498 |
+
ap = argparse.ArgumentParser(description="Pull Meta Ads into the tenant's DuckDB mirror.")
|
| 499 |
+
ap.add_argument("--sync", action="store_true")
|
| 500 |
+
ap.add_argument("--status", action="store_true")
|
| 501 |
+
ap.add_argument("--tenant", default="royal-imports")
|
| 502 |
+
ap.add_argument("--no-insights", action="store_true")
|
| 503 |
+
ap.add_argument("--insights-days", type=int, default=None,
|
| 504 |
+
help="override the Insights window for THIS run (default INSIGHTS_DAYS); the env var is the default, this is the caller's say")
|
| 505 |
+
a = ap.parse_args(argv)
|
| 506 |
+
if a.status:
|
| 507 |
+
for k, v in status(a.tenant).items():
|
| 508 |
+
print(f" {k:<20} {'(never synced)' if v is None else v}")
|
| 509 |
+
return 0
|
| 510 |
+
if not a.sync:
|
| 511 |
+
ap.print_help()
|
| 512 |
+
return 2
|
| 513 |
+
rep = sync(a.tenant, insights=not a.no_insights, insights_days=a.insights_days)
|
| 514 |
+
for p in rep["problems"]:
|
| 515 |
+
print(" PROBLEM:", p)
|
| 516 |
+
print(f" accounts: {len(rep['accounts'])} tables: {len(rep['tables'])}")
|
| 517 |
+
return 1 if rep["problems"] else 0
|
| 518 |
+
|
| 519 |
+
|
| 520 |
+
if __name__ == "__main__":
|
| 521 |
+
sys.exit(main())
|
platform/model/topics/odoo_accounts.yml
CHANGED
|
@@ -1,60 +1,60 @@
|
|
| 1 |
-
# ββ W33-T49 (owner item 13) β an ENTITY topic: the semantic layer can finally see
|
| 2 |
-
# the databases a person actually opens, not only the line and document grains.
|
| 3 |
-
#
|
| 4 |
-
# β THE `fields:` BLOCK IS GENERATED FROM THE GRID'S OWN FIELD CONTRACT
|
| 5 |
-
# and is held to it by `verify_query.py::section_entity_topics`. Do not hand-edit a
|
| 6 |
-
# field row: add the column to the grid contract and re-emit, or the agent is being
|
| 7 |
-
# trained on a schema the product does not have.
|
| 8 |
-
key: odoo_accounts
|
| 9 |
-
label: "Odoo GL accounts"
|
| 10 |
-
entity: account.account
|
| 11 |
-
# The database this topic DESCRIBES β the same store key the nav opens (W33-T46).
|
| 12 |
-
grid: ut_odoo_accounts
|
| 13 |
-
subject: "odoo:account.account"
|
| 14 |
-
grain: "one row per GL account in the chart of accounts"
|
| 15 |
-
scope:
|
| 16 |
-
population: "every account.account record (192 measured)"
|
| 17 |
-
store:
|
| 18 |
-
table: account_account
|
| 19 |
-
alias: a
|
| 20 |
-
# NO date_col β a registry is not a dated event stream. Stated rather than
|
| 21 |
-
# omitted, so its absence reads as a fact and not as an unfinished file.
|
| 22 |
-
dims:
|
| 23 |
-
account_type: {label: "Account type"}
|
| 24 |
-
|
| 25 |
-
# key / label / type / kind, derived from the grid contract. `kind` says where the
|
| 26 |
-
# value COMES FROM: a `data` column is stored on the row; a `rollup` is computed
|
| 27 |
-
# from another topic; a `link` points at another database.
|
| 28 |
-
fields:
|
| 29 |
-
- key: account_code
|
| 30 |
-
label: "Code"
|
| 31 |
-
type: text
|
| 32 |
-
kind: data
|
| 33 |
-
- key: account_name
|
| 34 |
-
label: "Account"
|
| 35 |
-
type: text
|
| 36 |
-
kind: data
|
| 37 |
-
- key: odoo_id
|
| 38 |
-
label: "Odoo ID"
|
| 39 |
-
type: int
|
| 40 |
-
kind: data
|
| 41 |
-
means: "The `account.account` id. Also this row's id."
|
| 42 |
-
- key: account_type
|
| 43 |
-
label: "Type"
|
| 44 |
-
type: select
|
| 45 |
-
kind: data
|
| 46 |
-
- key: is_expense
|
| 47 |
-
label: "Expense account"
|
| 48 |
-
type: checkbox
|
| 49 |
-
kind: data
|
| 50 |
-
means: "Ticked for the expense family - the same predicate the semantic layer's gl_lines topic uses, so this column and that topic cannot disagree."
|
| 51 |
-
- key: refreshed
|
| 52 |
-
label: "Refreshed"
|
| 53 |
-
type: date
|
| 54 |
-
kind: data
|
| 55 |
-
means: "When this row was last reconciled against Odoo."
|
| 56 |
-
|
| 57 |
-
ai_context: >
|
| 58 |
-
The chart of accounts β the registry gl_lines posts against.
|
| 59 |
-
Use it to resolve an account code to a name or a type.
|
| 60 |
-
β `is_expense` is a DERIVED boolean standing in for Odoo's five-value `internal_group` (asset/liability/equity/income/expense), which the mirror does not yet carry.
|
|
|
|
| 1 |
+
# ββ W33-T49 (owner item 13) β an ENTITY topic: the semantic layer can finally see
|
| 2 |
+
# the databases a person actually opens, not only the line and document grains.
|
| 3 |
+
#
|
| 4 |
+
# β THE `fields:` BLOCK IS GENERATED FROM THE GRID'S OWN FIELD CONTRACT
|
| 5 |
+
# and is held to it by `verify_query.py::section_entity_topics`. Do not hand-edit a
|
| 6 |
+
# field row: add the column to the grid contract and re-emit, or the agent is being
|
| 7 |
+
# trained on a schema the product does not have.
|
| 8 |
+
key: odoo_accounts
|
| 9 |
+
label: "Odoo GL accounts"
|
| 10 |
+
entity: account.account
|
| 11 |
+
# The database this topic DESCRIBES β the same store key the nav opens (W33-T46).
|
| 12 |
+
grid: ut_odoo_accounts
|
| 13 |
+
subject: "odoo:account.account"
|
| 14 |
+
grain: "one row per GL account in the chart of accounts"
|
| 15 |
+
scope:
|
| 16 |
+
population: "every account.account record (192 measured)"
|
| 17 |
+
store:
|
| 18 |
+
table: account_account
|
| 19 |
+
alias: a
|
| 20 |
+
# NO date_col β a registry is not a dated event stream. Stated rather than
|
| 21 |
+
# omitted, so its absence reads as a fact and not as an unfinished file.
|
| 22 |
+
dims:
|
| 23 |
+
account_type: {label: "Account type"}
|
| 24 |
+
|
| 25 |
+
# key / label / type / kind, derived from the grid contract. `kind` says where the
|
| 26 |
+
# value COMES FROM: a `data` column is stored on the row; a `rollup` is computed
|
| 27 |
+
# from another topic; a `link` points at another database.
|
| 28 |
+
fields:
|
| 29 |
+
- key: account_code
|
| 30 |
+
label: "Code"
|
| 31 |
+
type: text
|
| 32 |
+
kind: data
|
| 33 |
+
- key: account_name
|
| 34 |
+
label: "Account"
|
| 35 |
+
type: text
|
| 36 |
+
kind: data
|
| 37 |
+
- key: odoo_id
|
| 38 |
+
label: "Odoo ID"
|
| 39 |
+
type: int
|
| 40 |
+
kind: data
|
| 41 |
+
means: "The `account.account` id. Also this row's id."
|
| 42 |
+
- key: account_type
|
| 43 |
+
label: "Type"
|
| 44 |
+
type: select
|
| 45 |
+
kind: data
|
| 46 |
+
- key: is_expense
|
| 47 |
+
label: "Expense account"
|
| 48 |
+
type: checkbox
|
| 49 |
+
kind: data
|
| 50 |
+
means: "Ticked for the expense family - the same predicate the semantic layer's gl_lines topic uses, so this column and that topic cannot disagree."
|
| 51 |
+
- key: refreshed
|
| 52 |
+
label: "Refreshed"
|
| 53 |
+
type: date
|
| 54 |
+
kind: data
|
| 55 |
+
means: "When this row was last reconciled against Odoo."
|
| 56 |
+
|
| 57 |
+
ai_context: >
|
| 58 |
+
The chart of accounts β the registry gl_lines posts against.
|
| 59 |
+
Use it to resolve an account code to a name or a type.
|
| 60 |
+
β `is_expense` is a DERIVED boolean standing in for Odoo's five-value `internal_group` (asset/liability/equity/income/expense), which the mirror does not yet carry.
|
platform/model/topics/odoo_agents.yml
CHANGED
|
@@ -1,59 +1,59 @@
|
|
| 1 |
-
# ββ W33-T49 (owner item 13) β an ENTITY topic: the semantic layer can finally see
|
| 2 |
-
# the databases a person actually opens, not only the line and document grains.
|
| 3 |
-
#
|
| 4 |
-
# β THE `fields:` BLOCK IS GENERATED FROM THE GRID'S OWN FIELD CONTRACT
|
| 5 |
-
# and is held to it by `verify_query.py::section_entity_topics`. Do not hand-edit a
|
| 6 |
-
# field row: add the column to the grid contract and re-emit, or the agent is being
|
| 7 |
-
# trained on a schema the product does not have.
|
| 8 |
-
key: odoo_agents
|
| 9 |
-
label: "Odoo agents"
|
| 10 |
-
entity: res.partner
|
| 11 |
-
# The database this topic DESCRIBES β the same store key the nav opens (W33-T46).
|
| 12 |
-
grid: ut_odoo_agents
|
| 13 |
-
subject: "odoo:res.partner.agent"
|
| 14 |
-
grain: "one row per sales agent"
|
| 15 |
-
scope:
|
| 16 |
-
population: "a UNION of two disagreeing sources β partners carrying commission lines, and partners flagged res_partner.agent = TRUE"
|
| 17 |
-
why_union: "measured 2026-08-09: 16 carry commission lines, 17 carry the flag, and the union is 19 β either source alone silently drops real agents"
|
| 18 |
-
store:
|
| 19 |
-
table: res_partner
|
| 20 |
-
alias: p
|
| 21 |
-
# NO date_col β a registry is not a dated event stream. Stated rather than
|
| 22 |
-
# omitted, so its absence reads as a fact and not as an unfinished file.
|
| 23 |
-
|
| 24 |
-
# key / label / type / kind, derived from the grid contract. `kind` says where the
|
| 25 |
-
# value COMES FROM: a `data` column is stored on the row; a `rollup` is computed
|
| 26 |
-
# from another topic; a `link` points at another database.
|
| 27 |
-
fields:
|
| 28 |
-
- key: agent
|
| 29 |
-
label: "Agent"
|
| 30 |
-
type: text
|
| 31 |
-
kind: data
|
| 32 |
-
- key: odoo_id
|
| 33 |
-
label: "Odoo ID"
|
| 34 |
-
type: int
|
| 35 |
-
kind: data
|
| 36 |
-
means: "The `res.partner` id. Also this row's id."
|
| 37 |
-
- key: agent_id
|
| 38 |
-
label: "Odoo agent id"
|
| 39 |
-
type: int
|
| 40 |
-
kind: data
|
| 41 |
-
- key: flagged
|
| 42 |
-
label: "Flagged in Odoo"
|
| 43 |
-
type: checkbox
|
| 44 |
-
kind: data
|
| 45 |
-
means: "Ticked = `res.partner.agent` is set. Unticked agents were found by their commission lines instead - both are real, which is why this table is the union of the two."
|
| 46 |
-
- key: commissioned
|
| 47 |
-
label: "Has commission lines"
|
| 48 |
-
type: checkbox
|
| 49 |
-
kind: data
|
| 50 |
-
- key: refreshed
|
| 51 |
-
label: "Refreshed"
|
| 52 |
-
type: date
|
| 53 |
-
kind: data
|
| 54 |
-
means: "When this row was last reconciled against Odoo."
|
| 55 |
-
|
| 56 |
-
ai_context: >
|
| 57 |
-
The sales-agent registry.
|
| 58 |
-
`flagged` and `commissioned` are the two SOURCES, kept as separate columns rather than merged, because they disagree and the disagreement is information.
|
| 59 |
-
For an agent's BOOK use sales_lines/commission_lines with the agent dim.
|
|
|
|
| 1 |
+
# ββ W33-T49 (owner item 13) β an ENTITY topic: the semantic layer can finally see
|
| 2 |
+
# the databases a person actually opens, not only the line and document grains.
|
| 3 |
+
#
|
| 4 |
+
# β THE `fields:` BLOCK IS GENERATED FROM THE GRID'S OWN FIELD CONTRACT
|
| 5 |
+
# and is held to it by `verify_query.py::section_entity_topics`. Do not hand-edit a
|
| 6 |
+
# field row: add the column to the grid contract and re-emit, or the agent is being
|
| 7 |
+
# trained on a schema the product does not have.
|
| 8 |
+
key: odoo_agents
|
| 9 |
+
label: "Odoo agents"
|
| 10 |
+
entity: res.partner
|
| 11 |
+
# The database this topic DESCRIBES β the same store key the nav opens (W33-T46).
|
| 12 |
+
grid: ut_odoo_agents
|
| 13 |
+
subject: "odoo:res.partner.agent"
|
| 14 |
+
grain: "one row per sales agent"
|
| 15 |
+
scope:
|
| 16 |
+
population: "a UNION of two disagreeing sources β partners carrying commission lines, and partners flagged res_partner.agent = TRUE"
|
| 17 |
+
why_union: "measured 2026-08-09: 16 carry commission lines, 17 carry the flag, and the union is 19 β either source alone silently drops real agents"
|
| 18 |
+
store:
|
| 19 |
+
table: res_partner
|
| 20 |
+
alias: p
|
| 21 |
+
# NO date_col β a registry is not a dated event stream. Stated rather than
|
| 22 |
+
# omitted, so its absence reads as a fact and not as an unfinished file.
|
| 23 |
+
|
| 24 |
+
# key / label / type / kind, derived from the grid contract. `kind` says where the
|
| 25 |
+
# value COMES FROM: a `data` column is stored on the row; a `rollup` is computed
|
| 26 |
+
# from another topic; a `link` points at another database.
|
| 27 |
+
fields:
|
| 28 |
+
- key: agent
|
| 29 |
+
label: "Agent"
|
| 30 |
+
type: text
|
| 31 |
+
kind: data
|
| 32 |
+
- key: odoo_id
|
| 33 |
+
label: "Odoo ID"
|
| 34 |
+
type: int
|
| 35 |
+
kind: data
|
| 36 |
+
means: "The `res.partner` id. Also this row's id."
|
| 37 |
+
- key: agent_id
|
| 38 |
+
label: "Odoo agent id"
|
| 39 |
+
type: int
|
| 40 |
+
kind: data
|
| 41 |
+
- key: flagged
|
| 42 |
+
label: "Flagged in Odoo"
|
| 43 |
+
type: checkbox
|
| 44 |
+
kind: data
|
| 45 |
+
means: "Ticked = `res.partner.agent` is set. Unticked agents were found by their commission lines instead - both are real, which is why this table is the union of the two."
|
| 46 |
+
- key: commissioned
|
| 47 |
+
label: "Has commission lines"
|
| 48 |
+
type: checkbox
|
| 49 |
+
kind: data
|
| 50 |
+
- key: refreshed
|
| 51 |
+
label: "Refreshed"
|
| 52 |
+
type: date
|
| 53 |
+
kind: data
|
| 54 |
+
means: "When this row was last reconciled against Odoo."
|
| 55 |
+
|
| 56 |
+
ai_context: >
|
| 57 |
+
The sales-agent registry.
|
| 58 |
+
`flagged` and `commissioned` are the two SOURCES, kept as separate columns rather than merged, because they disagree and the disagreement is information.
|
| 59 |
+
For an agent's BOOK use sales_lines/commission_lines with the agent dim.
|
platform/model/topics/odoo_bills.yml
CHANGED
|
@@ -1,86 +1,86 @@
|
|
| 1 |
-
# ββ W33-T49 (owner item 13) β an ENTITY topic: the semantic layer can finally see
|
| 2 |
-
# the databases a person actually opens, not only the line and document grains.
|
| 3 |
-
#
|
| 4 |
-
# β THE `fields:` BLOCK IS GENERATED FROM THE GRID'S OWN FIELD CONTRACT
|
| 5 |
-
# and is held to it by `verify_query.py::section_entity_topics`. Do not hand-edit a
|
| 6 |
-
# field row: add the column to the grid contract and re-emit, or the agent is being
|
| 7 |
-
# trained on a schema the product does not have.
|
| 8 |
-
key: odoo_bills
|
| 9 |
-
label: "Odoo vendor bills"
|
| 10 |
-
entity: account.move
|
| 11 |
-
# The database this topic DESCRIBES β the same store key the nav opens (W33-T46).
|
| 12 |
-
grid: ut_odoo_bills
|
| 13 |
-
subject: "odoo:account.move.vendor"
|
| 14 |
-
grain: "one row per posted VENDOR bill or vendor credit note (document grain)"
|
| 15 |
-
scope:
|
| 16 |
-
population: "move_type in (in_invoice, in_refund) AND state = posted"
|
| 17 |
-
signs: "for payables the residual is negative on the Odoo side; take the absolute value for an AP figure"
|
| 18 |
-
store:
|
| 19 |
-
table: account_move
|
| 20 |
-
alias: m
|
| 21 |
-
date_col: "m.invoice_date"
|
| 22 |
-
dims:
|
| 23 |
-
vendor: {label: "Vendor"}
|
| 24 |
-
payment_state: {label: "Payment state"}
|
| 25 |
-
move_type: {label: "Document type"}
|
| 26 |
-
|
| 27 |
-
# key / label / type / kind, derived from the grid contract. `kind` says where the
|
| 28 |
-
# value COMES FROM: a `data` column is stored on the row; a `rollup` is computed
|
| 29 |
-
# from another topic; a `link` points at another database.
|
| 30 |
-
fields:
|
| 31 |
-
- key: bill_no
|
| 32 |
-
label: "Bill"
|
| 33 |
-
type: text
|
| 34 |
-
kind: data
|
| 35 |
-
- key: odoo_id
|
| 36 |
-
label: "Odoo ID"
|
| 37 |
-
type: int
|
| 38 |
-
kind: data
|
| 39 |
-
means: "The `account.move` id. Also this row's id."
|
| 40 |
-
- key: vendor
|
| 41 |
-
label: "Vendor"
|
| 42 |
-
type: text
|
| 43 |
-
kind: data
|
| 44 |
-
- key: vendor_id
|
| 45 |
-
label: "Odoo vendor id"
|
| 46 |
-
type: int
|
| 47 |
-
kind: data
|
| 48 |
-
- key: invoice_date
|
| 49 |
-
label: "Bill date"
|
| 50 |
-
type: date
|
| 51 |
-
kind: data
|
| 52 |
-
- key: due_date
|
| 53 |
-
label: "Due date"
|
| 54 |
-
type: date
|
| 55 |
-
kind: data
|
| 56 |
-
- key: amount_untaxed
|
| 57 |
-
label: "Billed $"
|
| 58 |
-
type: currency
|
| 59 |
-
kind: data
|
| 60 |
-
- key: residual
|
| 61 |
-
label: "Outstanding $"
|
| 62 |
-
type: currency
|
| 63 |
-
kind: data
|
| 64 |
-
- key: payment_state
|
| 65 |
-
label: "Payment state"
|
| 66 |
-
type: select
|
| 67 |
-
kind: data
|
| 68 |
-
- key: move_type
|
| 69 |
-
label: "Document"
|
| 70 |
-
type: select
|
| 71 |
-
kind: data
|
| 72 |
-
- key: vendor_link
|
| 73 |
-
label: "Vendor record"
|
| 74 |
-
type: link
|
| 75 |
-
kind: link
|
| 76 |
-
to_grid: ut_odoo_vendors
|
| 77 |
-
- key: refreshed
|
| 78 |
-
label: "Refreshed"
|
| 79 |
-
type: date
|
| 80 |
-
kind: data
|
| 81 |
-
means: "When this row was last reconciled against Odoo."
|
| 82 |
-
|
| 83 |
-
ai_context: >
|
| 84 |
-
Vendor bills at DOCUMENT grain β the payables side of account_move.
|
| 85 |
-
Use it for AP questions.
|
| 86 |
-
β The census found `ref` (the VENDOR's own invoice number), `journal_id` and `currency_id` populated in Odoo and absent from the mirror, so they cannot be answered from here yet.
|
|
|
|
| 1 |
+
# ββ W33-T49 (owner item 13) β an ENTITY topic: the semantic layer can finally see
|
| 2 |
+
# the databases a person actually opens, not only the line and document grains.
|
| 3 |
+
#
|
| 4 |
+
# β THE `fields:` BLOCK IS GENERATED FROM THE GRID'S OWN FIELD CONTRACT
|
| 5 |
+
# and is held to it by `verify_query.py::section_entity_topics`. Do not hand-edit a
|
| 6 |
+
# field row: add the column to the grid contract and re-emit, or the agent is being
|
| 7 |
+
# trained on a schema the product does not have.
|
| 8 |
+
key: odoo_bills
|
| 9 |
+
label: "Odoo vendor bills"
|
| 10 |
+
entity: account.move
|
| 11 |
+
# The database this topic DESCRIBES β the same store key the nav opens (W33-T46).
|
| 12 |
+
grid: ut_odoo_bills
|
| 13 |
+
subject: "odoo:account.move.vendor"
|
| 14 |
+
grain: "one row per posted VENDOR bill or vendor credit note (document grain)"
|
| 15 |
+
scope:
|
| 16 |
+
population: "move_type in (in_invoice, in_refund) AND state = posted"
|
| 17 |
+
signs: "for payables the residual is negative on the Odoo side; take the absolute value for an AP figure"
|
| 18 |
+
store:
|
| 19 |
+
table: account_move
|
| 20 |
+
alias: m
|
| 21 |
+
date_col: "m.invoice_date"
|
| 22 |
+
dims:
|
| 23 |
+
vendor: {label: "Vendor"}
|
| 24 |
+
payment_state: {label: "Payment state"}
|
| 25 |
+
move_type: {label: "Document type"}
|
| 26 |
+
|
| 27 |
+
# key / label / type / kind, derived from the grid contract. `kind` says where the
|
| 28 |
+
# value COMES FROM: a `data` column is stored on the row; a `rollup` is computed
|
| 29 |
+
# from another topic; a `link` points at another database.
|
| 30 |
+
fields:
|
| 31 |
+
- key: bill_no
|
| 32 |
+
label: "Bill"
|
| 33 |
+
type: text
|
| 34 |
+
kind: data
|
| 35 |
+
- key: odoo_id
|
| 36 |
+
label: "Odoo ID"
|
| 37 |
+
type: int
|
| 38 |
+
kind: data
|
| 39 |
+
means: "The `account.move` id. Also this row's id."
|
| 40 |
+
- key: vendor
|
| 41 |
+
label: "Vendor"
|
| 42 |
+
type: text
|
| 43 |
+
kind: data
|
| 44 |
+
- key: vendor_id
|
| 45 |
+
label: "Odoo vendor id"
|
| 46 |
+
type: int
|
| 47 |
+
kind: data
|
| 48 |
+
- key: invoice_date
|
| 49 |
+
label: "Bill date"
|
| 50 |
+
type: date
|
| 51 |
+
kind: data
|
| 52 |
+
- key: due_date
|
| 53 |
+
label: "Due date"
|
| 54 |
+
type: date
|
| 55 |
+
kind: data
|
| 56 |
+
- key: amount_untaxed
|
| 57 |
+
label: "Billed $"
|
| 58 |
+
type: currency
|
| 59 |
+
kind: data
|
| 60 |
+
- key: residual
|
| 61 |
+
label: "Outstanding $"
|
| 62 |
+
type: currency
|
| 63 |
+
kind: data
|
| 64 |
+
- key: payment_state
|
| 65 |
+
label: "Payment state"
|
| 66 |
+
type: select
|
| 67 |
+
kind: data
|
| 68 |
+
- key: move_type
|
| 69 |
+
label: "Document"
|
| 70 |
+
type: select
|
| 71 |
+
kind: data
|
| 72 |
+
- key: vendor_link
|
| 73 |
+
label: "Vendor record"
|
| 74 |
+
type: link
|
| 75 |
+
kind: link
|
| 76 |
+
to_grid: ut_odoo_vendors
|
| 77 |
+
- key: refreshed
|
| 78 |
+
label: "Refreshed"
|
| 79 |
+
type: date
|
| 80 |
+
kind: data
|
| 81 |
+
means: "When this row was last reconciled against Odoo."
|
| 82 |
+
|
| 83 |
+
ai_context: >
|
| 84 |
+
Vendor bills at DOCUMENT grain β the payables side of account_move.
|
| 85 |
+
Use it for AP questions.
|
| 86 |
+
β The census found `ref` (the VENDOR's own invoice number), `journal_id` and `currency_id` populated in Odoo and absent from the mirror, so they cannot be answered from here yet.
|
platform/model/topics/odoo_customers.yml
CHANGED
|
@@ -1,213 +1,213 @@
|
|
| 1 |
-
# ββ W33-T49 (owner item 13) β an ENTITY topic: the semantic layer can finally see
|
| 2 |
-
# the databases a person actually opens, not only the line and document grains.
|
| 3 |
-
#
|
| 4 |
-
# β THE `fields:` BLOCK IS GENERATED FROM THE GRID'S OWN FIELD CONTRACT
|
| 5 |
-
# and is held to it by `verify_query.py::section_entity_topics`. Do not hand-edit a
|
| 6 |
-
# field row: add the column to the grid contract and re-emit, or the agent is being
|
| 7 |
-
# trained on a schema the product does not have.
|
| 8 |
-
key: odoo_customers
|
| 9 |
-
label: "Odoo customers"
|
| 10 |
-
entity: res.partner
|
| 11 |
-
# The database this topic DESCRIBES β the same store key the nav opens (W33-T46).
|
| 12 |
-
grid: customer_data
|
| 13 |
-
subject: "odoo:res.partner"
|
| 14 |
-
grain: "one row per customer in tenant #0's scoped book β a REGISTRY, not a dated event stream. Identity is the Odoo `res.partner` id (the row's `pid` IS that id)."
|
| 15 |
-
scope:
|
| 16 |
-
population: "the customer book `modules/customer_data.pool()` builds; the grid's own contract states it as one row per customer who ordered in the last 24 months"
|
| 17 |
-
identity: "the Odoo res.partner id β exposed as the `partner_id` column (W33-T43) and used as the row pid, so there is ONE id per row and no second one"
|
| 18 |
-
merged: "W33-T44 retired `ut_odoo_customers`, which presented this same subject. This topic describes the SURVIVING database, `customer_data`, which keeps its store bucket so no saved view, grant, cohort or formula moved"
|
| 19 |
-
no_date: "a registry has no date dimension; ask sales_lines or customer_invoices for anything time-windowed about a customer"
|
| 20 |
-
store:
|
| 21 |
-
table: res_partner
|
| 22 |
-
alias: p
|
| 23 |
-
# NO date_col β a registry is not a dated event stream. Stated rather than
|
| 24 |
-
# omitted, so its absence reads as a fact and not as an unfinished file.
|
| 25 |
-
dims:
|
| 26 |
-
agent: {label: "Sales agent"}
|
| 27 |
-
state: {label: "State"}
|
| 28 |
-
country: {label: "Country"}
|
| 29 |
-
|
| 30 |
-
# key / label / type / kind, derived from the grid contract. `kind` says where the
|
| 31 |
-
# value COMES FROM: a `data` column is stored on the row; a `rollup` is computed
|
| 32 |
-
# from another topic; a `link` points at another database.
|
| 33 |
-
fields:
|
| 34 |
-
- key: customer
|
| 35 |
-
label: "Customer"
|
| 36 |
-
type: text
|
| 37 |
-
kind: data
|
| 38 |
-
means: "The customer's name in Odoo. One row per customer who ordered in the last 24 months."
|
| 39 |
-
- key: partner_id
|
| 40 |
-
label: "Odoo ID"
|
| 41 |
-
type: int
|
| 42 |
-
kind: data
|
| 43 |
-
means: "The Odoo res.partner id β this row's identity, and the key every Odoo document joins on. W33-T43 / owner item 12: 'One unique ID per database always.' It is DERIVED rather than read off the pool row because a customer row's pid IS the partner id, so the value is already on every row and a second copy in the pool would be a second source for one fact."
|
| 44 |
-
- key: odoo_status
|
| 45 |
-
label: "Odoo record"
|
| 46 |
-
type: status
|
| 47 |
-
kind: data
|
| 48 |
-
means: "Whether this customer still exists in Odoo. Archived means deleted there."
|
| 49 |
-
- key: agent
|
| 50 |
-
label: "Agent"
|
| 51 |
-
type: text
|
| 52 |
-
kind: data
|
| 53 |
-
means: "The sales agent who owns this account."
|
| 54 |
-
- key: dba
|
| 55 |
-
label: "DBA"
|
| 56 |
-
type: select
|
| 57 |
-
kind: data
|
| 58 |
-
means: "The brand this customer buys from - Fisch, Royal, or both. Amazon-channel orders are not a DBA."
|
| 59 |
-
- key: salesperson
|
| 60 |
-
label: "Salesperson"
|
| 61 |
-
type: text
|
| 62 |
-
kind: data
|
| 63 |
-
means: "Who keyed in most of this customer's orders β not the Agent, who owns the account."
|
| 64 |
-
- key: street
|
| 65 |
-
label: "Street"
|
| 66 |
-
type: text
|
| 67 |
-
kind: data
|
| 68 |
-
means: "First address line, from res.partner directly - not the geocoder, so a customer the map cannot place still shows its address."
|
| 69 |
-
- key: street2
|
| 70 |
-
label: "Street 2"
|
| 71 |
-
type: text
|
| 72 |
-
kind: data
|
| 73 |
-
means: "Second address line (suite, unit, floor) on the customer's Odoo address."
|
| 74 |
-
- key: city
|
| 75 |
-
label: "City"
|
| 76 |
-
type: text
|
| 77 |
-
kind: data
|
| 78 |
-
means: "City on the customer's Odoo address."
|
| 79 |
-
- key: state
|
| 80 |
-
label: "State"
|
| 81 |
-
type: text
|
| 82 |
-
kind: data
|
| 83 |
-
means: "State or province on the customer's Odoo address."
|
| 84 |
-
- key: country
|
| 85 |
-
label: "Country"
|
| 86 |
-
type: text
|
| 87 |
-
kind: data
|
| 88 |
-
means: "Country on the customer's Odoo address."
|
| 89 |
-
- key: zip
|
| 90 |
-
label: "ZIP"
|
| 91 |
-
type: text
|
| 92 |
-
kind: data
|
| 93 |
-
means: "Postal code on the customer's Odoo address."
|
| 94 |
-
- key: customer_since
|
| 95 |
-
label: "Customer since"
|
| 96 |
-
type: date
|
| 97 |
-
kind: data
|
| 98 |
-
means: "When this customer was first set up in Odoo."
|
| 99 |
-
- key: tags
|
| 100 |
-
label: "Tags"
|
| 101 |
-
type: text
|
| 102 |
-
kind: data
|
| 103 |
-
means: "Odoo labels on this customer, comma-separated."
|
| 104 |
-
- key: pricelist
|
| 105 |
-
label: "Price list"
|
| 106 |
-
type: text
|
| 107 |
-
kind: data
|
| 108 |
-
means: "The price list this customer buys on."
|
| 109 |
-
- key: payment_terms
|
| 110 |
-
label: "Payment terms"
|
| 111 |
-
type: text
|
| 112 |
-
kind: data
|
| 113 |
-
means: "Payment terms on this customer's account β Net 30, for example."
|
| 114 |
-
- key: last_order
|
| 115 |
-
label: "Last order"
|
| 116 |
-
type: date
|
| 117 |
-
kind: data
|
| 118 |
-
means: "Date of the most recent confirmed order."
|
| 119 |
-
- key: overdue_days
|
| 120 |
-
label: "Overdue days"
|
| 121 |
-
type: int
|
| 122 |
-
kind: data
|
| 123 |
-
means: "How many days late this customer is running against their own usual ordering rhythm."
|
| 124 |
-
- key: est_missed
|
| 125 |
-
label: "Est. missed $"
|
| 126 |
-
type: currency
|
| 127 |
-
kind: data
|
| 128 |
-
means: "Estimated sales missed while quiet: missed orders (capped at 3) times average order value. An estimate, not money owed."
|
| 129 |
-
- key: ar_open
|
| 130 |
-
label: "AR current $"
|
| 131 |
-
type: currency
|
| 132 |
-
kind: data
|
| 133 |
-
means: "Invoiced money owed but not yet due (a 5-day grace applies before it counts as overdue)."
|
| 134 |
-
- key: ar_overdue
|
| 135 |
-
label: "AR overdue $"
|
| 136 |
-
type: currency
|
| 137 |
-
kind: data
|
| 138 |
-
means: "Invoiced money past due β same basis as the Collections page."
|
| 139 |
-
- key: ar_outstanding
|
| 140 |
-
label: "AR outstanding $"
|
| 141 |
-
type: currency
|
| 142 |
-
kind: data
|
| 143 |
-
means: "Total invoiced money owed right now: AR current $ plus AR overdue $."
|
| 144 |
-
- key: ar_exposure
|
| 145 |
-
label: "Credit exposure $"
|
| 146 |
-
type: currency
|
| 147 |
-
kind: data
|
| 148 |
-
means: "The most you could be out if they stopped paying today: open, overdue, draft and not-yet-invoiced."
|
| 149 |
-
- key: ar_aged_1_30
|
| 150 |
-
label: "1-30 days $"
|
| 151 |
-
type: currency
|
| 152 |
-
kind: data
|
| 153 |
-
means: "Overdue between 1 and 30 days. The four aging buckets sum to AR overdue $."
|
| 154 |
-
- key: ar_aged_31_60
|
| 155 |
-
label: "31-60 days $"
|
| 156 |
-
type: currency
|
| 157 |
-
kind: data
|
| 158 |
-
means: "Overdue between 31 and 60 days. The four aging buckets sum to AR overdue $."
|
| 159 |
-
- key: ar_aged_61_90
|
| 160 |
-
label: "61-90 days $"
|
| 161 |
-
type: currency
|
| 162 |
-
kind: data
|
| 163 |
-
means: "Overdue between 61 and 90 days. The four aging buckets sum to AR overdue $."
|
| 164 |
-
- key: ar_aged_90_plus
|
| 165 |
-
label: "90+ days $"
|
| 166 |
-
type: currency
|
| 167 |
-
kind: data
|
| 168 |
-
means: "Overdue by more than 90 days. The four aging buckets sum to AR overdue $."
|
| 169 |
-
- key: days_to_pay
|
| 170 |
-
label: "Days to pay"
|
| 171 |
-
type: int
|
| 172 |
-
kind: data
|
| 173 |
-
means: "Average days to pay an invoice in full. Blank means no fully paid invoice yet."
|
| 174 |
-
- key: top_category
|
| 175 |
-
label: "Top category"
|
| 176 |
-
type: text
|
| 177 |
-
kind: data
|
| 178 |
-
means: "The category this customer spent the most on in the last 12 months."
|
| 179 |
-
- key: top_category_pct
|
| 180 |
-
label: "Top category %"
|
| 181 |
-
type: pct
|
| 182 |
-
kind: data
|
| 183 |
-
means: "Share of last-12-months spend that went to the top category."
|
| 184 |
-
- key: sku_count
|
| 185 |
-
label: "SKUs bought"
|
| 186 |
-
type: int
|
| 187 |
-
kind: data
|
| 188 |
-
means: "Distinct products bought in the last 12 months."
|
| 189 |
-
- key: top_sku
|
| 190 |
-
label: "Top SKU"
|
| 191 |
-
type: text
|
| 192 |
-
kind: data
|
| 193 |
-
means: "The product this customer spent the most on in the last 12 months."
|
| 194 |
-
- key: days_since
|
| 195 |
-
label: "Days since order"
|
| 196 |
-
type: int
|
| 197 |
-
kind: data
|
| 198 |
-
means: "Days since the last confirmed order."
|
| 199 |
-
- key: typical_gap_days
|
| 200 |
-
label: "Typical gap days"
|
| 201 |
-
type: int
|
| 202 |
-
kind: data
|
| 203 |
-
means: "Days this customer usually goes between orders, from their own history."
|
| 204 |
-
- key: notes
|
| 205 |
-
label: "Notes"
|
| 206 |
-
type: text
|
| 207 |
-
kind: data
|
| 208 |
-
means: "Your notes on this customer. Saved in this app only, visible only to you."
|
| 209 |
-
|
| 210 |
-
ai_context: >
|
| 211 |
-
The customer REGISTRY as a person sees it in the app β the database labelled 'Odoo customers'.
|
| 212 |
-
Use it to answer 'who is this customer', 'which customers exist', 'which agent owns them', 'where are they', and for the AR columns it carries (ar_open, ar_overdue, days_to_pay), which are reconciled by modules/ar.py.
|
| 213 |
-
β For anything WINDOWED or at line grain go to sales_lines / customer_invoices / receivables β those topics own the time dimension and this one has none.
|
|
|
|
| 1 |
+
# ββ W33-T49 (owner item 13) β an ENTITY topic: the semantic layer can finally see
|
| 2 |
+
# the databases a person actually opens, not only the line and document grains.
|
| 3 |
+
#
|
| 4 |
+
# β THE `fields:` BLOCK IS GENERATED FROM THE GRID'S OWN FIELD CONTRACT
|
| 5 |
+
# and is held to it by `verify_query.py::section_entity_topics`. Do not hand-edit a
|
| 6 |
+
# field row: add the column to the grid contract and re-emit, or the agent is being
|
| 7 |
+
# trained on a schema the product does not have.
|
| 8 |
+
key: odoo_customers
|
| 9 |
+
label: "Odoo customers"
|
| 10 |
+
entity: res.partner
|
| 11 |
+
# The database this topic DESCRIBES β the same store key the nav opens (W33-T46).
|
| 12 |
+
grid: customer_data
|
| 13 |
+
subject: "odoo:res.partner"
|
| 14 |
+
grain: "one row per customer in tenant #0's scoped book β a REGISTRY, not a dated event stream. Identity is the Odoo `res.partner` id (the row's `pid` IS that id)."
|
| 15 |
+
scope:
|
| 16 |
+
population: "the customer book `modules/customer_data.pool()` builds; the grid's own contract states it as one row per customer who ordered in the last 24 months"
|
| 17 |
+
identity: "the Odoo res.partner id β exposed as the `partner_id` column (W33-T43) and used as the row pid, so there is ONE id per row and no second one"
|
| 18 |
+
merged: "W33-T44 retired `ut_odoo_customers`, which presented this same subject. This topic describes the SURVIVING database, `customer_data`, which keeps its store bucket so no saved view, grant, cohort or formula moved"
|
| 19 |
+
no_date: "a registry has no date dimension; ask sales_lines or customer_invoices for anything time-windowed about a customer"
|
| 20 |
+
store:
|
| 21 |
+
table: res_partner
|
| 22 |
+
alias: p
|
| 23 |
+
# NO date_col β a registry is not a dated event stream. Stated rather than
|
| 24 |
+
# omitted, so its absence reads as a fact and not as an unfinished file.
|
| 25 |
+
dims:
|
| 26 |
+
agent: {label: "Sales agent"}
|
| 27 |
+
state: {label: "State"}
|
| 28 |
+
country: {label: "Country"}
|
| 29 |
+
|
| 30 |
+
# key / label / type / kind, derived from the grid contract. `kind` says where the
|
| 31 |
+
# value COMES FROM: a `data` column is stored on the row; a `rollup` is computed
|
| 32 |
+
# from another topic; a `link` points at another database.
|
| 33 |
+
fields:
|
| 34 |
+
- key: customer
|
| 35 |
+
label: "Customer"
|
| 36 |
+
type: text
|
| 37 |
+
kind: data
|
| 38 |
+
means: "The customer's name in Odoo. One row per customer who ordered in the last 24 months."
|
| 39 |
+
- key: partner_id
|
| 40 |
+
label: "Odoo ID"
|
| 41 |
+
type: int
|
| 42 |
+
kind: data
|
| 43 |
+
means: "The Odoo res.partner id β this row's identity, and the key every Odoo document joins on. W33-T43 / owner item 12: 'One unique ID per database always.' It is DERIVED rather than read off the pool row because a customer row's pid IS the partner id, so the value is already on every row and a second copy in the pool would be a second source for one fact."
|
| 44 |
+
- key: odoo_status
|
| 45 |
+
label: "Odoo record"
|
| 46 |
+
type: status
|
| 47 |
+
kind: data
|
| 48 |
+
means: "Whether this customer still exists in Odoo. Archived means deleted there."
|
| 49 |
+
- key: agent
|
| 50 |
+
label: "Agent"
|
| 51 |
+
type: text
|
| 52 |
+
kind: data
|
| 53 |
+
means: "The sales agent who owns this account."
|
| 54 |
+
- key: dba
|
| 55 |
+
label: "DBA"
|
| 56 |
+
type: select
|
| 57 |
+
kind: data
|
| 58 |
+
means: "The brand this customer buys from - Fisch, Royal, or both. Amazon-channel orders are not a DBA."
|
| 59 |
+
- key: salesperson
|
| 60 |
+
label: "Salesperson"
|
| 61 |
+
type: text
|
| 62 |
+
kind: data
|
| 63 |
+
means: "Who keyed in most of this customer's orders β not the Agent, who owns the account."
|
| 64 |
+
- key: street
|
| 65 |
+
label: "Street"
|
| 66 |
+
type: text
|
| 67 |
+
kind: data
|
| 68 |
+
means: "First address line, from res.partner directly - not the geocoder, so a customer the map cannot place still shows its address."
|
| 69 |
+
- key: street2
|
| 70 |
+
label: "Street 2"
|
| 71 |
+
type: text
|
| 72 |
+
kind: data
|
| 73 |
+
means: "Second address line (suite, unit, floor) on the customer's Odoo address."
|
| 74 |
+
- key: city
|
| 75 |
+
label: "City"
|
| 76 |
+
type: text
|
| 77 |
+
kind: data
|
| 78 |
+
means: "City on the customer's Odoo address."
|
| 79 |
+
- key: state
|
| 80 |
+
label: "State"
|
| 81 |
+
type: text
|
| 82 |
+
kind: data
|
| 83 |
+
means: "State or province on the customer's Odoo address."
|
| 84 |
+
- key: country
|
| 85 |
+
label: "Country"
|
| 86 |
+
type: text
|
| 87 |
+
kind: data
|
| 88 |
+
means: "Country on the customer's Odoo address."
|
| 89 |
+
- key: zip
|
| 90 |
+
label: "ZIP"
|
| 91 |
+
type: text
|
| 92 |
+
kind: data
|
| 93 |
+
means: "Postal code on the customer's Odoo address."
|
| 94 |
+
- key: customer_since
|
| 95 |
+
label: "Customer since"
|
| 96 |
+
type: date
|
| 97 |
+
kind: data
|
| 98 |
+
means: "When this customer was first set up in Odoo."
|
| 99 |
+
- key: tags
|
| 100 |
+
label: "Tags"
|
| 101 |
+
type: text
|
| 102 |
+
kind: data
|
| 103 |
+
means: "Odoo labels on this customer, comma-separated."
|
| 104 |
+
- key: pricelist
|
| 105 |
+
label: "Price list"
|
| 106 |
+
type: text
|
| 107 |
+
kind: data
|
| 108 |
+
means: "The price list this customer buys on."
|
| 109 |
+
- key: payment_terms
|
| 110 |
+
label: "Payment terms"
|
| 111 |
+
type: text
|
| 112 |
+
kind: data
|
| 113 |
+
means: "Payment terms on this customer's account β Net 30, for example."
|
| 114 |
+
- key: last_order
|
| 115 |
+
label: "Last order"
|
| 116 |
+
type: date
|
| 117 |
+
kind: data
|
| 118 |
+
means: "Date of the most recent confirmed order."
|
| 119 |
+
- key: overdue_days
|
| 120 |
+
label: "Overdue days"
|
| 121 |
+
type: int
|
| 122 |
+
kind: data
|
| 123 |
+
means: "How many days late this customer is running against their own usual ordering rhythm."
|
| 124 |
+
- key: est_missed
|
| 125 |
+
label: "Est. missed $"
|
| 126 |
+
type: currency
|
| 127 |
+
kind: data
|
| 128 |
+
means: "Estimated sales missed while quiet: missed orders (capped at 3) times average order value. An estimate, not money owed."
|
| 129 |
+
- key: ar_open
|
| 130 |
+
label: "AR current $"
|
| 131 |
+
type: currency
|
| 132 |
+
kind: data
|
| 133 |
+
means: "Invoiced money owed but not yet due (a 5-day grace applies before it counts as overdue)."
|
| 134 |
+
- key: ar_overdue
|
| 135 |
+
label: "AR overdue $"
|
| 136 |
+
type: currency
|
| 137 |
+
kind: data
|
| 138 |
+
means: "Invoiced money past due β same basis as the Collections page."
|
| 139 |
+
- key: ar_outstanding
|
| 140 |
+
label: "AR outstanding $"
|
| 141 |
+
type: currency
|
| 142 |
+
kind: data
|
| 143 |
+
means: "Total invoiced money owed right now: AR current $ plus AR overdue $."
|
| 144 |
+
- key: ar_exposure
|
| 145 |
+
label: "Credit exposure $"
|
| 146 |
+
type: currency
|
| 147 |
+
kind: data
|
| 148 |
+
means: "The most you could be out if they stopped paying today: open, overdue, draft and not-yet-invoiced."
|
| 149 |
+
- key: ar_aged_1_30
|
| 150 |
+
label: "1-30 days $"
|
| 151 |
+
type: currency
|
| 152 |
+
kind: data
|
| 153 |
+
means: "Overdue between 1 and 30 days. The four aging buckets sum to AR overdue $."
|
| 154 |
+
- key: ar_aged_31_60
|
| 155 |
+
label: "31-60 days $"
|
| 156 |
+
type: currency
|
| 157 |
+
kind: data
|
| 158 |
+
means: "Overdue between 31 and 60 days. The four aging buckets sum to AR overdue $."
|
| 159 |
+
- key: ar_aged_61_90
|
| 160 |
+
label: "61-90 days $"
|
| 161 |
+
type: currency
|
| 162 |
+
kind: data
|
| 163 |
+
means: "Overdue between 61 and 90 days. The four aging buckets sum to AR overdue $."
|
| 164 |
+
- key: ar_aged_90_plus
|
| 165 |
+
label: "90+ days $"
|
| 166 |
+
type: currency
|
| 167 |
+
kind: data
|
| 168 |
+
means: "Overdue by more than 90 days. The four aging buckets sum to AR overdue $."
|
| 169 |
+
- key: days_to_pay
|
| 170 |
+
label: "Days to pay"
|
| 171 |
+
type: int
|
| 172 |
+
kind: data
|
| 173 |
+
means: "Average days to pay an invoice in full. Blank means no fully paid invoice yet."
|
| 174 |
+
- key: top_category
|
| 175 |
+
label: "Top category"
|
| 176 |
+
type: text
|
| 177 |
+
kind: data
|
| 178 |
+
means: "The category this customer spent the most on in the last 12 months."
|
| 179 |
+
- key: top_category_pct
|
| 180 |
+
label: "Top category %"
|
| 181 |
+
type: pct
|
| 182 |
+
kind: data
|
| 183 |
+
means: "Share of last-12-months spend that went to the top category."
|
| 184 |
+
- key: sku_count
|
| 185 |
+
label: "SKUs bought"
|
| 186 |
+
type: int
|
| 187 |
+
kind: data
|
| 188 |
+
means: "Distinct products bought in the last 12 months."
|
| 189 |
+
- key: top_sku
|
| 190 |
+
label: "Top SKU"
|
| 191 |
+
type: text
|
| 192 |
+
kind: data
|
| 193 |
+
means: "The product this customer spent the most on in the last 12 months."
|
| 194 |
+
- key: days_since
|
| 195 |
+
label: "Days since order"
|
| 196 |
+
type: int
|
| 197 |
+
kind: data
|
| 198 |
+
means: "Days since the last confirmed order."
|
| 199 |
+
- key: typical_gap_days
|
| 200 |
+
label: "Typical gap days"
|
| 201 |
+
type: int
|
| 202 |
+
kind: data
|
| 203 |
+
means: "Days this customer usually goes between orders, from their own history."
|
| 204 |
+
- key: notes
|
| 205 |
+
label: "Notes"
|
| 206 |
+
type: text
|
| 207 |
+
kind: data
|
| 208 |
+
means: "Your notes on this customer. Saved in this app only, visible only to you."
|
| 209 |
+
|
| 210 |
+
ai_context: >
|
| 211 |
+
The customer REGISTRY as a person sees it in the app β the database labelled 'Odoo customers'.
|
| 212 |
+
Use it to answer 'who is this customer', 'which customers exist', 'which agent owns them', 'where are they', and for the AR columns it carries (ar_open, ar_overdue, days_to_pay), which are reconciled by modules/ar.py.
|
| 213 |
+
β For anything WINDOWED or at line grain go to sales_lines / customer_invoices / receivables β those topics own the time dimension and this one has none.
|
platform/model/topics/odoo_invoices.yml
CHANGED
|
@@ -1,98 +1,98 @@
|
|
| 1 |
-
# ββ W33-T49 (owner item 13) β an ENTITY topic: the semantic layer can finally see
|
| 2 |
-
# the databases a person actually opens, not only the line and document grains.
|
| 3 |
-
#
|
| 4 |
-
# β THE `fields:` BLOCK IS GENERATED FROM THE GRID'S OWN FIELD CONTRACT
|
| 5 |
-
# and is held to it by `verify_query.py::section_entity_topics`. Do not hand-edit a
|
| 6 |
-
# field row: add the column to the grid contract and re-emit, or the agent is being
|
| 7 |
-
# trained on a schema the product does not have.
|
| 8 |
-
key: odoo_invoices
|
| 9 |
-
label: "Odoo invoices"
|
| 10 |
-
entity: account.move
|
| 11 |
-
# The database this topic DESCRIBES β the same store key the nav opens (W33-T46).
|
| 12 |
-
grid: ut_odoo_invoices
|
| 13 |
-
subject: "odoo:account.move.customer"
|
| 14 |
-
grain: "one row per posted CUSTOMER invoice or credit note (document grain)"
|
| 15 |
-
scope:
|
| 16 |
-
population: "move_type in (out_invoice, out_refund) AND state = posted"
|
| 17 |
-
signs: "amount_residual_signed is POSITIVE for an invoice and NEGATIVE for a credit note, so summing it nets the credit notes correctly"
|
| 18 |
-
store:
|
| 19 |
-
table: account_move
|
| 20 |
-
alias: m
|
| 21 |
-
date_col: "m.invoice_date"
|
| 22 |
-
dims:
|
| 23 |
-
customer: {label: "Customer"}
|
| 24 |
-
payment_state: {label: "Payment state"}
|
| 25 |
-
move_type: {label: "Document type"}
|
| 26 |
-
|
| 27 |
-
# key / label / type / kind, derived from the grid contract. `kind` says where the
|
| 28 |
-
# value COMES FROM: a `data` column is stored on the row; a `rollup` is computed
|
| 29 |
-
# from another topic; a `link` points at another database.
|
| 30 |
-
fields:
|
| 31 |
-
- key: invoice_no
|
| 32 |
-
label: "Invoice"
|
| 33 |
-
type: text
|
| 34 |
-
kind: data
|
| 35 |
-
- key: odoo_id
|
| 36 |
-
label: "Odoo ID"
|
| 37 |
-
type: int
|
| 38 |
-
kind: data
|
| 39 |
-
means: "The `account.move` id. Also this row's id."
|
| 40 |
-
- key: customer
|
| 41 |
-
label: "Customer"
|
| 42 |
-
type: text
|
| 43 |
-
kind: data
|
| 44 |
-
- key: partner_id
|
| 45 |
-
label: "Odoo partner id"
|
| 46 |
-
type: int
|
| 47 |
-
kind: data
|
| 48 |
-
- key: invoice_date
|
| 49 |
-
label: "Invoice date"
|
| 50 |
-
type: date
|
| 51 |
-
kind: data
|
| 52 |
-
- key: due_date
|
| 53 |
-
label: "Due date"
|
| 54 |
-
type: date
|
| 55 |
-
kind: data
|
| 56 |
-
- key: residual
|
| 57 |
-
label: "Outstanding $"
|
| 58 |
-
type: currency
|
| 59 |
-
kind: data
|
| 60 |
-
means: "Odoo's signed residual. Exactly 0 on every settled document, which is why AR rollups need no filter."
|
| 61 |
-
- key: amount_untaxed
|
| 62 |
-
label: "Invoiced $"
|
| 63 |
-
type: currency
|
| 64 |
-
kind: data
|
| 65 |
-
- key: payment_state
|
| 66 |
-
label: "Payment state"
|
| 67 |
-
type: select
|
| 68 |
-
kind: data
|
| 69 |
-
- key: move_type
|
| 70 |
-
label: "Document"
|
| 71 |
-
type: select
|
| 72 |
-
kind: data
|
| 73 |
-
- key: wholesale_scope
|
| 74 |
-
label: "In wholesale scope"
|
| 75 |
-
type: checkbox
|
| 76 |
-
kind: data
|
| 77 |
-
means: "Unticked = the GIFTWARE DEALS / Amazon channel, which every wholesale metric in this product excludes. The row is kept so no Odoo id is missing; filter on this column to reconcile against the AR page."
|
| 78 |
-
- key: origin_order
|
| 79 |
-
label: "Source order"
|
| 80 |
-
type: text
|
| 81 |
-
kind: data
|
| 82 |
-
means: "Odoo's `invoice_origin` - usually the order name, sometimes blank."
|
| 83 |
-
- key: order_link
|
| 84 |
-
label: "Order record"
|
| 85 |
-
type: link
|
| 86 |
-
kind: link
|
| 87 |
-
to_grid: ut_odoo_orders
|
| 88 |
-
- key: refreshed
|
| 89 |
-
label: "Refreshed"
|
| 90 |
-
type: date
|
| 91 |
-
kind: data
|
| 92 |
-
means: "When this row was last reconciled against Odoo."
|
| 93 |
-
|
| 94 |
-
ai_context: >
|
| 95 |
-
Customer invoices at DOCUMENT grain β one row per invoice, not per line.
|
| 96 |
-
Use it for AR questions: what is open, what is overdue, what was invoiced.
|
| 97 |
-
For anything at product or line grain use invoice_lines; for the reconciled receivables view use receivables.
|
| 98 |
-
`origin_order` is the single stored key that joins an invoice back to the order it was raised from.
|
|
|
|
| 1 |
+
# ββ W33-T49 (owner item 13) β an ENTITY topic: the semantic layer can finally see
|
| 2 |
+
# the databases a person actually opens, not only the line and document grains.
|
| 3 |
+
#
|
| 4 |
+
# β THE `fields:` BLOCK IS GENERATED FROM THE GRID'S OWN FIELD CONTRACT
|
| 5 |
+
# and is held to it by `verify_query.py::section_entity_topics`. Do not hand-edit a
|
| 6 |
+
# field row: add the column to the grid contract and re-emit, or the agent is being
|
| 7 |
+
# trained on a schema the product does not have.
|
| 8 |
+
key: odoo_invoices
|
| 9 |
+
label: "Odoo invoices"
|
| 10 |
+
entity: account.move
|
| 11 |
+
# The database this topic DESCRIBES β the same store key the nav opens (W33-T46).
|
| 12 |
+
grid: ut_odoo_invoices
|
| 13 |
+
subject: "odoo:account.move.customer"
|
| 14 |
+
grain: "one row per posted CUSTOMER invoice or credit note (document grain)"
|
| 15 |
+
scope:
|
| 16 |
+
population: "move_type in (out_invoice, out_refund) AND state = posted"
|
| 17 |
+
signs: "amount_residual_signed is POSITIVE for an invoice and NEGATIVE for a credit note, so summing it nets the credit notes correctly"
|
| 18 |
+
store:
|
| 19 |
+
table: account_move
|
| 20 |
+
alias: m
|
| 21 |
+
date_col: "m.invoice_date"
|
| 22 |
+
dims:
|
| 23 |
+
customer: {label: "Customer"}
|
| 24 |
+
payment_state: {label: "Payment state"}
|
| 25 |
+
move_type: {label: "Document type"}
|
| 26 |
+
|
| 27 |
+
# key / label / type / kind, derived from the grid contract. `kind` says where the
|
| 28 |
+
# value COMES FROM: a `data` column is stored on the row; a `rollup` is computed
|
| 29 |
+
# from another topic; a `link` points at another database.
|
| 30 |
+
fields:
|
| 31 |
+
- key: invoice_no
|
| 32 |
+
label: "Invoice"
|
| 33 |
+
type: text
|
| 34 |
+
kind: data
|
| 35 |
+
- key: odoo_id
|
| 36 |
+
label: "Odoo ID"
|
| 37 |
+
type: int
|
| 38 |
+
kind: data
|
| 39 |
+
means: "The `account.move` id. Also this row's id."
|
| 40 |
+
- key: customer
|
| 41 |
+
label: "Customer"
|
| 42 |
+
type: text
|
| 43 |
+
kind: data
|
| 44 |
+
- key: partner_id
|
| 45 |
+
label: "Odoo partner id"
|
| 46 |
+
type: int
|
| 47 |
+
kind: data
|
| 48 |
+
- key: invoice_date
|
| 49 |
+
label: "Invoice date"
|
| 50 |
+
type: date
|
| 51 |
+
kind: data
|
| 52 |
+
- key: due_date
|
| 53 |
+
label: "Due date"
|
| 54 |
+
type: date
|
| 55 |
+
kind: data
|
| 56 |
+
- key: residual
|
| 57 |
+
label: "Outstanding $"
|
| 58 |
+
type: currency
|
| 59 |
+
kind: data
|
| 60 |
+
means: "Odoo's signed residual. Exactly 0 on every settled document, which is why AR rollups need no filter."
|
| 61 |
+
- key: amount_untaxed
|
| 62 |
+
label: "Invoiced $"
|
| 63 |
+
type: currency
|
| 64 |
+
kind: data
|
| 65 |
+
- key: payment_state
|
| 66 |
+
label: "Payment state"
|
| 67 |
+
type: select
|
| 68 |
+
kind: data
|
| 69 |
+
- key: move_type
|
| 70 |
+
label: "Document"
|
| 71 |
+
type: select
|
| 72 |
+
kind: data
|
| 73 |
+
- key: wholesale_scope
|
| 74 |
+
label: "In wholesale scope"
|
| 75 |
+
type: checkbox
|
| 76 |
+
kind: data
|
| 77 |
+
means: "Unticked = the GIFTWARE DEALS / Amazon channel, which every wholesale metric in this product excludes. The row is kept so no Odoo id is missing; filter on this column to reconcile against the AR page."
|
| 78 |
+
- key: origin_order
|
| 79 |
+
label: "Source order"
|
| 80 |
+
type: text
|
| 81 |
+
kind: data
|
| 82 |
+
means: "Odoo's `invoice_origin` - usually the order name, sometimes blank."
|
| 83 |
+
- key: order_link
|
| 84 |
+
label: "Order record"
|
| 85 |
+
type: link
|
| 86 |
+
kind: link
|
| 87 |
+
to_grid: ut_odoo_orders
|
| 88 |
+
- key: refreshed
|
| 89 |
+
label: "Refreshed"
|
| 90 |
+
type: date
|
| 91 |
+
kind: data
|
| 92 |
+
means: "When this row was last reconciled against Odoo."
|
| 93 |
+
|
| 94 |
+
ai_context: >
|
| 95 |
+
Customer invoices at DOCUMENT grain β one row per invoice, not per line.
|
| 96 |
+
Use it for AR questions: what is open, what is overdue, what was invoiced.
|
| 97 |
+
For anything at product or line grain use invoice_lines; for the reconciled receivables view use receivables.
|
| 98 |
+
`origin_order` is the single stored key that joins an invoice back to the order it was raised from.
|
platform/model/topics/odoo_orders.yml
CHANGED
|
@@ -1,87 +1,87 @@
|
|
| 1 |
-
# ββ W33-T49 (owner item 13) β an ENTITY topic: the semantic layer can finally see
|
| 2 |
-
# the databases a person actually opens, not only the line and document grains.
|
| 3 |
-
#
|
| 4 |
-
# β THE `fields:` BLOCK IS GENERATED FROM THE GRID'S OWN FIELD CONTRACT
|
| 5 |
-
# and is held to it by `verify_query.py::section_entity_topics`. Do not hand-edit a
|
| 6 |
-
# field row: add the column to the grid contract and re-emit, or the agent is being
|
| 7 |
-
# trained on a schema the product does not have.
|
| 8 |
-
key: odoo_orders
|
| 9 |
-
label: "Odoo orders"
|
| 10 |
-
entity: sale.order
|
| 11 |
-
# The database this topic DESCRIBES β the same store key the nav opens (W33-T46).
|
| 12 |
-
grid: ut_odoo_orders
|
| 13 |
-
subject: "odoo:sale.order"
|
| 14 |
-
grain: "one row per confirmed sales order (document grain)"
|
| 15 |
-
scope:
|
| 16 |
-
population: "state in (sale, done) β quotations and cancellations excluded"
|
| 17 |
-
vs_sales_orders: "β DISTINCT from the `sales_orders` topic: that one is the wholesale-scoped analytical view (teams 5/6, house accounts excluded); THIS one is the registry a person opens as a grid, and carries the row set the app shows"
|
| 18 |
-
store:
|
| 19 |
-
table: sale_order
|
| 20 |
-
alias: o
|
| 21 |
-
date_col: "o.date_order"
|
| 22 |
-
dims:
|
| 23 |
-
customer: {label: "Customer"}
|
| 24 |
-
team: {label: "Business unit"}
|
| 25 |
-
state: {label: "State"}
|
| 26 |
-
invoice_status: {label: "Invoice status"}
|
| 27 |
-
|
| 28 |
-
# key / label / type / kind, derived from the grid contract. `kind` says where the
|
| 29 |
-
# value COMES FROM: a `data` column is stored on the row; a `rollup` is computed
|
| 30 |
-
# from another topic; a `link` points at another database.
|
| 31 |
-
fields:
|
| 32 |
-
- key: order_no
|
| 33 |
-
label: "Order"
|
| 34 |
-
type: text
|
| 35 |
-
kind: data
|
| 36 |
-
- key: odoo_id
|
| 37 |
-
label: "Odoo ID"
|
| 38 |
-
type: int
|
| 39 |
-
kind: data
|
| 40 |
-
means: "The `sale.order` id. Also this row's id."
|
| 41 |
-
- key: customer
|
| 42 |
-
label: "Customer"
|
| 43 |
-
type: text
|
| 44 |
-
kind: data
|
| 45 |
-
- key: partner_id
|
| 46 |
-
label: "Odoo partner id"
|
| 47 |
-
type: int
|
| 48 |
-
kind: data
|
| 49 |
-
- key: order_date
|
| 50 |
-
label: "Order date"
|
| 51 |
-
type: date
|
| 52 |
-
kind: data
|
| 53 |
-
- key: amount_untaxed
|
| 54 |
-
label: "Order $"
|
| 55 |
-
type: currency
|
| 56 |
-
kind: data
|
| 57 |
-
- key: team
|
| 58 |
-
label: "Business unit"
|
| 59 |
-
type: select
|
| 60 |
-
kind: data
|
| 61 |
-
- key: state
|
| 62 |
-
label: "State"
|
| 63 |
-
type: select
|
| 64 |
-
kind: data
|
| 65 |
-
- key: invoice_status
|
| 66 |
-
label: "Invoice status"
|
| 67 |
-
type: select
|
| 68 |
-
kind: data
|
| 69 |
-
- key: wholesale_scope
|
| 70 |
-
label: "In wholesale scope"
|
| 71 |
-
type: checkbox
|
| 72 |
-
kind: data
|
| 73 |
-
means: "Unticked = the GIFTWARE DEALS / Amazon channel, which every wholesale metric in this product excludes. The row is kept so no Odoo id is missing; filter on this column to reconcile against the AR page."
|
| 74 |
-
- key: invoices
|
| 75 |
-
label: "Invoices"
|
| 76 |
-
type: link
|
| 77 |
-
kind: link
|
| 78 |
-
to_grid: ut_odoo_invoices
|
| 79 |
-
- key: refreshed
|
| 80 |
-
label: "Refreshed"
|
| 81 |
-
type: date
|
| 82 |
-
kind: data
|
| 83 |
-
means: "When this row was last reconciled against Odoo."
|
| 84 |
-
|
| 85 |
-
ai_context: >
|
| 86 |
-
Sales orders at DOCUMENT grain, as the app's grid shows them.
|
| 87 |
-
β If you are asked for revenue, prefer `sales_lines` (line grain, wholesale scope) or `sales_orders` β this topic exists to describe the ORDER RECORD a person can open, and its scope is the grid's, not the analytical model's.
|
|
|
|
| 1 |
+
# ββ W33-T49 (owner item 13) β an ENTITY topic: the semantic layer can finally see
|
| 2 |
+
# the databases a person actually opens, not only the line and document grains.
|
| 3 |
+
#
|
| 4 |
+
# β THE `fields:` BLOCK IS GENERATED FROM THE GRID'S OWN FIELD CONTRACT
|
| 5 |
+
# and is held to it by `verify_query.py::section_entity_topics`. Do not hand-edit a
|
| 6 |
+
# field row: add the column to the grid contract and re-emit, or the agent is being
|
| 7 |
+
# trained on a schema the product does not have.
|
| 8 |
+
key: odoo_orders
|
| 9 |
+
label: "Odoo orders"
|
| 10 |
+
entity: sale.order
|
| 11 |
+
# The database this topic DESCRIBES β the same store key the nav opens (W33-T46).
|
| 12 |
+
grid: ut_odoo_orders
|
| 13 |
+
subject: "odoo:sale.order"
|
| 14 |
+
grain: "one row per confirmed sales order (document grain)"
|
| 15 |
+
scope:
|
| 16 |
+
population: "state in (sale, done) β quotations and cancellations excluded"
|
| 17 |
+
vs_sales_orders: "β DISTINCT from the `sales_orders` topic: that one is the wholesale-scoped analytical view (teams 5/6, house accounts excluded); THIS one is the registry a person opens as a grid, and carries the row set the app shows"
|
| 18 |
+
store:
|
| 19 |
+
table: sale_order
|
| 20 |
+
alias: o
|
| 21 |
+
date_col: "o.date_order"
|
| 22 |
+
dims:
|
| 23 |
+
customer: {label: "Customer"}
|
| 24 |
+
team: {label: "Business unit"}
|
| 25 |
+
state: {label: "State"}
|
| 26 |
+
invoice_status: {label: "Invoice status"}
|
| 27 |
+
|
| 28 |
+
# key / label / type / kind, derived from the grid contract. `kind` says where the
|
| 29 |
+
# value COMES FROM: a `data` column is stored on the row; a `rollup` is computed
|
| 30 |
+
# from another topic; a `link` points at another database.
|
| 31 |
+
fields:
|
| 32 |
+
- key: order_no
|
| 33 |
+
label: "Order"
|
| 34 |
+
type: text
|
| 35 |
+
kind: data
|
| 36 |
+
- key: odoo_id
|
| 37 |
+
label: "Odoo ID"
|
| 38 |
+
type: int
|
| 39 |
+
kind: data
|
| 40 |
+
means: "The `sale.order` id. Also this row's id."
|
| 41 |
+
- key: customer
|
| 42 |
+
label: "Customer"
|
| 43 |
+
type: text
|
| 44 |
+
kind: data
|
| 45 |
+
- key: partner_id
|
| 46 |
+
label: "Odoo partner id"
|
| 47 |
+
type: int
|
| 48 |
+
kind: data
|
| 49 |
+
- key: order_date
|
| 50 |
+
label: "Order date"
|
| 51 |
+
type: date
|
| 52 |
+
kind: data
|
| 53 |
+
- key: amount_untaxed
|
| 54 |
+
label: "Order $"
|
| 55 |
+
type: currency
|
| 56 |
+
kind: data
|
| 57 |
+
- key: team
|
| 58 |
+
label: "Business unit"
|
| 59 |
+
type: select
|
| 60 |
+
kind: data
|
| 61 |
+
- key: state
|
| 62 |
+
label: "State"
|
| 63 |
+
type: select
|
| 64 |
+
kind: data
|
| 65 |
+
- key: invoice_status
|
| 66 |
+
label: "Invoice status"
|
| 67 |
+
type: select
|
| 68 |
+
kind: data
|
| 69 |
+
- key: wholesale_scope
|
| 70 |
+
label: "In wholesale scope"
|
| 71 |
+
type: checkbox
|
| 72 |
+
kind: data
|
| 73 |
+
means: "Unticked = the GIFTWARE DEALS / Amazon channel, which every wholesale metric in this product excludes. The row is kept so no Odoo id is missing; filter on this column to reconcile against the AR page."
|
| 74 |
+
- key: invoices
|
| 75 |
+
label: "Invoices"
|
| 76 |
+
type: link
|
| 77 |
+
kind: link
|
| 78 |
+
to_grid: ut_odoo_invoices
|
| 79 |
+
- key: refreshed
|
| 80 |
+
label: "Refreshed"
|
| 81 |
+
type: date
|
| 82 |
+
kind: data
|
| 83 |
+
means: "When this row was last reconciled against Odoo."
|
| 84 |
+
|
| 85 |
+
ai_context: >
|
| 86 |
+
Sales orders at DOCUMENT grain, as the app's grid shows them.
|
| 87 |
+
β If you are asked for revenue, prefer `sales_lines` (line grain, wholesale scope) or `sales_orders` β this topic exists to describe the ORDER RECORD a person can open, and its scope is the grid's, not the analytical model's.
|
platform/model/topics/odoo_products.yml
CHANGED
|
@@ -1,147 +1,147 @@
|
|
| 1 |
-
# ββ W33-T49 (owner item 13) β an ENTITY topic: the semantic layer can finally see
|
| 2 |
-
# the databases a person actually opens, not only the line and document grains.
|
| 3 |
-
#
|
| 4 |
-
# β THE `fields:` BLOCK IS GENERATED FROM THE GRID'S OWN FIELD CONTRACT
|
| 5 |
-
# and is held to it by `verify_query.py::section_entity_topics`. Do not hand-edit a
|
| 6 |
-
# field row: add the column to the grid contract and re-emit, or the agent is being
|
| 7 |
-
# trained on a schema the product does not have.
|
| 8 |
-
key: odoo_products
|
| 9 |
-
label: "Odoo products"
|
| 10 |
-
entity: product.product
|
| 11 |
-
# The database this topic DESCRIBES β the same store key the nav opens (W33-T46).
|
| 12 |
-
grid: product_data
|
| 13 |
-
subject: "odoo:product.product"
|
| 14 |
-
grain: "one row per SKU in the active catalogue β a CATALOGUE, not a dated event stream"
|
| 15 |
-
scope:
|
| 16 |
-
population: "every ACTIVE product, sold or not (measured 5,862). Deliberately not the sales-window universe: that lands at 2,717 and hides ~2,550 SKUs that have never sold in wholesale scope"
|
| 17 |
-
identity: "the SKU code (`default_code`). β 33 active products carry NO code and are keyed `pid:<odoo product id>` instead, so every active product has exactly one row and none is dropped"
|
| 18 |
-
merged: "W33-T44 retired `ut_odoo_products`, which presented this same subject on Odoo's product_id. This topic describes the SURVIVING database, `product_data`"
|
| 19 |
-
no_date: "a catalogue has no date dimension; ask sales_lines for movement"
|
| 20 |
-
store:
|
| 21 |
-
table: product_product
|
| 22 |
-
alias: pp
|
| 23 |
-
# NO date_col β a registry is not a dated event stream. Stated rather than
|
| 24 |
-
# omitted, so its absence reads as a fact and not as an unfinished file.
|
| 25 |
-
dims:
|
| 26 |
-
category: {label: "Category"}
|
| 27 |
-
supplier: {label: "Supplier"}
|
| 28 |
-
|
| 29 |
-
# key / label / type / kind, derived from the grid contract. `kind` says where the
|
| 30 |
-
# value COMES FROM: a `data` column is stored on the row; a `rollup` is computed
|
| 31 |
-
# from another topic; a `link` points at another database.
|
| 32 |
-
fields:
|
| 33 |
-
- key: code
|
| 34 |
-
label: "SKU"
|
| 35 |
-
type: text
|
| 36 |
-
kind: data
|
| 37 |
-
means: "The SKU code β the product's real business key. `pid` is a stable CRC32 of it because the grid keys on an integer."
|
| 38 |
-
- key: product
|
| 39 |
-
label: "Product"
|
| 40 |
-
type: text
|
| 41 |
-
kind: data
|
| 42 |
-
means: "Product name as it appears in Odoo."
|
| 43 |
-
- key: category
|
| 44 |
-
label: "Category"
|
| 45 |
-
type: select
|
| 46 |
-
kind: data
|
| 47 |
-
means: "Product category; '(uncategorized)' when Odoo carries none."
|
| 48 |
-
- key: supplier
|
| 49 |
-
label: "Supplier"
|
| 50 |
-
type: text
|
| 51 |
-
kind: data
|
| 52 |
-
means: "Who makes it. Editable here and shared with everyone in the workspace; seeded from the inventory mastersheet."
|
| 53 |
-
- key: origin_country
|
| 54 |
-
label: "Country"
|
| 55 |
-
type: text
|
| 56 |
-
kind: data
|
| 57 |
-
means: "Country of origin. Editable here and shared with everyone; seeded from the inventory mastersheet."
|
| 58 |
-
- key: lead_days
|
| 59 |
-
label: "Lead time (days)"
|
| 60 |
-
type: int
|
| 61 |
-
kind: data
|
| 62 |
-
means: "Order-to-arrival days for this supplier. Drives the buy signal. Editable and shared with everyone."
|
| 63 |
-
- key: first_cost
|
| 64 |
-
label: "First cost"
|
| 65 |
-
type: currency
|
| 66 |
-
kind: data
|
| 67 |
-
means: "Quoted unit cost at origin, before freight and duty. Editable and shared with everyone."
|
| 68 |
-
- key: price_fisch
|
| 69 |
-
label: "Fisch price"
|
| 70 |
-
type: currency
|
| 71 |
-
kind: data
|
| 72 |
-
means: "Fisch pricelist price for this SKU. Blank when that list prices it nowhere."
|
| 73 |
-
- key: price_royal_1
|
| 74 |
-
label: "Royal 1 price"
|
| 75 |
-
type: currency
|
| 76 |
-
kind: data
|
| 77 |
-
means: "Royal 1 pricelist price for this SKU. Blank when that list prices it nowhere."
|
| 78 |
-
- key: price_royal_2
|
| 79 |
-
label: "Royal 2 price"
|
| 80 |
-
type: currency
|
| 81 |
-
kind: data
|
| 82 |
-
means: "Royal 2 pricelist price for this SKU. Blank when that list prices it nowhere."
|
| 83 |
-
- key: rev_ytd
|
| 84 |
-
label: "Revenue YTD"
|
| 85 |
-
type: currency
|
| 86 |
-
kind: data
|
| 87 |
-
means: "Year-to-date revenue for this SKU, BU-scoped when the caller is."
|
| 88 |
-
- key: rev_ly
|
| 89 |
-
label: "Revenue LY"
|
| 90 |
-
type: currency
|
| 91 |
-
kind: data
|
| 92 |
-
means: "Same period last year β seasonal wholesale compares like for like."
|
| 93 |
-
- key: yoy_pct
|
| 94 |
-
label: "YoY %"
|
| 95 |
-
type: pct
|
| 96 |
-
kind: data
|
| 97 |
-
means: "Year-over-year change; null when last year was zero (a ratio to zero is not a number)."
|
| 98 |
-
- key: qty_ytd
|
| 99 |
-
label: "Units YTD"
|
| 100 |
-
type: int
|
| 101 |
-
kind: data
|
| 102 |
-
means: "Units sold year to date."
|
| 103 |
-
- key: orders_ytd
|
| 104 |
-
label: "Orders YTD"
|
| 105 |
-
type: int
|
| 106 |
-
kind: data
|
| 107 |
-
means: "Distinct orders containing this SKU, year to date."
|
| 108 |
-
- key: on_hand
|
| 109 |
-
label: "On hand"
|
| 110 |
-
type: int
|
| 111 |
-
kind: data
|
| 112 |
-
means: "Units in stock. CONSOLIDATED β one physical warehouse, not brand-tagged, so this column is ABSENT for a BU-scoped caller rather than silently company-wide."
|
| 113 |
-
- key: unit_cost
|
| 114 |
-
label: "Unit cost"
|
| 115 |
-
type: currency
|
| 116 |
-
kind: data
|
| 117 |
-
means: "Inventory unit cost. Consolidated; absent for a BU-scoped caller."
|
| 118 |
-
- key: inv_value
|
| 119 |
-
label: "Stock value"
|
| 120 |
-
type: currency
|
| 121 |
-
kind: data
|
| 122 |
-
means: "On-hand value at cost. Consolidated; absent for a BU-scoped caller."
|
| 123 |
-
- key: qty_ltm
|
| 124 |
-
label: "Units LTM"
|
| 125 |
-
type: int
|
| 126 |
-
kind: data
|
| 127 |
-
means: "Units sold in the last twelve months. Consolidated; absent for a BU-scoped caller."
|
| 128 |
-
- key: dos
|
| 129 |
-
label: "Days of supply"
|
| 130 |
-
type: int
|
| 131 |
-
kind: data
|
| 132 |
-
means: "Days of supply at the LTM rate; null means it never sells through. Consolidated; absent for a BU-scoped caller."
|
| 133 |
-
- key: cover_gap_d
|
| 134 |
-
label: "Cover gap (days)"
|
| 135 |
-
type: int
|
| 136 |
-
kind: data
|
| 137 |
-
means: "Days of supply minus lead time. Negative means it runs out before a reorder lands."
|
| 138 |
-
- key: stock_bucket
|
| 139 |
-
label: "Stock status"
|
| 140 |
-
type: select
|
| 141 |
-
kind: data
|
| 142 |
-
means: "Dead / excess / healthy bucket from the inventory module. Consolidated; absent for a BU-scoped caller."
|
| 143 |
-
|
| 144 |
-
ai_context: >
|
| 145 |
-
The product CATALOGUE β the database labelled 'Odoo products'.
|
| 146 |
-
Use it for 'what do we sell', 'what does it cost' (first_cost/unit_cost), 'who supplies it', 'what is on hand' and the stock-coverage columns.
|
| 147 |
-
β Revenue and unit columns here are WINDOWED SNAPSHOTS (rev_ytd, qty_ltm); for any other window, or for line grain, go to sales_lines.
|
|
|
|
| 1 |
+
# ββ W33-T49 (owner item 13) β an ENTITY topic: the semantic layer can finally see
|
| 2 |
+
# the databases a person actually opens, not only the line and document grains.
|
| 3 |
+
#
|
| 4 |
+
# β THE `fields:` BLOCK IS GENERATED FROM THE GRID'S OWN FIELD CONTRACT
|
| 5 |
+
# and is held to it by `verify_query.py::section_entity_topics`. Do not hand-edit a
|
| 6 |
+
# field row: add the column to the grid contract and re-emit, or the agent is being
|
| 7 |
+
# trained on a schema the product does not have.
|
| 8 |
+
key: odoo_products
|
| 9 |
+
label: "Odoo products"
|
| 10 |
+
entity: product.product
|
| 11 |
+
# The database this topic DESCRIBES β the same store key the nav opens (W33-T46).
|
| 12 |
+
grid: product_data
|
| 13 |
+
subject: "odoo:product.product"
|
| 14 |
+
grain: "one row per SKU in the active catalogue β a CATALOGUE, not a dated event stream"
|
| 15 |
+
scope:
|
| 16 |
+
population: "every ACTIVE product, sold or not (measured 5,862). Deliberately not the sales-window universe: that lands at 2,717 and hides ~2,550 SKUs that have never sold in wholesale scope"
|
| 17 |
+
identity: "the SKU code (`default_code`). β 33 active products carry NO code and are keyed `pid:<odoo product id>` instead, so every active product has exactly one row and none is dropped"
|
| 18 |
+
merged: "W33-T44 retired `ut_odoo_products`, which presented this same subject on Odoo's product_id. This topic describes the SURVIVING database, `product_data`"
|
| 19 |
+
no_date: "a catalogue has no date dimension; ask sales_lines for movement"
|
| 20 |
+
store:
|
| 21 |
+
table: product_product
|
| 22 |
+
alias: pp
|
| 23 |
+
# NO date_col β a registry is not a dated event stream. Stated rather than
|
| 24 |
+
# omitted, so its absence reads as a fact and not as an unfinished file.
|
| 25 |
+
dims:
|
| 26 |
+
category: {label: "Category"}
|
| 27 |
+
supplier: {label: "Supplier"}
|
| 28 |
+
|
| 29 |
+
# key / label / type / kind, derived from the grid contract. `kind` says where the
|
| 30 |
+
# value COMES FROM: a `data` column is stored on the row; a `rollup` is computed
|
| 31 |
+
# from another topic; a `link` points at another database.
|
| 32 |
+
fields:
|
| 33 |
+
- key: code
|
| 34 |
+
label: "SKU"
|
| 35 |
+
type: text
|
| 36 |
+
kind: data
|
| 37 |
+
means: "The SKU code β the product's real business key. `pid` is a stable CRC32 of it because the grid keys on an integer."
|
| 38 |
+
- key: product
|
| 39 |
+
label: "Product"
|
| 40 |
+
type: text
|
| 41 |
+
kind: data
|
| 42 |
+
means: "Product name as it appears in Odoo."
|
| 43 |
+
- key: category
|
| 44 |
+
label: "Category"
|
| 45 |
+
type: select
|
| 46 |
+
kind: data
|
| 47 |
+
means: "Product category; '(uncategorized)' when Odoo carries none."
|
| 48 |
+
- key: supplier
|
| 49 |
+
label: "Supplier"
|
| 50 |
+
type: text
|
| 51 |
+
kind: data
|
| 52 |
+
means: "Who makes it. Editable here and shared with everyone in the workspace; seeded from the inventory mastersheet."
|
| 53 |
+
- key: origin_country
|
| 54 |
+
label: "Country"
|
| 55 |
+
type: text
|
| 56 |
+
kind: data
|
| 57 |
+
means: "Country of origin. Editable here and shared with everyone; seeded from the inventory mastersheet."
|
| 58 |
+
- key: lead_days
|
| 59 |
+
label: "Lead time (days)"
|
| 60 |
+
type: int
|
| 61 |
+
kind: data
|
| 62 |
+
means: "Order-to-arrival days for this supplier. Drives the buy signal. Editable and shared with everyone."
|
| 63 |
+
- key: first_cost
|
| 64 |
+
label: "First cost"
|
| 65 |
+
type: currency
|
| 66 |
+
kind: data
|
| 67 |
+
means: "Quoted unit cost at origin, before freight and duty. Editable and shared with everyone."
|
| 68 |
+
- key: price_fisch
|
| 69 |
+
label: "Fisch price"
|
| 70 |
+
type: currency
|
| 71 |
+
kind: data
|
| 72 |
+
means: "Fisch pricelist price for this SKU. Blank when that list prices it nowhere."
|
| 73 |
+
- key: price_royal_1
|
| 74 |
+
label: "Royal 1 price"
|
| 75 |
+
type: currency
|
| 76 |
+
kind: data
|
| 77 |
+
means: "Royal 1 pricelist price for this SKU. Blank when that list prices it nowhere."
|
| 78 |
+
- key: price_royal_2
|
| 79 |
+
label: "Royal 2 price"
|
| 80 |
+
type: currency
|
| 81 |
+
kind: data
|
| 82 |
+
means: "Royal 2 pricelist price for this SKU. Blank when that list prices it nowhere."
|
| 83 |
+
- key: rev_ytd
|
| 84 |
+
label: "Revenue YTD"
|
| 85 |
+
type: currency
|
| 86 |
+
kind: data
|
| 87 |
+
means: "Year-to-date revenue for this SKU, BU-scoped when the caller is."
|
| 88 |
+
- key: rev_ly
|
| 89 |
+
label: "Revenue LY"
|
| 90 |
+
type: currency
|
| 91 |
+
kind: data
|
| 92 |
+
means: "Same period last year β seasonal wholesale compares like for like."
|
| 93 |
+
- key: yoy_pct
|
| 94 |
+
label: "YoY %"
|
| 95 |
+
type: pct
|
| 96 |
+
kind: data
|
| 97 |
+
means: "Year-over-year change; null when last year was zero (a ratio to zero is not a number)."
|
| 98 |
+
- key: qty_ytd
|
| 99 |
+
label: "Units YTD"
|
| 100 |
+
type: int
|
| 101 |
+
kind: data
|
| 102 |
+
means: "Units sold year to date."
|
| 103 |
+
- key: orders_ytd
|
| 104 |
+
label: "Orders YTD"
|
| 105 |
+
type: int
|
| 106 |
+
kind: data
|
| 107 |
+
means: "Distinct orders containing this SKU, year to date."
|
| 108 |
+
- key: on_hand
|
| 109 |
+
label: "On hand"
|
| 110 |
+
type: int
|
| 111 |
+
kind: data
|
| 112 |
+
means: "Units in stock. CONSOLIDATED β one physical warehouse, not brand-tagged, so this column is ABSENT for a BU-scoped caller rather than silently company-wide."
|
| 113 |
+
- key: unit_cost
|
| 114 |
+
label: "Unit cost"
|
| 115 |
+
type: currency
|
| 116 |
+
kind: data
|
| 117 |
+
means: "Inventory unit cost. Consolidated; absent for a BU-scoped caller."
|
| 118 |
+
- key: inv_value
|
| 119 |
+
label: "Stock value"
|
| 120 |
+
type: currency
|
| 121 |
+
kind: data
|
| 122 |
+
means: "On-hand value at cost. Consolidated; absent for a BU-scoped caller."
|
| 123 |
+
- key: qty_ltm
|
| 124 |
+
label: "Units LTM"
|
| 125 |
+
type: int
|
| 126 |
+
kind: data
|
| 127 |
+
means: "Units sold in the last twelve months. Consolidated; absent for a BU-scoped caller."
|
| 128 |
+
- key: dos
|
| 129 |
+
label: "Days of supply"
|
| 130 |
+
type: int
|
| 131 |
+
kind: data
|
| 132 |
+
means: "Days of supply at the LTM rate; null means it never sells through. Consolidated; absent for a BU-scoped caller."
|
| 133 |
+
- key: cover_gap_d
|
| 134 |
+
label: "Cover gap (days)"
|
| 135 |
+
type: int
|
| 136 |
+
kind: data
|
| 137 |
+
means: "Days of supply minus lead time. Negative means it runs out before a reorder lands."
|
| 138 |
+
- key: stock_bucket
|
| 139 |
+
label: "Stock status"
|
| 140 |
+
type: select
|
| 141 |
+
kind: data
|
| 142 |
+
means: "Dead / excess / healthy bucket from the inventory module. Consolidated; absent for a BU-scoped caller."
|
| 143 |
+
|
| 144 |
+
ai_context: >
|
| 145 |
+
The product CATALOGUE β the database labelled 'Odoo products'.
|
| 146 |
+
Use it for 'what do we sell', 'what does it cost' (first_cost/unit_cost), 'who supplies it', 'what is on hand' and the stock-coverage columns.
|
| 147 |
+
β Revenue and unit columns here are WINDOWED SNAPSHOTS (rev_ytd, qty_ltm); for any other window, or for line grain, go to sales_lines.
|
platform/model/topics/odoo_vendors.yml
CHANGED
|
@@ -1,103 +1,103 @@
|
|
| 1 |
-
# ββ W33-T49 (owner item 13) β an ENTITY topic: the semantic layer can finally see
|
| 2 |
-
# the databases a person actually opens, not only the line and document grains.
|
| 3 |
-
#
|
| 4 |
-
# β THE `fields:` BLOCK IS GENERATED FROM THE GRID'S OWN FIELD CONTRACT
|
| 5 |
-
# and is held to it by `verify_query.py::section_entity_topics`. Do not hand-edit a
|
| 6 |
-
# field row: add the column to the grid contract and re-emit, or the agent is being
|
| 7 |
-
# trained on a schema the product does not have.
|
| 8 |
-
key: odoo_vendors
|
| 9 |
-
label: "Odoo vendors"
|
| 10 |
-
entity: res.partner
|
| 11 |
-
# The database this topic DESCRIBES β the same store key the nav opens (W33-T46).
|
| 12 |
-
grid: ut_odoo_vendors
|
| 13 |
-
subject: "odoo:res.partner.vendor"
|
| 14 |
-
grain: "one row per partner we have posted a vendor bill to"
|
| 15 |
-
scope:
|
| 16 |
-
population: "DERIVED FROM THE BILLS β a partner with no posted vendor bill has no payable history to show, so the bill is the population"
|
| 17 |
-
asymmetry: "deliberately unlike odoo_customers, which is NOT derived from its documents; there is no second document universe for vendors"
|
| 18 |
-
overlap: "measured: only 9 of the vendors also appear in the customer population, so a vendor is not a customer row"
|
| 19 |
-
store:
|
| 20 |
-
table: res_partner
|
| 21 |
-
alias: p
|
| 22 |
-
# NO date_col β a registry is not a dated event stream. Stated rather than
|
| 23 |
-
# omitted, so its absence reads as a fact and not as an unfinished file.
|
| 24 |
-
dims:
|
| 25 |
-
country: {label: "Country"}
|
| 26 |
-
|
| 27 |
-
# key / label / type / kind, derived from the grid contract. `kind` says where the
|
| 28 |
-
# value COMES FROM: a `data` column is stored on the row; a `rollup` is computed
|
| 29 |
-
# from another topic; a `link` points at another database.
|
| 30 |
-
fields:
|
| 31 |
-
- key: vendor
|
| 32 |
-
label: "Vendor"
|
| 33 |
-
type: text
|
| 34 |
-
kind: data
|
| 35 |
-
- key: odoo_id
|
| 36 |
-
label: "Odoo ID"
|
| 37 |
-
type: int
|
| 38 |
-
kind: data
|
| 39 |
-
means: "The `res.partner` id. Also this row's id."
|
| 40 |
-
- key: vendor_id
|
| 41 |
-
label: "Odoo vendor id"
|
| 42 |
-
type: int
|
| 43 |
-
kind: data
|
| 44 |
-
- key: country
|
| 45 |
-
label: "Country"
|
| 46 |
-
type: text
|
| 47 |
-
kind: data
|
| 48 |
-
- key: email
|
| 49 |
-
label: "Email"
|
| 50 |
-
type: text
|
| 51 |
-
kind: data
|
| 52 |
-
- key: phone
|
| 53 |
-
label: "Phone"
|
| 54 |
-
type: text
|
| 55 |
-
kind: data
|
| 56 |
-
- key: mobile
|
| 57 |
-
label: "Mobile"
|
| 58 |
-
type: text
|
| 59 |
-
kind: data
|
| 60 |
-
- key: vat
|
| 61 |
-
label: "Tax ID"
|
| 62 |
-
type: text
|
| 63 |
-
kind: data
|
| 64 |
-
means: "Odoo `vat` β the vendor's tax/VAT registration number."
|
| 65 |
-
- key: vendor_ref
|
| 66 |
-
label: "Vendor reference"
|
| 67 |
-
type: text
|
| 68 |
-
kind: data
|
| 69 |
-
means: "Odoo `res.partner.ref` β our internal reference for this vendor."
|
| 70 |
-
- key: website
|
| 71 |
-
label: "Website"
|
| 72 |
-
type: url
|
| 73 |
-
kind: data
|
| 74 |
-
- key: street
|
| 75 |
-
label: "Street"
|
| 76 |
-
type: text
|
| 77 |
-
kind: data
|
| 78 |
-
- key: street2
|
| 79 |
-
label: "Street 2"
|
| 80 |
-
type: text
|
| 81 |
-
kind: data
|
| 82 |
-
- key: city
|
| 83 |
-
label: "City"
|
| 84 |
-
type: text
|
| 85 |
-
kind: data
|
| 86 |
-
- key: zip
|
| 87 |
-
label: "ZIP"
|
| 88 |
-
type: text
|
| 89 |
-
kind: data
|
| 90 |
-
- key: bills
|
| 91 |
-
label: "Bills"
|
| 92 |
-
type: link
|
| 93 |
-
kind: link
|
| 94 |
-
to_grid: ut_odoo_bills
|
| 95 |
-
- key: refreshed
|
| 96 |
-
label: "Refreshed"
|
| 97 |
-
type: date
|
| 98 |
-
kind: data
|
| 99 |
-
means: "When this row was last reconciled against Odoo."
|
| 100 |
-
|
| 101 |
-
ai_context: >
|
| 102 |
-
The vendor/supplier registry, with contact and tax identity (W33-T48 widened it from 5 columns to 15 after a census found 76 populated fields on the underlying partner)
|
| 103 |
-
β The contact columns are SPARSE by nature β measured 65/393 with an email, 13/393 with a tax id β so 'blank' means Odoo has no value, never that the sync failed.
|
|
|
|
| 1 |
+
# ββ W33-T49 (owner item 13) β an ENTITY topic: the semantic layer can finally see
|
| 2 |
+
# the databases a person actually opens, not only the line and document grains.
|
| 3 |
+
#
|
| 4 |
+
# β THE `fields:` BLOCK IS GENERATED FROM THE GRID'S OWN FIELD CONTRACT
|
| 5 |
+
# and is held to it by `verify_query.py::section_entity_topics`. Do not hand-edit a
|
| 6 |
+
# field row: add the column to the grid contract and re-emit, or the agent is being
|
| 7 |
+
# trained on a schema the product does not have.
|
| 8 |
+
key: odoo_vendors
|
| 9 |
+
label: "Odoo vendors"
|
| 10 |
+
entity: res.partner
|
| 11 |
+
# The database this topic DESCRIBES β the same store key the nav opens (W33-T46).
|
| 12 |
+
grid: ut_odoo_vendors
|
| 13 |
+
subject: "odoo:res.partner.vendor"
|
| 14 |
+
grain: "one row per partner we have posted a vendor bill to"
|
| 15 |
+
scope:
|
| 16 |
+
population: "DERIVED FROM THE BILLS β a partner with no posted vendor bill has no payable history to show, so the bill is the population"
|
| 17 |
+
asymmetry: "deliberately unlike odoo_customers, which is NOT derived from its documents; there is no second document universe for vendors"
|
| 18 |
+
overlap: "measured: only 9 of the vendors also appear in the customer population, so a vendor is not a customer row"
|
| 19 |
+
store:
|
| 20 |
+
table: res_partner
|
| 21 |
+
alias: p
|
| 22 |
+
# NO date_col β a registry is not a dated event stream. Stated rather than
|
| 23 |
+
# omitted, so its absence reads as a fact and not as an unfinished file.
|
| 24 |
+
dims:
|
| 25 |
+
country: {label: "Country"}
|
| 26 |
+
|
| 27 |
+
# key / label / type / kind, derived from the grid contract. `kind` says where the
|
| 28 |
+
# value COMES FROM: a `data` column is stored on the row; a `rollup` is computed
|
| 29 |
+
# from another topic; a `link` points at another database.
|
| 30 |
+
fields:
|
| 31 |
+
- key: vendor
|
| 32 |
+
label: "Vendor"
|
| 33 |
+
type: text
|
| 34 |
+
kind: data
|
| 35 |
+
- key: odoo_id
|
| 36 |
+
label: "Odoo ID"
|
| 37 |
+
type: int
|
| 38 |
+
kind: data
|
| 39 |
+
means: "The `res.partner` id. Also this row's id."
|
| 40 |
+
- key: vendor_id
|
| 41 |
+
label: "Odoo vendor id"
|
| 42 |
+
type: int
|
| 43 |
+
kind: data
|
| 44 |
+
- key: country
|
| 45 |
+
label: "Country"
|
| 46 |
+
type: text
|
| 47 |
+
kind: data
|
| 48 |
+
- key: email
|
| 49 |
+
label: "Email"
|
| 50 |
+
type: text
|
| 51 |
+
kind: data
|
| 52 |
+
- key: phone
|
| 53 |
+
label: "Phone"
|
| 54 |
+
type: text
|
| 55 |
+
kind: data
|
| 56 |
+
- key: mobile
|
| 57 |
+
label: "Mobile"
|
| 58 |
+
type: text
|
| 59 |
+
kind: data
|
| 60 |
+
- key: vat
|
| 61 |
+
label: "Tax ID"
|
| 62 |
+
type: text
|
| 63 |
+
kind: data
|
| 64 |
+
means: "Odoo `vat` β the vendor's tax/VAT registration number."
|
| 65 |
+
- key: vendor_ref
|
| 66 |
+
label: "Vendor reference"
|
| 67 |
+
type: text
|
| 68 |
+
kind: data
|
| 69 |
+
means: "Odoo `res.partner.ref` β our internal reference for this vendor."
|
| 70 |
+
- key: website
|
| 71 |
+
label: "Website"
|
| 72 |
+
type: url
|
| 73 |
+
kind: data
|
| 74 |
+
- key: street
|
| 75 |
+
label: "Street"
|
| 76 |
+
type: text
|
| 77 |
+
kind: data
|
| 78 |
+
- key: street2
|
| 79 |
+
label: "Street 2"
|
| 80 |
+
type: text
|
| 81 |
+
kind: data
|
| 82 |
+
- key: city
|
| 83 |
+
label: "City"
|
| 84 |
+
type: text
|
| 85 |
+
kind: data
|
| 86 |
+
- key: zip
|
| 87 |
+
label: "ZIP"
|
| 88 |
+
type: text
|
| 89 |
+
kind: data
|
| 90 |
+
- key: bills
|
| 91 |
+
label: "Bills"
|
| 92 |
+
type: link
|
| 93 |
+
kind: link
|
| 94 |
+
to_grid: ut_odoo_bills
|
| 95 |
+
- key: refreshed
|
| 96 |
+
label: "Refreshed"
|
| 97 |
+
type: date
|
| 98 |
+
kind: data
|
| 99 |
+
means: "When this row was last reconciled against Odoo."
|
| 100 |
+
|
| 101 |
+
ai_context: >
|
| 102 |
+
The vendor/supplier registry, with contact and tax identity (W33-T48 widened it from 5 columns to 15 after a census found 76 populated fields on the underlying partner)
|
| 103 |
+
β The contact columns are SPARSE by nature β measured 65/393 with an email, 13/393 with a tax id β so 'blank' means Odoo has no value, never that the sync failed.
|
platform/modules/collections_send.py
CHANGED
|
@@ -180,16 +180,20 @@ def load_collection_list(odoo):
|
|
| 180 |
|
| 181 |
|
| 182 |
# --------------------------------------------------------------- statement email
|
| 183 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 184 |
DEFAULT_INTRO = (
|
| 185 |
'Dear {customer},<br><br>'
|
| 186 |
'Please find below your current statement of account with {company}. '
|
| 187 |
'According to our records, the following invoices remain open:'
|
| 188 |
)
|
| 189 |
DEFAULT_FOOTER = (
|
| 190 |
-
'If you have already sent payment, please disregard this notice
|
| 191 |
'For any questions about an invoice, simply reply to this email.<br><br>'
|
| 192 |
-
'Thank you for your business,<br>{company}
|
| 193 |
)
|
| 194 |
|
| 195 |
|
|
|
|
| 180 |
|
| 181 |
|
| 182 |
# --------------------------------------------------------------- statement email
|
| 183 |
+
# β STANDING RULE 2, AND THIS IS THE ONE STRING IN THE PRODUCT THAT LEAVES THE BUILDING.
|
| 184 |
+
# Every other finding `web_prose` reports is copy on a screen somebody here opens; this is the
|
| 185 |
+
# SUBJECT LINE of mail queued to a real debtor, over the single sanctioned Odoo write. It read
|
| 186 |
+
# 'Statement of Account (em dash) {company} (em dash) {month}' until W36-T42.
|
| 187 |
+
DEFAULT_SUBJECT = 'Statement of Account from {company}, {month}'
|
| 188 |
DEFAULT_INTRO = (
|
| 189 |
'Dear {customer},<br><br>'
|
| 190 |
'Please find below your current statement of account with {company}. '
|
| 191 |
'According to our records, the following invoices remain open:'
|
| 192 |
)
|
| 193 |
DEFAULT_FOOTER = (
|
| 194 |
+
'If you have already sent payment, please disregard this notice, and thank you. '
|
| 195 |
'For any questions about an invoice, simply reply to this email.<br><br>'
|
| 196 |
+
'Thank you for your business,<br>{company}<br>Accounts Receivable'
|
| 197 |
)
|
| 198 |
|
| 199 |
|
platform/modules/product_data.py
CHANGED
|
@@ -1,804 +1,804 @@
|
|
| 1 |
-
"""modules/product_data.py β the PRODUCT table's pool (wave 15 item 9/10, contract C-TOPIC).
|
| 2 |
-
|
| 3 |
-
The second object on the table-page factory: same grid, same engine, same permission wall β the
|
| 4 |
-
only difference is the field schema and the identity. `modules/customer_data.pool()` is the
|
| 5 |
-
template and this deliberately mirrors its signature and its row shape.
|
| 6 |
-
|
| 7 |
-
β THREE DECISIONS THIS FILE HAD TO MAKE, EACH RECORDED BECAUSE A LATER READER WILL WONDER.
|
| 8 |
-
|
| 9 |
-
1. **THE IDENTITY IS A SKU CODE, WHICH IS A STRING, AND THE GRID WANTS AN INTEGER `pid`.**
|
| 10 |
-
Cohort membership, `allowed_pids`, the measure channel and `rows_from_pool` all key on an
|
| 11 |
-
integer. So each row carries BOTH: `pid` (a stable CRC32 of the code, so the same SKU gets the
|
| 12 |
-
same id on every pull and across processes β never an enumeration index, which would reshuffle
|
| 13 |
-
whenever the catalogue changes) and `code`, the real business key, as a visible column.
|
| 14 |
-
`_assert_no_pid_collision` fails the BUILD rather than the read: two SKUs sharing a pid would
|
| 15 |
-
silently merge in every downstream set operation, and a loud build failure is the only version
|
| 16 |
-
of that anyone would notice.
|
| 17 |
-
|
| 18 |
-
2. **THE SCOPE RULE β inventory columns are CONSOLIDATED and are therefore OMITTED for a
|
| 19 |
-
BU-scoped caller.** `products.directory(t, team_id)` is brand-shaped; `inventory.sku_inventory`
|
| 20 |
-
is explicitly NOT (its own docstring: "on-hand stock is one physical warehouse, not
|
| 21 |
-
brand-tagged"). Joining them for a Fisch-only user would put BU-shaped revenue beside
|
| 22 |
-
company-wide stock IN THE SAME ROW β the mixed-scope value defect wave 15's amendment 3 exists
|
| 23 |
-
for, arriving through a different door. The honest options were "omit" or "label the columns
|
| 24 |
-
company-wide"; omit is the fail-closed one, and a column that is absent asks a question,
|
| 25 |
-
whereas a column that is silently company-wide answers one wrongly.
|
| 26 |
-
|
| 27 |
-
3. **WHAT IS NOT HERE, NAMED RATHER THAN QUIETLY MISSING.** R4 lists ~20 fields. Vendor and
|
| 28 |
-
COUNTRY are not in Odoo at all β they live in the inventory WORKBOOK
|
| 29 |
-
([[odoo-vendor-country-origin]]) and need a loader this module deliberately does not invent.
|
| 30 |
-
Margin % comes from `modules/pricing.table`, which is a heavier build (channel-rate cost
|
| 31 |
-
allocation) and is left for the wave that needs it. `validate()` reconciles what SHIPS; it
|
| 32 |
-
does not pretend to cover columns that are absent.
|
| 33 |
-
|
| 34 |
-
4. **THE POOL IS CATALOGUE-FIRST, WITH REVENUE LEFT-JOINED** (wave 29, owner item 22 / R12,
|
| 35 |
-
2026-08-11). It was `for r in products.directory(...)` β and `directory()`'s row set IS the
|
| 36 |
-
union of two revenue read-groups, so **a SKU that never sold could not exist** and the grid
|
| 37 |
-
showed **2,717** of **5,875** active products. Three things that will be re-derived otherwise:
|
| 38 |
-
|
| 39 |
-
Β· β THE CAUSE IS A JOIN, NOT A LIMIT. No row cap exists on this path. Removing the date
|
| 40 |
-
window from `directory()` would ALSO be wrong twice: it breaks four SKU-health metrics
|
| 41 |
-
that legitimately want a sales window, and it lands at 3,327 (all-time-sold), because
|
| 42 |
-
~2,550 active SKUs have never sold in wholesale scope at all.
|
| 43 |
-
Β· β REVENUE ON A NEVER-SOLD ROW IS **BLANK, NOT $0** (R12). A row that appears in the
|
| 44 |
-
revenue universe carries measured numbers INCLUDING a real 0.0 β it sold last year and
|
| 45 |
-
not this one, and that zero is a measurement. A row that appears in NO revenue read
|
| 46 |
-
carries `None`, which the wire keeps (`aios_grid._round` passes None through) and the
|
| 47 |
-
client renders as an empty cell. Blank is an admission; zero is a measurement.
|
| 48 |
-
Β· β A BU-SCOPED CALLER NOW SEES THE WHOLE CATALOGUE, with ITS OWN revenue and blanks where
|
| 49 |
-
that BU never sold. That is not decision 2's mixed-scope defect: the catalogue is the ROW
|
| 50 |
-
UNIVERSE, not a company-wide VALUE sitting beside a BU-shaped one. `product.product`
|
| 51 |
-
carries no team, so a catalogue cannot be BU-shaped at all β which is exactly why the two
|
| 52 |
-
sources are JOINED rather than merged.
|
| 53 |
-
"""
|
| 54 |
-
import json
|
| 55 |
-
import zlib
|
| 56 |
-
from pathlib import Path
|
| 57 |
-
|
| 58 |
-
import core.odoo as O
|
| 59 |
-
import core.periods as P
|
| 60 |
-
import core.shared_overlay as shared_overlay
|
| 61 |
-
import core.table_store as table_store
|
| 62 |
-
import modules.products as products
|
| 63 |
-
|
| 64 |
-
#: POOL-ROW KEYS that exist ONLY on a consolidated pull. See decision 2.
|
| 65 |
-
#: `cover_gap_d` / `buy_now` (wave 17) join it because both are computed FROM `dos`, and a buy
|
| 66 |
-
#: signal built on a stock number the caller cannot see would be a recommendation nobody could
|
| 67 |
-
#: check ([[no-unverifiable-aggregates]]).
|
| 68 |
-
#:
|
| 69 |
-
#: β ROW KEYS, NOT FIELDS, and the distinction started mattering on 2026-08-03. Two readers:
|
| 70 |
-
#: `verify_perm_scope` asserts the shape of a POOL ROW against this, and `routes_products`
|
| 71 |
-
#: narrows the FIELD list with it. Since owner item 5 `buy_now` is a row key with no Field β
|
| 72 |
-
#: computed as `validate()`'s oracle, projected onto no wire β so it belongs here for the first
|
| 73 |
-
#: reader and is inert for the second. Removing it would stop the scope gate proving that a
|
| 74 |
-
#: BU-scoped pull withholds it.
|
| 75 |
-
#:
|
| 76 |
-
#: ββ OWNER RULING 2026-08-11 β THIS SET IS NOW EMPTY, AND THAT IS THE POINT. Verbatim:
|
| 77 |
-
#: *"Just scope any inventory with Sales from Fisch, leave the rest."* Withholding the whole
|
| 78 |
-
#: inventory block from a BU-scoped reader meant a Fisch salesperson could not see whether
|
| 79 |
-
#: anything was IN STOCK, which is the first question they ask. The ruling splits the block by
|
| 80 |
-
#: what a business unit can actually shape:
|
| 81 |
-
#:
|
| 82 |
-
#: * `on_hand` / `unit_cost` / `inv_value` β ONE physical warehouse, no Fisch shelf and no
|
| 83 |
-
#: Royal shelf. Served UNSCOPED to everybody, identical in both pulls. ("leave the rest")
|
| 84 |
-
#: * `qty_ltm` / `dos` / `stock_bucket` / `cover_gap_d` / `buy_now` β all SALES-derived, so a
|
| 85 |
-
#: BU-scoped pull recomputes them from THAT unit's LTM units. ("scope any inventory with
|
| 86 |
-
#: Sales from Fisch")
|
| 87 |
-
#:
|
| 88 |
-
#: This deliberately relaxes decision 2's "never a company-wide value beside a BU-shaped one":
|
| 89 |
-
#: stock is not a value a BU can own, and a column that is absent for a Fisch reader asks a
|
| 90 |
-
#: question they cannot answer anywhere else in the product. `verify_perm_scope` no longer
|
| 91 |
-
#: asserts the columns are WITHHELD β it asserts the split above, which is a stronger claim and
|
| 92 |
-
#: cannot pass on an empty scoped payload (the shape the old rule and an outage share).
|
| 93 |
-
CONSOLIDATED_ONLY = ()
|
| 94 |
-
|
| 95 |
-
#: β POOL-ROW KEYS THAT DELIBERATELY HAVE NO FIELD β they must never reach a browser.
|
| 96 |
-
#:
|
| 97 |
-
#: `rows_from_pool` projects strictly through the field contract, so "no Field" already means
|
| 98 |
-
#: "no cell on the wire". This tuple makes that a CHECKED fact rather than a consequence nobody
|
| 99 |
-
#: is watching: `verify_perm_scope` asserts every member is absent from BOTH product contracts,
|
| 100 |
-
#: and that every OTHER `CONSOLIDATED_ONLY` key is present in the consolidated one.
|
| 101 |
-
#:
|
| 102 |
-
#: Written because the two tuples silently diverged the moment owner item 5 retired `buy_now`'s
|
| 103 |
-
#: Field while keeping its computation, and the gate β which was asserting a FIELD property from
|
| 104 |
-
#: a ROW-key list β went red with no way to tell a deliberate divergence from a dropped column.
|
| 105 |
-
UNSHIPPED_ROW_KEYS = ("buy_now",)
|
| 106 |
-
|
| 107 |
-
#: Wave 17 (owner item 13, ruling R3) β the SUPPLIER MASTER, from the curated mastersheet map
|
| 108 |
-
#: `procurement_suppliers.json` (3,963 SKUs; supplier on 3,624, lead time on 3,563). NOT Odoo:
|
| 109 |
-
#: the owner confirmed this data came from the Inventory System Mastersheet, which is why
|
| 110 |
-
#: `modules/product_data`'s header said Vendor/Country were "NOT HERE, named rather than
|
| 111 |
-
#: quietly missing" and needed "a loader this module deliberately does not invent". This is
|
| 112 |
-
#: that loader.
|
| 113 |
-
#:
|
| 114 |
-
#: β THESE ARE CONTRACT COLUMNS, NOT USER-CREATED FIELDS, AND THE REASON IS A PERMISSION FACT.
|
| 115 |
-
#: A user-created field and its values live in the PER-USER strata (`table_store.workspace`
|
| 116 |
-
#: reads `store.get(key)[username]`); only VIEWS are shared. So a shared "Buy list" view that
|
| 117 |
-
#: filtered on a user-created column would, for every OTHER account, name a column that does not
|
| 118 |
-
#: exist β and an unknown column is an INACTIVE condition in the tri-state engine, which IGNORES
|
| 119 |
-
#: it and therefore WIDENS. The buy list would silently show the whole catalogue to everyone but
|
| 120 |
-
#: its author. Contract columns are identical for every reader, so the view means one thing.
|
| 121 |
-
#:
|
| 122 |
-
#: ββ 2026-08-12 (W30-T36) β THE OWNER'S ASK IS NOW DELIVERED, AND THE PARAGRAPH ABOVE IS WHY IT
|
| 123 |
-
#: TOOK THREE WAVES. Owner: *"turn this Excel sheet into a User created Field that we can edit."*
|
| 124 |
-
#: It was parked because a per-user column silently WIDENS a shared view β not because editing was
|
| 125 |
-
#: hard. Wave 29's `core/shared_overlay.py` (whose header quotes this very comment) is the stratum
|
| 126 |
-
#: that removes the objection: **one value per (row, column) for the whole tenant**, so the column
|
| 127 |
-
#: still means ONE thing to every reader and a shared view still filters honestly.
|
| 128 |
-
#: β The four columns below are now `source: "overlay"` + `shared: true` in the canonical
|
| 129 |
-
#: contract, their values live in `<TABLE_KEY>__shared`, and the master map is what SEEDS an
|
| 130 |
-
#: unedited cell rather than what freezes it. See `SHARED_KEYS` and `_ProductTableStore`.
|
| 131 |
-
_SUPPLIER_MAP_PATH = Path(__file__).resolve().parent.parent / "procurement_suppliers.json"
|
| 132 |
-
_SUPPLIER_CACHE = {}
|
| 133 |
-
|
| 134 |
-
|
| 135 |
-
def supplier_master():
|
| 136 |
-
"""`{code: {supplier, lead_days, origin_country, first_cost}}`, read once per process.
|
| 137 |
-
|
| 138 |
-
Degrades to `{}` when the file is unreadable, matching `_inventory_by_code`: a product table
|
| 139 |
-
that will not render because a master map is missing is a worse failure than one with blank
|
| 140 |
-
supplier columns.
|
| 141 |
-
"""
|
| 142 |
-
if _SUPPLIER_CACHE:
|
| 143 |
-
return _SUPPLIER_CACHE
|
| 144 |
-
try:
|
| 145 |
-
raw = json.loads(_SUPPLIER_MAP_PATH.read_text(encoding="utf-8"))
|
| 146 |
-
except Exception:
|
| 147 |
-
return {}
|
| 148 |
-
for code, meta in (raw or {}).items():
|
| 149 |
-
if not isinstance(meta, dict):
|
| 150 |
-
continue
|
| 151 |
-
lead = meta.get("lead")
|
| 152 |
-
_SUPPLIER_CACHE[str(code)] = {
|
| 153 |
-
"supplier": (meta.get("vendor") or "") or None,
|
| 154 |
-
"lead_days": int(lead) if isinstance(lead, (int, float)) else None,
|
| 155 |
-
"origin_country": (meta.get("country") or "") or None,
|
| 156 |
-
"first_cost": meta.get("first_cost"),
|
| 157 |
-
}
|
| 158 |
-
return _SUPPLIER_CACHE
|
| 159 |
-
|
| 160 |
-
#: Wave 16 C-TOPIC β the PRODUCT table's OWN workspace bucket. β Never the customer one:
|
| 161 |
-
#: product pids are CRC32 hashes of SKU codes and customer pids are Odoo partner ids, so in a
|
| 162 |
-
#: SHARED overlay bucket a hash collision would silently write a product note onto somebody's
|
| 163 |
-
#: customer (or the reverse). Separate store keys make that structurally impossible, which is
|
| 164 |
-
#: the whole reason the table-page factory exists ("a new table object gets its own
|
| 165 |
-
#: table_store.make('<its>_table_workspace')").
|
| 166 |
-
TABLE_KEY = 'product_table_workspace'
|
| 167 |
-
|
| 168 |
-
_GRID_FIELDS_PATH = Path(__file__).resolve().parent.parent / 'aios_grid_fields.json'
|
| 169 |
-
_SHARED_KEYS = None
|
| 170 |
-
|
| 171 |
-
|
| 172 |
-
def SHARED_KEYS():
|
| 173 |
-
"""The product columns whose values are TENANT-WIDE β derived from the canonical contract's
|
| 174 |
-
own `shared: true`, never typed out a second time.
|
| 175 |
-
|
| 176 |
-
β IT DELIBERATELY DOES NOT SWALLOW A READ FAILURE. Degrading to `()` would send a shared
|
| 177 |
-
write into the per-user stratum with nothing going wrong anywhere β the widening defect
|
| 178 |
-
reappearing silently, which is the one outcome this whole mechanism exists to prevent. If the
|
| 179 |
-
canonical contract is unreadable the product grid cannot render at all (`pd_fields` parses the
|
| 180 |
-
same file with no guard), so a raise here costs nothing that was still working.
|
| 181 |
-
"""
|
| 182 |
-
global _SHARED_KEYS
|
| 183 |
-
if _SHARED_KEYS is None:
|
| 184 |
-
doc = json.loads(_GRID_FIELDS_PATH.read_text(encoding='utf-8'))
|
| 185 |
-
_SHARED_KEYS = tuple(f['key'] for f in (doc.get('product_data') or {}).get('fields') or []
|
| 186 |
-
if f.get('shared'))
|
| 187 |
-
return _SHARED_KEYS
|
| 188 |
-
|
| 189 |
-
|
| 190 |
-
class _ProductTableStore(table_store.TableStore):
|
| 191 |
-
"""The product workspace, with the SHARED columns routed to the tenant-wide stratum.
|
| 192 |
-
|
| 193 |
-
ββ THIS SUBCLASS IS THE WHOLE OF W30-T36's WRITE PATH, AND THE REASON IT LIVES HERE RATHER
|
| 194 |
-
THAN AT A ROUTE IS THAT **THE BROWSER NEVER CALLS `PATCH /products/{pid}`** β measured, zero
|
| 195 |
-
call sites in `aios-web/web/src`. A cell edit travels `POST /grid/events` β `grid_events.
|
| 196 |
-
handle_one` β `_tops(ctx).patch_overlay(...)`, and `_tops` returns `ctx.table`, which
|
| 197 |
-
`routes_grid._ctx` sets to `TABLE_OPS` for the product scope. So this object IS the seam both
|
| 198 |
-
doors pass through; intercepting at either route would have left the other one writing a
|
| 199 |
-
per-user value that only its author could see.
|
| 200 |
-
|
| 201 |
-
β `st=self.st`, NEVER the module default. The shared stratum must resolve to the SAME store
|
| 202 |
-
handle as the per-user one it sits beside β `_tops`' own comment explains that a split, where
|
| 203 |
-
one side is tenant-scoped and the other is not, is worse than a stated residency error because
|
| 204 |
-
a user's value would vanish the moment they saved it. Reading `self.st` means both strata move
|
| 205 |
-
together the day that singleton gains a tenant handle.
|
| 206 |
-
"""
|
| 207 |
-
|
| 208 |
-
def patch_overlay(self, username, pid, updates):
|
| 209 |
-
clean = dict(updates or {})
|
| 210 |
-
if not clean:
|
| 211 |
-
return
|
| 212 |
-
keys = set(SHARED_KEYS())
|
| 213 |
-
shared = {k: v for k, v in clean.items() if k in keys}
|
| 214 |
-
personal = {k: v for k, v in clean.items() if k not in keys}
|
| 215 |
-
if shared:
|
| 216 |
-
shared_overlay.put_cells(TABLE_KEY, pid, shared, st=self.st)
|
| 217 |
-
if personal:
|
| 218 |
-
super().patch_overlay(username, pid, personal)
|
| 219 |
-
|
| 220 |
-
|
| 221 |
-
TABLE_OPS = _ProductTableStore(TABLE_KEY)
|
| 222 |
-
|
| 223 |
-
|
| 224 |
-
def shared_cells(pids, st=None):
|
| 225 |
-
"""`{"<pid>": {key: value}}` for the SHARED columns of the rows named by `pids`.
|
| 226 |
-
|
| 227 |
-
β `pids` is required and positional all the way down β `shared_overlay.cells` refuses to serve
|
| 228 |
-
"everything" by design, and the caller here always holds an already-scoped pool.
|
| 229 |
-
"""
|
| 230 |
-
return shared_overlay.cells(TABLE_KEY, pids, st=st if st is not None else TABLE_OPS.st)
|
| 231 |
-
|
| 232 |
-
|
| 233 |
-
def sku_pid(code):
|
| 234 |
-
"""A stable integer id for a SKU code. CRC32, masked to 31 bits so it is always positive and
|
| 235 |
-
always JSON-safe. Stable across processes and pulls, which an enumeration index is not."""
|
| 236 |
-
return zlib.crc32(str(code).encode("utf-8")) & 0x7FFFFFFF
|
| 237 |
-
|
| 238 |
-
|
| 239 |
-
def _assert_no_pid_collision(rows):
|
| 240 |
-
"""Two SKUs sharing a pid would MERGE in every set operation downstream β cohort membership,
|
| 241 |
-
allowed_pids, the measure channel β and nothing would report it. Fail the build instead."""
|
| 242 |
-
seen = {}
|
| 243 |
-
for r in rows:
|
| 244 |
-
prior = seen.get(r["pid"])
|
| 245 |
-
if prior is not None and prior != r["code"]:
|
| 246 |
-
raise ValueError(
|
| 247 |
-
f"product_data: pid collision β {prior!r} and {r['code']!r} both hash to "
|
| 248 |
-
f"{r['pid']}. Downstream set operations would merge them silently; widen the id "
|
| 249 |
-
f"before shipping this catalogue.")
|
| 250 |
-
seen[r["pid"]] = r["code"]
|
| 251 |
-
|
| 252 |
-
|
| 253 |
-
def _inventory_by_code(t):
|
| 254 |
-
"""`{code: {...}}` from the inventory module, or `{}` if it cannot be read.
|
| 255 |
-
|
| 256 |
-
Degrades to empty rather than raising, matching `customer_data._pool_build`'s treatment of
|
| 257 |
-
its own slow families: a product table that will not render because inventory is momentarily
|
| 258 |
-
unreachable is a worse failure than one with blank stock columns.
|
| 259 |
-
"""
|
| 260 |
-
try:
|
| 261 |
-
import modules.inventory as inventory
|
| 262 |
-
return inventory.sku_inventory(t=t) or {}
|
| 263 |
-
except Exception:
|
| 264 |
-
return {}
|
| 265 |
-
|
| 266 |
-
|
| 267 |
-
def _bu_ltm_share(t, team_id):
|
| 268 |
-
"""`{code: 0.0..1.0}` β this unit's SHARE of the SKU's last-twelve-months units.
|
| 269 |
-
|
| 270 |
-
β OWNER 2026-08-11: *"scope any inventory with Sales from Fisch"*. The velocity half of the
|
| 271 |
-
inventory block has to be re-shaped by a BU fact, and this is that fact.
|
| 272 |
-
|
| 273 |
-
β A SHARE, NOT THE UNIT COUNT ITSELF β AND THE REASON IS A MEASURED IMPOSSIBILITY. My first
|
| 274 |
-
version returned `products._sku_rev(...)['qty']` and used it directly as the scoped `qty_ltm`,
|
| 275 |
-
leaving the consolidated column on `inventory.sku_inventory`'s own figure. Two readers of one
|
| 276 |
-
question ([[one-question-two-normalizers]]): they disagree, and on 5 SKUs of 5,871 the live
|
| 277 |
-
check found **Fisch's LTM units EXCEEDING the company's** β a subset larger than its superset,
|
| 278 |
-
which no reader could explain and no reconciliation could survive.
|
| 279 |
-
|
| 280 |
-
Both halves of the ratio come from the SAME read here, so the share is in [0, 1] by
|
| 281 |
-
construction and the scoped figure can never exceed the consolidated one. It also leaves the
|
| 282 |
-
consolidated column exactly as it was β the Inventory page and the Product grid still agree
|
| 283 |
-
about company-wide units, which is what "leave the rest" asked for.
|
| 284 |
-
|
| 285 |
-
`all_qty == 0` implies `bu_qty == 0` (same reader), so a share of 0 is the honest answer for
|
| 286 |
-
"this unit never sold it": `_bucket` turns that into 'No recent sales', not zero cover.
|
| 287 |
-
|
| 288 |
-
Degrades to `{}` on any failure, matching `_inventory_by_code` β and a missing code then takes
|
| 289 |
-
the `_DEFAULT_SHARE` below rather than a silent 0.
|
| 290 |
-
"""
|
| 291 |
-
try:
|
| 292 |
-
f, to = P.ltm(t)
|
| 293 |
-
allq = {code: (r.get('qty') or 0.0)
|
| 294 |
-
for code, r in (products._sku_rev(f, to, None) or {}).items()}
|
| 295 |
-
buq = {code: (r.get('qty') or 0.0)
|
| 296 |
-
for code, r in (products._sku_rev(f, to, team_id) or {}).items()}
|
| 297 |
-
except Exception:
|
| 298 |
-
return {}
|
| 299 |
-
out = {}
|
| 300 |
-
for code, total in allq.items():
|
| 301 |
-
out[code] = min(1.0, max(0.0, (buq.get(code, 0.0) / total))) if total > 0 else 0.0
|
| 302 |
-
return out
|
| 303 |
-
|
| 304 |
-
|
| 305 |
-
#: What a SKU absent from the LTM sales read is worth to a business unit. ZERO β it did not sell
|
| 306 |
-
#: in anybody's book over the window, so no unit can claim its velocity. Named rather than
|
| 307 |
-
#: inlined so the choice is visible: the alternative (1.0, "assume it is all ours") would print
|
| 308 |
-
#: company-wide cover on a BU grid, which is the mixed-scope defect this whole rule avoids.
|
| 309 |
-
_DEFAULT_SHARE = 0.0
|
| 310 |
-
|
| 311 |
-
|
| 312 |
-
def _rescope_inventory(e, share):
|
| 313 |
-
"""`(qty_ltm, dos, bucket)` recomputed for ONE business unit's sales rate.
|
| 314 |
-
|
| 315 |
-
The formulas are `inventory.sku_inventory`'s, applied to a BU-shaped numerator β NOT a second
|
| 316 |
-
idea of what days-of-supply means. `_bucket` is imported from there for the same reason: two
|
| 317 |
-
copies of a threshold table is how the Product grid and the Inventory page start disagreeing
|
| 318 |
-
about which SKUs are dead.
|
| 319 |
-
|
| 320 |
-
β `on_hand` is whatever the warehouse holds, unscoped β so a Fisch reader's `dos` answers
|
| 321 |
-
"how long does ALL our stock last at Fisch's rate", which is the question a Fisch salesperson
|
| 322 |
-
actually has. It is deliberately NOT a pro-rated share of the shelf: there is no such shelf,
|
| 323 |
-
and inventing one would put a number on screen that no Odoo query could reproduce.
|
| 324 |
-
"""
|
| 325 |
-
on_hand = e.get("on_hand")
|
| 326 |
-
if on_hand is None:
|
| 327 |
-
return None, None, None # no inventory row for this SKU: blank, never zero
|
| 328 |
-
qty = float(e.get("qty_ltm") or 0.0) * float(share or 0.0)
|
| 329 |
-
daily = qty / 365.0
|
| 330 |
-
if daily > 0:
|
| 331 |
-
dos_raw = on_hand / daily
|
| 332 |
-
else:
|
| 333 |
-
dos_raw = float('inf') if on_hand > 0 else 0.0
|
| 334 |
-
try:
|
| 335 |
-
import modules.inventory as inventory
|
| 336 |
-
bucket = inventory._bucket(dos_raw, on_hand, qty)
|
| 337 |
-
except Exception:
|
| 338 |
-
bucket = None
|
| 339 |
-
return qty, (None if dos_raw == float('inf') else round(float(dos_raw), 0)), bucket
|
| 340 |
-
|
| 341 |
-
|
| 342 |
-
def _catalogue_by_code():
|
| 343 |
-
"""`{code: {'product', 'category'}}` β the CATALOGUE universe this pool is built from.
|
| 344 |
-
|
| 345 |
-
β UNLIKE `_inventory_by_code`, THIS ONE RAISES, and the asymmetry is the whole point.
|
| 346 |
-
Inventory degrades to `{}` because a product table with blank stock columns beats one that
|
| 347 |
-
will not render. The CATALOGUE is not a column β it is the ROW SET. A catalogue read that
|
| 348 |
-
failed quietly would drop the grid straight back to the sold-only 2,717 with every gate
|
| 349 |
-
green and nothing on screen saying so, which is the defect this seam exists to end
|
| 350 |
-
([[gate-can-report-green-on-nothing]]). `routes_products._pool_for` already turns the raise
|
| 351 |
-
into a 503 that names the cause, so the loud failure has somewhere honest to land.
|
| 352 |
-
|
| 353 |
-
It exists as a `pd`-level function rather than an inline `products.catalogue()` call for the
|
| 354 |
-
same reason `_inventory_by_code` does: it is the seam `verify_perm_scope`'s section H stubs
|
| 355 |
-
to build a pool without Odoo.
|
| 356 |
-
"""
|
| 357 |
-
return products.catalogue()
|
| 358 |
-
|
| 359 |
-
|
| 360 |
-
def _pricelist_by_code():
|
| 361 |
-
"""`({code: {price_*: price}}, report)` from `products.pricelist_by_code`, or `({}, β¦)`.
|
| 362 |
-
|
| 363 |
-
A SEAM for the same two reasons `_inventory_by_code` is one: it degrades rather than raises
|
| 364 |
-
(these are columns, not the row set), and `verify_perm_scope`'s section H stubs it to build a
|
| 365 |
-
pool without Odoo. β Unstubbed there, section H would reach live Odoo through the back door
|
| 366 |
-
and the whole file would stop being runnable offline.
|
| 367 |
-
"""
|
| 368 |
-
try:
|
| 369 |
-
return products.pricelist_by_code()
|
| 370 |
-
except Exception as e:
|
| 371 |
-
return {}, {"error": f"{type(e).__name__}: {str(e)[:200]}"}
|
| 372 |
-
|
| 373 |
-
|
| 374 |
-
def pool(team_id=None, t=None):
|
| 375 |
-
"""One row per ACTIVE product β the PRODUCT analogue of `customer_data.pool`.
|
| 376 |
-
|
| 377 |
-
CATALOGUE-FIRST, REVENUE LEFT-JOINED (R12 β see decision 4 in the module header). The row set
|
| 378 |
-
is `products.catalogue()`; `products.directory()` supplies the revenue columns for the SKUs
|
| 379 |
-
that sold in its window and contributes NO rows of its own.
|
| 380 |
-
|
| 381 |
-
`team_id` shapes the revenue columns exactly as it does for customers (`products.directory`
|
| 382 |
-
passes it into `_sku_rev`), which is why `core.perm_scope.derive_pool_scope` must keep
|
| 383 |
-
driving it rather than a post-filter deciding the BU. It does NOT shape the row set: a
|
| 384 |
-
catalogue has no team.
|
| 385 |
-
"""
|
| 386 |
-
t = t or P.today()
|
| 387 |
-
consolidated = team_id is None
|
| 388 |
-
# β OWNER 2026-08-11: read inventory on EVERY pull, not just a consolidated one. The stock
|
| 389 |
-
# itself is company-wide; only the sales-derived half is re-scoped, by `_rescope_inventory`.
|
| 390 |
-
inv = _inventory_by_code(t)
|
| 391 |
-
bu_share = {} if consolidated else _bu_ltm_share(t, team_id)
|
| 392 |
-
sup = supplier_master()
|
| 393 |
-
prices, _price_report = _pricelist_by_code()
|
| 394 |
-
cat = _catalogue_by_code()
|
| 395 |
-
# The LEFT side of the join, indexed by the same code key. β A code here that the catalogue
|
| 396 |
-
# does not carry belongs to a product no ACTIVE record claims β archived, and R12 keeps those
|
| 397 |
-
# out. `validate()` asserts that of Odoo rather than assuming it, and reports the revenue
|
| 398 |
-
# that therefore sits outside the grid (MEASURED 2026-08-11: 3 codes, $0.00 YTD / $294.50 LY).
|
| 399 |
-
rev = {r["code"]: r for r in products.directory(t=t, team_id=team_id)}
|
| 400 |
-
|
| 401 |
-
rows = []
|
| 402 |
-
for code, meta in cat.items():
|
| 403 |
-
r = rev.get(code)
|
| 404 |
-
row = {
|
| 405 |
-
"pid": sku_pid(code),
|
| 406 |
-
"code": code,
|
| 407 |
-
# β ONE source for the name and the category, the CATALOGUE β not the sale line's
|
| 408 |
-
# m2o. For a re-SKUed code the line's name can be the ARCHIVED record's; the active
|
| 409 |
-
# record's `display_name` is the current truth, and it is the same string for every
|
| 410 |
-
# SKU that is not re-SKUed. `directory()` derives the category identically.
|
| 411 |
-
"product": meta.get("product") or code,
|
| 412 |
-
"category": meta.get("category") or "(uncategorized)",
|
| 413 |
-
# ββ W33-T43 (R2 / amendment A2) β ODOO'S OWN PRODUCT ID, beside the hashed `pid`.
|
| 414 |
-
# R2 retires `ut_odoo_products` onto this key and keeps every data column it had; this
|
| 415 |
-
# is that column. β NOT DERIVABLE DOWNSTREAM: `pid` is `crc32(default_code)` here,
|
| 416 |
-
# unlike a customer row whose `pid` IS the partner id, so `aios_grid.py` cannot recover
|
| 417 |
-
# it and it has to be carried from `products.catalogue()`.
|
| 418 |
-
# β `None`, never 0, when the catalogue somehow has no id β 0 is a real Odoo id.
|
| 419 |
-
"product_id": meta.get("id"),
|
| 420 |
-
# β R12 β BLANK, NEVER $0, on a SKU no revenue read ever saw. A row that IS in `rev`
|
| 421 |
-
# keeps its measured numbers including a real 0.0 (it sold last year, not this one).
|
| 422 |
-
"rev_ytd": r.get("rev_ytd", 0.0) if r else None,
|
| 423 |
-
"rev_ly": r.get("rev_ly", 0.0) if r else None,
|
| 424 |
-
"yoy_pct": r.get("yoy_pct") if r else None,
|
| 425 |
-
"qty_ytd": r.get("qty_ytd", 0.0) if r else None,
|
| 426 |
-
"orders_ytd": r.get("orders_ytd", 0) if r else None,
|
| 427 |
-
}
|
| 428 |
-
# β WAVE 30 W30-T34 β the PRICELIST stratum, one column per declared list. Catalogue
|
| 429 |
-
# data like the supplier block, so it rides every pull, scoped or not: a price book is
|
| 430 |
-
# not a thing a business unit owns a slice of, and a Fisch reader who cannot see the
|
| 431 |
-
# Fisch price is exactly who this column is for.
|
| 432 |
-
#
|
| 433 |
-
# β `None`, NEVER 0 AND NEVER A FALLBACK, for the 1,883 SKUs no list prices. That is
|
| 434 |
-
# W29-T52's own negative control: the single `3_global` rule computes over `list_price`,
|
| 435 |
-
# which is 1.00 on 5,817 of 5,871 products, so the fallback is not a cheaper answer β
|
| 436 |
-
# it is a wrong one wearing a currency sign.
|
| 437 |
-
p = prices.get(code) or {}
|
| 438 |
-
for _col, _name in products.PRICELIST_COLUMNS:
|
| 439 |
-
row[_col] = p.get(_col)
|
| 440 |
-
# Wave 17 R3 β the supplier master, on every pull (it is catalogue data, not stock).
|
| 441 |
-
s = sup.get(code) or {}
|
| 442 |
-
row.update({
|
| 443 |
-
"supplier": s.get("supplier") or "(none)",
|
| 444 |
-
"lead_days": s.get("lead_days"),
|
| 445 |
-
"origin_country": s.get("origin_country") or "(none)",
|
| 446 |
-
"first_cost": s.get("first_cost"),
|
| 447 |
-
})
|
| 448 |
-
e = inv.get(code) or {}
|
| 449 |
-
if consolidated:
|
| 450 |
-
qty_ltm, dos, bucket = e.get("qty_ltm"), e.get("dos"), e.get("bucket")
|
| 451 |
-
else:
|
| 452 |
-
qty_ltm, dos, bucket = _rescope_inventory(e, bu_share.get(code, _DEFAULT_SHARE))
|
| 453 |
-
row.update({
|
| 454 |
-
# UNSCOPED on purpose (owner 2026-08-11): one warehouse, no per-brand shelf.
|
| 455 |
-
"on_hand": e.get("on_hand"),
|
| 456 |
-
"unit_cost": e.get("unit_cost"),
|
| 457 |
-
"inv_value": e.get("inv_value"),
|
| 458 |
-
# SCOPED: these three are functions of how fast THIS unit sells the SKU.
|
| 459 |
-
"qty_ltm": qty_ltm,
|
| 460 |
-
"dos": dos,
|
| 461 |
-
"stock_bucket": bucket,
|
| 462 |
-
})
|
| 463 |
-
# THE BUY TRIGGER (R3), ported from `modules/procurement`'s lead-time-cover rule:
|
| 464 |
-
# buy when the shelf runs out before a reorder could land.
|
| 465 |
-
#
|
| 466 |
-
# β BLANK, NOT "OK", WHEN EITHER INPUT IS MISSING. 339 SKUs have no supplier and 400
|
| 467 |
-
# no lead time; `dos` is null for anything that never sells through. "We don't know"
|
| 468 |
-
# and "you're fine" are different sentences, and only one of them is safe to print
|
| 469 |
-
# beside a purchasing decision.
|
| 470 |
-
#
|
| 471 |
-
# β `buy_now` SHIPS NOWHERE ANY MORE, and is computed anyway. OWNER 2026-08-03: it
|
| 472 |
-
# is not a preset field β it is a formula over the two columns beside it, and the
|
| 473 |
-
# platform has a formula field type for exactly that. Its Field is gone from
|
| 474 |
-
# `aios_grid_fields.json`, and `rows_from_pool` projects rows STRICTLY through that
|
| 475 |
-
# contract, so no Field means no cell on the wire. Nothing renders this key.
|
| 476 |
-
#
|
| 477 |
-
# It stays computed because deleting it would delete the PROOF. `validate()` below
|
| 478 |
-
# reconciles the formula's predicate against this one, row for row, which is the only
|
| 479 |
-
# thing that makes "the same exact figures" a claim rather than an assertion β and
|
| 480 |
-
# this repo has the scar already (wave 17: archiving `ar` "would have DELETED the
|
| 481 |
-
# proof"). Two comparisons per SKU is what that costs.
|
| 482 |
-
#
|
| 483 |
-
# β READS THE SCOPED `dos`, NOT `e['dos']`. On a Fisch pull the cover gap must answer
|
| 484 |
-
# "does the shelf outlast a reorder AT FISCH'S RATE" β reading the consolidated figure
|
| 485 |
-
# here would print a buy signal computed from both units' velocity beside a days-of-supply
|
| 486 |
-
# computed from one, and the two columns would disagree on the same row.
|
| 487 |
-
lead = s.get("lead_days")
|
| 488 |
-
if isinstance(dos, (int, float)) and isinstance(lead, (int, float)) and lead > 0:
|
| 489 |
-
row["cover_gap_d"] = int(round(dos - lead))
|
| 490 |
-
row["buy_now"] = "Buy now" if dos < lead else "OK"
|
| 491 |
-
else:
|
| 492 |
-
row["cover_gap_d"] = None
|
| 493 |
-
row["buy_now"] = None
|
| 494 |
-
rows.append(row)
|
| 495 |
-
|
| 496 |
-
# `or 0.0` reads a BLANK as zero FOR SORTING ONLY β never for the cell. A never-sold SKU
|
| 497 |
-
# settles among the zero-revenue ones at the bottom, which is where it belongs; the stored
|
| 498 |
-
# value stays None so nothing downstream can mistake "we never saw a sale" for "we measured
|
| 499 |
-
# nothing sold".
|
| 500 |
-
rows.sort(key=lambda r: -(r["rev_ytd"] or 0.0))
|
| 501 |
-
_assert_no_pid_collision(rows)
|
| 502 |
-
return rows
|
| 503 |
-
|
| 504 |
-
|
| 505 |
-
#: β THE BUY SIGNAL, AS THE FORMULA FIELD EVALUATES IT (owner, 2026-08-03).
|
| 506 |
-
#:
|
| 507 |
-
#: The owner's ruling was that "Buy signal" is not a preset field β it is a formula over two
|
| 508 |
-
#: columns the product table already carries, and the platform has a formula field type for it.
|
| 509 |
-
#: The formula, verbatim, is what `_seed_wave17.BUY_SIGNAL_FORMULA` creates and what the JSON
|
| 510 |
-
#: contract's `_product_removed_buy_now` note records:
|
| 511 |
-
#:
|
| 512 |
-
#: IF({lead_days} > 0, IF({dos} < {lead_days}, "Buy now", "OK"), "")
|
| 513 |
-
#:
|
| 514 |
-
#: This function is a PORT of how `customer-grid/formulaEngine.ts` evaluates that tree, not a
|
| 515 |
-
#: restatement of the business rule β that is the whole point, because the two could drift and
|
| 516 |
-
#: `validate()` is where the drift must show. Three engine behaviours it reproduces exactly:
|
| 517 |
-
#:
|
| 518 |
-
#: Β· a `ref` to a missing/non-numeric cell is None (`case "ref"` returns null for anything
|
| 519 |
-
#: that is not a finite number or a string);
|
| 520 |
-
#: Β· `cmp` returns BLANK unless BOTH sides read as numbers β it never coerces a blank to 0,
|
| 521 |
-
#: which is the difference between this and a filter engine's `toNum(null) === 0`;
|
| 522 |
-
#: Β· `IF` with a non-boolean condition returns BLANK ("no truthiness"), so a blank comparison
|
| 523 |
-
#: propagates out as a blank cell rather than taking the false branch.
|
| 524 |
-
BUY_SIGNAL_FORMULA = 'IF({lead_days} > 0, IF({dos} < {lead_days}, "Buy now", "OK"), "")'
|
| 525 |
-
|
| 526 |
-
|
| 527 |
-
def _buy_signal_formula(row):
|
| 528 |
-
"""Evaluate `BUY_SIGNAL_FORMULA` over one pool row -> 'Buy now' | 'OK' | '' (blank)."""
|
| 529 |
-
def num(v):
|
| 530 |
-
# The engine's `ref` + `asNumber`: booleans are not values here, and a non-finite
|
| 531 |
-
# number is null. `isinstance(True, int)` is True in Python, so bool is excluded first.
|
| 532 |
-
if isinstance(v, bool) or not isinstance(v, (int, float)):
|
| 533 |
-
return None
|
| 534 |
-
return v if v == v and v not in (float('inf'), float('-inf')) else None
|
| 535 |
-
|
| 536 |
-
lead, dos = num(row.get("lead_days")), num(row.get("dos"))
|
| 537 |
-
if lead is None: # `{lead_days} > 0` is blank -> IF(blank, β¦) is blank
|
| 538 |
-
return ""
|
| 539 |
-
if not lead > 0:
|
| 540 |
-
return "" # the formula's own else-branch
|
| 541 |
-
if dos is None: # `{dos} < {lead_days}` is blank -> IF(blank, β¦) is blank
|
| 542 |
-
return ""
|
| 543 |
-
return "Buy now" if dos < lead else "OK"
|
| 544 |
-
|
| 545 |
-
|
| 546 |
-
def validate(team_id=None, t=None):
|
| 547 |
-
"""Reconcile what SHIPS to an independent aggregate β the platform's own rule that a number
|
| 548 |
-
which does not tie to Odoo does not ship.
|
| 549 |
-
|
| 550 |
-
ββ THE POPULATION LEG IS NEW, AND IT IS HERE BECAUSE THIS FUNCTION USED TO BE SELF-SEALING
|
| 551 |
-
(wave 29, contract C8). It reconciled Ξ£ per-SKU YTD revenue against `products._sku_rev` β the
|
| 552 |
-
same function `pool()` built its rows from. BOTH SIDES CAME FROM THE JOIN, so the oracle could
|
| 553 |
-
not see a missing row, and a grid holding 2,717 of 5,875 active products passed for a wave.
|
| 554 |
-
No gate anywhere asserted a pool row COUNT against an independent Odoo `search_count` either.
|
| 555 |
-
That is the leg below, and it is the control that would have caught it
|
| 556 |
-
([[no-unverifiable-aggregates]]).
|
| 557 |
-
|
| 558 |
-
β The revenue leg was NOT deleted with it β it answers a different question ("did the join
|
| 559 |
-
lose money?") and is still the right shape for that one. What changed is that it now has to
|
| 560 |
-
account for revenue belonging to codes NO ACTIVE PRODUCT CARRIES, because R12 keeps archived
|
| 561 |
-
products out of the grid; the total is decomposed rather than compared loosely, so a growing
|
| 562 |
-
"outside the grid" figure shows up as a number rather than as slack in a tolerance.
|
| 563 |
-
|
| 564 |
-
Still deliberately NOT covered, and said rather than implied: the inventory columns are a
|
| 565 |
-
different module's reconciliation, and they are absent on a scoped pull anyway.
|
| 566 |
-
"""
|
| 567 |
-
t = t or P.today()
|
| 568 |
-
rows = pool(team_id=team_id, t=t)
|
| 569 |
-
|
| 570 |
-
# ββ THE POPULATION, AGAINST AN ORACLE THAT CANNOT SEE OUR JOIN. A bare Odoo count of active
|
| 571 |
-
# products, asked fresh β not a `len()` over anything this module or `products.catalogue()`
|
| 572 |
-
# built. NC: drop one row from `pool()` and this goes red; that is the whole point of it.
|
| 573 |
-
n_active = products.catalogue_count()
|
| 574 |
-
checks = [{
|
| 575 |
-
"check": "the pool holds one row per ACTIVE product (R12) β row count == an INDEPENDENT "
|
| 576 |
-
"Odoo search_count('product.product', active=True)",
|
| 577 |
-
"ours": len(rows), "theirs": n_active, "ok": len(rows) == n_active,
|
| 578 |
-
}]
|
| 579 |
-
|
| 580 |
-
yf, yt = P.ytd(t)
|
| 581 |
-
sku_rev = products._sku_rev(yf, yt, team_id) or {}
|
| 582 |
-
in_pool = {r["code"] for r in rows}
|
| 583 |
-
outside = {c: v for c, v in sku_rev.items() if c not in in_pool}
|
| 584 |
-
ours = round(sum(r["rev_ytd"] or 0.0 for r in rows), 2)
|
| 585 |
-
theirs = round(sum(v.get("rev", 0.0) for v in sku_rev.values()), 2)
|
| 586 |
-
dropped = round(sum(v.get("rev", 0.0) for v in outside.values()), 2)
|
| 587 |
-
checks.append({
|
| 588 |
-
"check": "Ξ£ per-SKU YTD revenue in the grid + Ξ£ YTD revenue of codes no ACTIVE product "
|
| 589 |
-
"carries == modules.products._sku_rev over the same window",
|
| 590 |
-
"ours": round(ours + dropped, 2), "theirs": theirs,
|
| 591 |
-
"ok": abs((ours + dropped) - theirs) < 0.01,
|
| 592 |
-
"detail": {"in_the_grid": ours, "outside_the_grid": dropped,
|
| 593 |
-
"codes_outside": len(outside)},
|
| 594 |
-
})
|
| 595 |
-
# β AND THE EXCLUSION IS ASSERTED, NOT ASSUMED. "Everything I dropped was archived" is
|
| 596 |
-
# trivially true when checked against the dict that did the dropping β that is the
|
| 597 |
-
# self-sealing shape all over again. So it is asked of ODOO. A code here that a live ACTIVE
|
| 598 |
-
# product carries means `products.catalogue()` missed a row that has revenue, which is the
|
| 599 |
-
# 2,717 defect returning in a smaller costume.
|
| 600 |
-
# β `pid:N` keys are not `default_code`s and are skipped: an UNCODED active product is in the
|
| 601 |
-
# catalogue under its own `pid:N` key and therefore cannot be in `outside` at all.
|
| 602 |
-
coded_outside = sorted(c for c in outside if not str(c).startswith("pid:"))
|
| 603 |
-
still_active = (O.get_odoo().search_count(
|
| 604 |
-
'product.product', [('active', '=', True), ('default_code', 'in', coded_outside)])
|
| 605 |
-
if coded_outside else 0)
|
| 606 |
-
checks.append({
|
| 607 |
-
"check": "every SKU with revenue but NO grid row is genuinely ARCHIVED (R12: archived "
|
| 608 |
-
"stay out) β asked of Odoo, never of the catalogue that did the dropping",
|
| 609 |
-
"ours": still_active, "theirs": 0, "ok": still_active == 0,
|
| 610 |
-
"detail": {"codes": coded_outside[:10], "n_codes": len(outside)},
|
| 611 |
-
})
|
| 612 |
-
# ββ ββ WAVE 30 W30-T34 β THE PRICELIST COLUMNS, AGAINST ORACLES THAT CANNOT SEE OUR JOIN ββ
|
| 613 |
-
#
|
| 614 |
-
# Three legs, because the column can fail in three different ways and only one of them is a
|
| 615 |
-
# count. `products.pricelist_by_code` builds a `{code: {col: price}}` map by expanding each
|
| 616 |
-
# rule over the products it names, IN PYTHON β that expansion is the fragile part, so every
|
| 617 |
-
# leg below re-asks ODOO instead of re-reading the map.
|
| 618 |
-
price_report = _pricelist_by_code()[1]
|
| 619 |
-
_pl_rows = O.search_read('product.pricelist', [], ['id', 'name'])
|
| 620 |
-
_pl_id = {}
|
| 621 |
-
for _p in _pl_rows:
|
| 622 |
-
_pl_id.setdefault(str(_p.get('name') or '').strip(), _p['id'])
|
| 623 |
-
_today = P.today().isoformat()
|
| 624 |
-
|
| 625 |
-
_targets_memo = {}
|
| 626 |
-
|
| 627 |
-
def _rule_targets(list_name):
|
| 628 |
-
"""(variant_ids, template_ids) a pricelist prices today β read FRESH from Odoo.
|
| 629 |
-
|
| 630 |
-
Memoised for the LIFE OF THIS CALL only: legs 1 and 2 both need all three lists, and
|
| 631 |
-
without this `validate()` makes six identical round trips instead of three (measured:
|
| 632 |
-
~48s of the run). Deliberately NOT an `lru_cache` β an oracle that survives the process
|
| 633 |
-
is an oracle reading yesterday's Odoo.
|
| 634 |
-
"""
|
| 635 |
-
if list_name in _targets_memo:
|
| 636 |
-
return _targets_memo[list_name]
|
| 637 |
-
var, tmpl = set(), set()
|
| 638 |
-
plid = _pl_id.get(list_name)
|
| 639 |
-
if plid is None:
|
| 640 |
-
return var, tmpl
|
| 641 |
-
for r in O.search_read(
|
| 642 |
-
'product.pricelist.item',
|
| 643 |
-
[('pricelist_id', '=', plid), ('compute_price', '=', 'fixed'),
|
| 644 |
-
('applied_on', 'in', ['0_product_variant', '1_product'])],
|
| 645 |
-
['product_id', 'product_tmpl_id', 'applied_on', 'fixed_price',
|
| 646 |
-
'date_start', 'date_end']):
|
| 647 |
-
ds, de = str(r.get('date_start') or '')[:10], str(r.get('date_end') or '')[:10]
|
| 648 |
-
if (ds and ds > _today) or (de and de < _today):
|
| 649 |
-
continue
|
| 650 |
-
fp = r.get('fixed_price')
|
| 651 |
-
if not isinstance(fp, (int, float)) or fp <= 0:
|
| 652 |
-
continue
|
| 653 |
-
if r.get('applied_on') == '0_product_variant' and r.get('product_id'):
|
| 654 |
-
var.add(O.m2o_id(r['product_id']))
|
| 655 |
-
elif r.get('product_tmpl_id'):
|
| 656 |
-
tmpl.add(O.m2o_id(r['product_tmpl_id']))
|
| 657 |
-
_targets_memo[list_name] = (var, tmpl)
|
| 658 |
-
return var, tmpl
|
| 659 |
-
|
| 660 |
-
# LEG 1 β COVERAGE PER LIST. Ours: cells we filled. Theirs: a bare Odoo `search_count` of
|
| 661 |
-
# ACTIVE products a fresh read of that list's rules reaches. The rule set is shared (it IS
|
| 662 |
-
# the data) but the EXPANSION is not, and the expansion is what breaks.
|
| 663 |
-
for _col, _name in products.PRICELIST_COLUMNS:
|
| 664 |
-
_var, _tmpl = _rule_targets(_name)
|
| 665 |
-
_dom = [('active', '=', True), '|', ('id', 'in', sorted(_var)),
|
| 666 |
-
('product_tmpl_id', 'in', sorted(_tmpl))]
|
| 667 |
-
theirs = O.get_odoo().search_count('product.product', _dom) if (_var or _tmpl) else 0
|
| 668 |
-
ours = sum(1 for r in rows if isinstance(r.get(_col), (int, float)))
|
| 669 |
-
checks.append({
|
| 670 |
-
"check": f"{_name} pricelist: SKUs priced in the grid == an INDEPENDENT Odoo count "
|
| 671 |
-
f"of ACTIVE products its date-valid fixed rules reach",
|
| 672 |
-
"ours": ours, "theirs": theirs, "ok": ours == theirs,
|
| 673 |
-
"detail": {"column": _col, "variant_rules": len(_var), "template_rules": len(_tmpl)},
|
| 674 |
-
})
|
| 675 |
-
|
| 676 |
-
# LEG 2 β β THE TICKET'S OWN NEGATIVE CONTROL, AS A LEG. A SKU with no specific item must
|
| 677 |
-
# render BLANK. Asked of ODOO, never of the map that produced the blank β "everything I left
|
| 678 |
-
# empty was genuinely unpriced" is trivially true when checked against the dict that emptied
|
| 679 |
-
# it, which is the self-sealing shape the population leg above exists to end.
|
| 680 |
-
_unpriced = [r for r in rows
|
| 681 |
-
if not any(isinstance(r.get(c), (int, float))
|
| 682 |
-
for c, _n in products.PRICELIST_COLUMNS)]
|
| 683 |
-
_codes = {r["code"] for r in _unpriced}
|
| 684 |
-
_stray = 0
|
| 685 |
-
if _codes:
|
| 686 |
-
# Resolve those codes back to Odoo ids and ask whether ANY declared list prices them.
|
| 687 |
-
_ids, _tmpls = set(), set()
|
| 688 |
-
for _p in O.search_read('product.product',
|
| 689 |
-
[('active', '=', True),
|
| 690 |
-
('default_code', 'in', sorted(c for c in _codes
|
| 691 |
-
if not c.startswith("pid:")))],
|
| 692 |
-
['id', 'product_tmpl_id']):
|
| 693 |
-
_ids.add(_p['id'])
|
| 694 |
-
_tmpls.add(O.m2o_id(_p.get('product_tmpl_id')))
|
| 695 |
-
for _col, _name in products.PRICELIST_COLUMNS:
|
| 696 |
-
_v, _t = _rule_targets(_name)
|
| 697 |
-
_stray += len((_v & _ids) | ({t for t in _tmpls if t in _t}))
|
| 698 |
-
checks.append({
|
| 699 |
-
"check": "every SKU rendering a BLANK price is genuinely unpriced on all three declared "
|
| 700 |
-
"lists (W29-T52's NC: no fallback dressed as a price) β asked of Odoo",
|
| 701 |
-
"ours": _stray, "theirs": 0, "ok": _stray == 0,
|
| 702 |
-
"detail": {"blank_skus": len(_unpriced), "priced_skus": len(rows) - len(_unpriced)},
|
| 703 |
-
})
|
| 704 |
-
|
| 705 |
-
# LEG 3 β β R6's SECOND SENTENCE, AS A NUMBER. *"If there is lag or it can't be done, you
|
| 706 |
-
# need to explicitly tell me why and recommend a fix."* Everything the reader cannot see is
|
| 707 |
-
# counted here rather than dropped. The ASSERTION is the one thing that must never be true β
|
| 708 |
-
# a price cell that is present and not a positive number, i.e. a "free" SKU β while the
|
| 709 |
-
# rest rides `detail` so it is reported without reddening an honest day's data.
|
| 710 |
-
_bad = [r["code"] for r in rows
|
| 711 |
-
for c, _n in products.PRICELIST_COLUMNS
|
| 712 |
-
if r.get(c) is not None and not (isinstance(r.get(c), (int, float)) and r[c] > 0)]
|
| 713 |
-
# β MEASURED 2026-08-12 AND NOT FIXABLE FROM THIS FENCE: `aios_grid.rows_from_pool` sends
|
| 714 |
-
# every non-text field through `_round(v)` = `round(v)` with no ndigits, i.e. to a whole
|
| 715 |
-
# dollar. 52.1% of fixed prices carry cents and the median relative error is 5.26%, so this
|
| 716 |
-
# column ties to the cent HERE and ships rounded. Reported, never silently enforced.
|
| 717 |
-
_cents = sum(1 for r in rows for c, _n in products.PRICELIST_COLUMNS
|
| 718 |
-
if isinstance(r.get(c), (int, float)) and abs(r[c] - round(r[c])) > 1e-9)
|
| 719 |
-
_to_zero = sum(1 for r in rows for c, _n in products.PRICELIST_COLUMNS
|
| 720 |
-
if isinstance(r.get(c), (int, float)) and round(r[c]) == 0)
|
| 721 |
-
checks.append({
|
| 722 |
-
"check": "no price cell is present-but-not-a-positive-number (a 0 would read as FREE; "
|
| 723 |
-
"an unset rule must be blank), and what the reader cannot see is COUNTED",
|
| 724 |
-
"ours": len(_bad), "theirs": 0, "ok": not _bad,
|
| 725 |
-
"detail": {"bad_cells": _bad[:10], "reader_report": price_report,
|
| 726 |
-
"wire_rounding_loses_cents_on": _cents,
|
| 727 |
-
"wire_rounding_to_zero_dollars": _to_zero},
|
| 728 |
-
})
|
| 729 |
-
|
| 730 |
-
# WAVE 17 R3 β the BUY SIGNAL must be a total, exact partition of the catalogue, and every
|
| 731 |
-
# member of it must be re-derivable from the two columns beside it. A signal somebody buys
|
| 732 |
-
# stock on cannot be "mostly right": the failure that matters is a row that says OK because
|
| 733 |
-
# an input was missing, so the blank leg is checked as hard as the other two.
|
| 734 |
-
if team_id is None: # consolidated only β the inputs exist only there
|
| 735 |
-
buy = [r for r in rows if r.get("buy_now") == "Buy now"]
|
| 736 |
-
ok_rows = [r for r in rows if r.get("buy_now") == "OK"]
|
| 737 |
-
blank = [r for r in rows if r.get("buy_now") is None]
|
| 738 |
-
mis = sum(1 for r in buy if not (r["dos"] < r["lead_days"]))
|
| 739 |
-
mis += sum(1 for r in ok_rows if not (r["dos"] >= r["lead_days"]))
|
| 740 |
-
# A blank must be UNKNOWN β never a row we could have answered and quietly did not.
|
| 741 |
-
mis += sum(1 for r in blank
|
| 742 |
-
if isinstance(r.get("dos"), (int, float))
|
| 743 |
-
and isinstance(r.get("lead_days"), (int, float)) and r["lead_days"] > 0)
|
| 744 |
-
checks.append({
|
| 745 |
-
"check": "Buy signal partitions the catalogue (buy + ok + unknown == rows, none "
|
| 746 |
-
"misclassified)",
|
| 747 |
-
"ours": len(buy) + len(ok_rows) + len(blank) - mis, "theirs": len(rows),
|
| 748 |
-
"ok": mis == 0 and len(buy) + len(ok_rows) + len(blank) == len(rows),
|
| 749 |
-
"detail": {"buy_now": len(buy), "ok": len(ok_rows), "unknown": len(blank),
|
| 750 |
-
"misclassified": mis},
|
| 751 |
-
})
|
| 752 |
-
# β OWNER 2026-08-03 β "use the formula fields to come up to the same EXACT figures".
|
| 753 |
-
#
|
| 754 |
-
# This is that sentence, as a check. The retired preset column and the formula that
|
| 755 |
-
# replaces it must agree on every SKU, in all three states, or the replacement is not a
|
| 756 |
-
# replacement. `_buy_signal_formula` is a line-by-line port of the client formula
|
| 757 |
-
# engine's evaluation of
|
| 758 |
-
#
|
| 759 |
-
# IF({lead_days} > 0, IF({dos} < {lead_days}, "Buy now", "OK"), "")
|
| 760 |
-
#
|
| 761 |
-
# including the part that is easy to get wrong: a comparison against a BLANK is blank,
|
| 762 |
-
# never a coerced 0. (The client engine's `cmp` returns null unless both sides read as
|
| 763 |
-
# numbers, and `IF` refuses a non-boolean condition β so a missing `dos` yields "" and
|
| 764 |
-
# not "Buy now". A filter engine would have said `0 < 30` and swept in every SKU that
|
| 765 |
-
# never sells through; the formula engine does not, and this check is what holds it.)
|
| 766 |
-
#
|
| 767 |
-
# β It compares SETS OF SKUs, not counts. Two different partitions can share a shape.
|
| 768 |
-
disagree = sorted(r["code"] for r in rows
|
| 769 |
-
if (r.get("buy_now") or "") != _buy_signal_formula(r))
|
| 770 |
-
checks.append({
|
| 771 |
-
"check": 'Buy signal as a FORMULA field == the retired preset column, per SKU '
|
| 772 |
-
'(IF({lead_days} > 0, IF({dos} < {lead_days}, "Buy now", "OK"), ""))',
|
| 773 |
-
"ours": len(rows) - len(disagree), "theirs": len(rows),
|
| 774 |
-
"ok": not disagree,
|
| 775 |
-
"detail": {"disagreeing_skus": disagree[:10], "n_disagree": len(disagree)},
|
| 776 |
-
})
|
| 777 |
-
# The other half of "the same figures": the SHARED Buy list view can no longer filter on
|
| 778 |
-
# the retired column, and its replacement conditions must select the same SKUs. They are
|
| 779 |
-
# `dos isNotEmpty AND lead_days isNotEmpty AND lead_days > 0 AND dos < lead_days`
|
| 780 |
-
# (_seed_wave17.views), so this reproduces exactly that conjunction.
|
| 781 |
-
#
|
| 782 |
-
# β THE NEGATIVE CONTROL IS WHY THIS IS NOT `cover_gap_d < 0`, which reads like the
|
| 783 |
-
# obvious filter and is WRONG: `cover_gap_d` is `int(round(dos - lead))`, so a genuine
|
| 784 |
-
# gap of -0.4 days rounds to 0 and that SKU drops off a buy list it belongs on.
|
| 785 |
-
view_rows = {r["code"] for r in rows
|
| 786 |
-
if isinstance(r.get("dos"), (int, float))
|
| 787 |
-
and isinstance(r.get("lead_days"), (int, float))
|
| 788 |
-
and r["lead_days"] > 0 and r["dos"] < r["lead_days"]}
|
| 789 |
-
signal_rows = {r["code"] for r in buy}
|
| 790 |
-
rounding_would_miss = sorted(
|
| 791 |
-
c for c in signal_rows
|
| 792 |
-
if next((r for r in rows if r["code"] == c), {}).get("cover_gap_d") == 0)
|
| 793 |
-
checks.append({
|
| 794 |
-
"check": "Buy list view conditions (dos/lead_days, no retired column) select "
|
| 795 |
-
"exactly the Buy-now SKUs",
|
| 796 |
-
"ours": len(view_rows), "theirs": len(signal_rows),
|
| 797 |
-
"ok": view_rows == signal_rows,
|
| 798 |
-
"detail": {"only_in_view": sorted(view_rows - signal_rows)[:10],
|
| 799 |
-
"only_in_signal": sorted(signal_rows - view_rows)[:10],
|
| 800 |
-
# Reported, not asserted: how many SKUs a `cover_gap_d < 0` filter would
|
| 801 |
-
# have silently dropped. 0 today does not make that filter correct.
|
| 802 |
-
"cover_gap_rounds_to_zero": len(rounding_would_miss)},
|
| 803 |
-
})
|
| 804 |
-
return checks
|
|
|
|
| 1 |
+
"""modules/product_data.py β the PRODUCT table's pool (wave 15 item 9/10, contract C-TOPIC).
|
| 2 |
+
|
| 3 |
+
The second object on the table-page factory: same grid, same engine, same permission wall β the
|
| 4 |
+
only difference is the field schema and the identity. `modules/customer_data.pool()` is the
|
| 5 |
+
template and this deliberately mirrors its signature and its row shape.
|
| 6 |
+
|
| 7 |
+
β THREE DECISIONS THIS FILE HAD TO MAKE, EACH RECORDED BECAUSE A LATER READER WILL WONDER.
|
| 8 |
+
|
| 9 |
+
1. **THE IDENTITY IS A SKU CODE, WHICH IS A STRING, AND THE GRID WANTS AN INTEGER `pid`.**
|
| 10 |
+
Cohort membership, `allowed_pids`, the measure channel and `rows_from_pool` all key on an
|
| 11 |
+
integer. So each row carries BOTH: `pid` (a stable CRC32 of the code, so the same SKU gets the
|
| 12 |
+
same id on every pull and across processes β never an enumeration index, which would reshuffle
|
| 13 |
+
whenever the catalogue changes) and `code`, the real business key, as a visible column.
|
| 14 |
+
`_assert_no_pid_collision` fails the BUILD rather than the read: two SKUs sharing a pid would
|
| 15 |
+
silently merge in every downstream set operation, and a loud build failure is the only version
|
| 16 |
+
of that anyone would notice.
|
| 17 |
+
|
| 18 |
+
2. **THE SCOPE RULE β inventory columns are CONSOLIDATED and are therefore OMITTED for a
|
| 19 |
+
BU-scoped caller.** `products.directory(t, team_id)` is brand-shaped; `inventory.sku_inventory`
|
| 20 |
+
is explicitly NOT (its own docstring: "on-hand stock is one physical warehouse, not
|
| 21 |
+
brand-tagged"). Joining them for a Fisch-only user would put BU-shaped revenue beside
|
| 22 |
+
company-wide stock IN THE SAME ROW β the mixed-scope value defect wave 15's amendment 3 exists
|
| 23 |
+
for, arriving through a different door. The honest options were "omit" or "label the columns
|
| 24 |
+
company-wide"; omit is the fail-closed one, and a column that is absent asks a question,
|
| 25 |
+
whereas a column that is silently company-wide answers one wrongly.
|
| 26 |
+
|
| 27 |
+
3. **WHAT IS NOT HERE, NAMED RATHER THAN QUIETLY MISSING.** R4 lists ~20 fields. Vendor and
|
| 28 |
+
COUNTRY are not in Odoo at all β they live in the inventory WORKBOOK
|
| 29 |
+
([[odoo-vendor-country-origin]]) and need a loader this module deliberately does not invent.
|
| 30 |
+
Margin % comes from `modules/pricing.table`, which is a heavier build (channel-rate cost
|
| 31 |
+
allocation) and is left for the wave that needs it. `validate()` reconciles what SHIPS; it
|
| 32 |
+
does not pretend to cover columns that are absent.
|
| 33 |
+
|
| 34 |
+
4. **THE POOL IS CATALOGUE-FIRST, WITH REVENUE LEFT-JOINED** (wave 29, owner item 22 / R12,
|
| 35 |
+
2026-08-11). It was `for r in products.directory(...)` β and `directory()`'s row set IS the
|
| 36 |
+
union of two revenue read-groups, so **a SKU that never sold could not exist** and the grid
|
| 37 |
+
showed **2,717** of **5,875** active products. Three things that will be re-derived otherwise:
|
| 38 |
+
|
| 39 |
+
Β· β THE CAUSE IS A JOIN, NOT A LIMIT. No row cap exists on this path. Removing the date
|
| 40 |
+
window from `directory()` would ALSO be wrong twice: it breaks four SKU-health metrics
|
| 41 |
+
that legitimately want a sales window, and it lands at 3,327 (all-time-sold), because
|
| 42 |
+
~2,550 active SKUs have never sold in wholesale scope at all.
|
| 43 |
+
Β· β REVENUE ON A NEVER-SOLD ROW IS **BLANK, NOT $0** (R12). A row that appears in the
|
| 44 |
+
revenue universe carries measured numbers INCLUDING a real 0.0 β it sold last year and
|
| 45 |
+
not this one, and that zero is a measurement. A row that appears in NO revenue read
|
| 46 |
+
carries `None`, which the wire keeps (`aios_grid._round` passes None through) and the
|
| 47 |
+
client renders as an empty cell. Blank is an admission; zero is a measurement.
|
| 48 |
+
Β· β A BU-SCOPED CALLER NOW SEES THE WHOLE CATALOGUE, with ITS OWN revenue and blanks where
|
| 49 |
+
that BU never sold. That is not decision 2's mixed-scope defect: the catalogue is the ROW
|
| 50 |
+
UNIVERSE, not a company-wide VALUE sitting beside a BU-shaped one. `product.product`
|
| 51 |
+
carries no team, so a catalogue cannot be BU-shaped at all β which is exactly why the two
|
| 52 |
+
sources are JOINED rather than merged.
|
| 53 |
+
"""
|
| 54 |
+
import json
|
| 55 |
+
import zlib
|
| 56 |
+
from pathlib import Path
|
| 57 |
+
|
| 58 |
+
import core.odoo as O
|
| 59 |
+
import core.periods as P
|
| 60 |
+
import core.shared_overlay as shared_overlay
|
| 61 |
+
import core.table_store as table_store
|
| 62 |
+
import modules.products as products
|
| 63 |
+
|
| 64 |
+
#: POOL-ROW KEYS that exist ONLY on a consolidated pull. See decision 2.
|
| 65 |
+
#: `cover_gap_d` / `buy_now` (wave 17) join it because both are computed FROM `dos`, and a buy
|
| 66 |
+
#: signal built on a stock number the caller cannot see would be a recommendation nobody could
|
| 67 |
+
#: check ([[no-unverifiable-aggregates]]).
|
| 68 |
+
#:
|
| 69 |
+
#: β ROW KEYS, NOT FIELDS, and the distinction started mattering on 2026-08-03. Two readers:
|
| 70 |
+
#: `verify_perm_scope` asserts the shape of a POOL ROW against this, and `routes_products`
|
| 71 |
+
#: narrows the FIELD list with it. Since owner item 5 `buy_now` is a row key with no Field β
|
| 72 |
+
#: computed as `validate()`'s oracle, projected onto no wire β so it belongs here for the first
|
| 73 |
+
#: reader and is inert for the second. Removing it would stop the scope gate proving that a
|
| 74 |
+
#: BU-scoped pull withholds it.
|
| 75 |
+
#:
|
| 76 |
+
#: ββ OWNER RULING 2026-08-11 β THIS SET IS NOW EMPTY, AND THAT IS THE POINT. Verbatim:
|
| 77 |
+
#: *"Just scope any inventory with Sales from Fisch, leave the rest."* Withholding the whole
|
| 78 |
+
#: inventory block from a BU-scoped reader meant a Fisch salesperson could not see whether
|
| 79 |
+
#: anything was IN STOCK, which is the first question they ask. The ruling splits the block by
|
| 80 |
+
#: what a business unit can actually shape:
|
| 81 |
+
#:
|
| 82 |
+
#: * `on_hand` / `unit_cost` / `inv_value` β ONE physical warehouse, no Fisch shelf and no
|
| 83 |
+
#: Royal shelf. Served UNSCOPED to everybody, identical in both pulls. ("leave the rest")
|
| 84 |
+
#: * `qty_ltm` / `dos` / `stock_bucket` / `cover_gap_d` / `buy_now` β all SALES-derived, so a
|
| 85 |
+
#: BU-scoped pull recomputes them from THAT unit's LTM units. ("scope any inventory with
|
| 86 |
+
#: Sales from Fisch")
|
| 87 |
+
#:
|
| 88 |
+
#: This deliberately relaxes decision 2's "never a company-wide value beside a BU-shaped one":
|
| 89 |
+
#: stock is not a value a BU can own, and a column that is absent for a Fisch reader asks a
|
| 90 |
+
#: question they cannot answer anywhere else in the product. `verify_perm_scope` no longer
|
| 91 |
+
#: asserts the columns are WITHHELD β it asserts the split above, which is a stronger claim and
|
| 92 |
+
#: cannot pass on an empty scoped payload (the shape the old rule and an outage share).
|
| 93 |
+
CONSOLIDATED_ONLY = ()
|
| 94 |
+
|
| 95 |
+
#: β POOL-ROW KEYS THAT DELIBERATELY HAVE NO FIELD β they must never reach a browser.
|
| 96 |
+
#:
|
| 97 |
+
#: `rows_from_pool` projects strictly through the field contract, so "no Field" already means
|
| 98 |
+
#: "no cell on the wire". This tuple makes that a CHECKED fact rather than a consequence nobody
|
| 99 |
+
#: is watching: `verify_perm_scope` asserts every member is absent from BOTH product contracts,
|
| 100 |
+
#: and that every OTHER `CONSOLIDATED_ONLY` key is present in the consolidated one.
|
| 101 |
+
#:
|
| 102 |
+
#: Written because the two tuples silently diverged the moment owner item 5 retired `buy_now`'s
|
| 103 |
+
#: Field while keeping its computation, and the gate β which was asserting a FIELD property from
|
| 104 |
+
#: a ROW-key list β went red with no way to tell a deliberate divergence from a dropped column.
|
| 105 |
+
UNSHIPPED_ROW_KEYS = ("buy_now",)
|
| 106 |
+
|
| 107 |
+
#: Wave 17 (owner item 13, ruling R3) β the SUPPLIER MASTER, from the curated mastersheet map
|
| 108 |
+
#: `procurement_suppliers.json` (3,963 SKUs; supplier on 3,624, lead time on 3,563). NOT Odoo:
|
| 109 |
+
#: the owner confirmed this data came from the Inventory System Mastersheet, which is why
|
| 110 |
+
#: `modules/product_data`'s header said Vendor/Country were "NOT HERE, named rather than
|
| 111 |
+
#: quietly missing" and needed "a loader this module deliberately does not invent". This is
|
| 112 |
+
#: that loader.
|
| 113 |
+
#:
|
| 114 |
+
#: β THESE ARE CONTRACT COLUMNS, NOT USER-CREATED FIELDS, AND THE REASON IS A PERMISSION FACT.
|
| 115 |
+
#: A user-created field and its values live in the PER-USER strata (`table_store.workspace`
|
| 116 |
+
#: reads `store.get(key)[username]`); only VIEWS are shared. So a shared "Buy list" view that
|
| 117 |
+
#: filtered on a user-created column would, for every OTHER account, name a column that does not
|
| 118 |
+
#: exist β and an unknown column is an INACTIVE condition in the tri-state engine, which IGNORES
|
| 119 |
+
#: it and therefore WIDENS. The buy list would silently show the whole catalogue to everyone but
|
| 120 |
+
#: its author. Contract columns are identical for every reader, so the view means one thing.
|
| 121 |
+
#:
|
| 122 |
+
#: ββ 2026-08-12 (W30-T36) β THE OWNER'S ASK IS NOW DELIVERED, AND THE PARAGRAPH ABOVE IS WHY IT
|
| 123 |
+
#: TOOK THREE WAVES. Owner: *"turn this Excel sheet into a User created Field that we can edit."*
|
| 124 |
+
#: It was parked because a per-user column silently WIDENS a shared view β not because editing was
|
| 125 |
+
#: hard. Wave 29's `core/shared_overlay.py` (whose header quotes this very comment) is the stratum
|
| 126 |
+
#: that removes the objection: **one value per (row, column) for the whole tenant**, so the column
|
| 127 |
+
#: still means ONE thing to every reader and a shared view still filters honestly.
|
| 128 |
+
#: β The four columns below are now `source: "overlay"` + `shared: true` in the canonical
|
| 129 |
+
#: contract, their values live in `<TABLE_KEY>__shared`, and the master map is what SEEDS an
|
| 130 |
+
#: unedited cell rather than what freezes it. See `SHARED_KEYS` and `_ProductTableStore`.
|
| 131 |
+
_SUPPLIER_MAP_PATH = Path(__file__).resolve().parent.parent / "procurement_suppliers.json"
|
| 132 |
+
_SUPPLIER_CACHE = {}
|
| 133 |
+
|
| 134 |
+
|
| 135 |
+
def supplier_master():
|
| 136 |
+
"""`{code: {supplier, lead_days, origin_country, first_cost}}`, read once per process.
|
| 137 |
+
|
| 138 |
+
Degrades to `{}` when the file is unreadable, matching `_inventory_by_code`: a product table
|
| 139 |
+
that will not render because a master map is missing is a worse failure than one with blank
|
| 140 |
+
supplier columns.
|
| 141 |
+
"""
|
| 142 |
+
if _SUPPLIER_CACHE:
|
| 143 |
+
return _SUPPLIER_CACHE
|
| 144 |
+
try:
|
| 145 |
+
raw = json.loads(_SUPPLIER_MAP_PATH.read_text(encoding="utf-8"))
|
| 146 |
+
except Exception:
|
| 147 |
+
return {}
|
| 148 |
+
for code, meta in (raw or {}).items():
|
| 149 |
+
if not isinstance(meta, dict):
|
| 150 |
+
continue
|
| 151 |
+
lead = meta.get("lead")
|
| 152 |
+
_SUPPLIER_CACHE[str(code)] = {
|
| 153 |
+
"supplier": (meta.get("vendor") or "") or None,
|
| 154 |
+
"lead_days": int(lead) if isinstance(lead, (int, float)) else None,
|
| 155 |
+
"origin_country": (meta.get("country") or "") or None,
|
| 156 |
+
"first_cost": meta.get("first_cost"),
|
| 157 |
+
}
|
| 158 |
+
return _SUPPLIER_CACHE
|
| 159 |
+
|
| 160 |
+
#: Wave 16 C-TOPIC β the PRODUCT table's OWN workspace bucket. β Never the customer one:
|
| 161 |
+
#: product pids are CRC32 hashes of SKU codes and customer pids are Odoo partner ids, so in a
|
| 162 |
+
#: SHARED overlay bucket a hash collision would silently write a product note onto somebody's
|
| 163 |
+
#: customer (or the reverse). Separate store keys make that structurally impossible, which is
|
| 164 |
+
#: the whole reason the table-page factory exists ("a new table object gets its own
|
| 165 |
+
#: table_store.make('<its>_table_workspace')").
|
| 166 |
+
TABLE_KEY = 'product_table_workspace'
|
| 167 |
+
|
| 168 |
+
_GRID_FIELDS_PATH = Path(__file__).resolve().parent.parent / 'aios_grid_fields.json'
|
| 169 |
+
_SHARED_KEYS = None
|
| 170 |
+
|
| 171 |
+
|
| 172 |
+
def SHARED_KEYS():
|
| 173 |
+
"""The product columns whose values are TENANT-WIDE β derived from the canonical contract's
|
| 174 |
+
own `shared: true`, never typed out a second time.
|
| 175 |
+
|
| 176 |
+
β IT DELIBERATELY DOES NOT SWALLOW A READ FAILURE. Degrading to `()` would send a shared
|
| 177 |
+
write into the per-user stratum with nothing going wrong anywhere β the widening defect
|
| 178 |
+
reappearing silently, which is the one outcome this whole mechanism exists to prevent. If the
|
| 179 |
+
canonical contract is unreadable the product grid cannot render at all (`pd_fields` parses the
|
| 180 |
+
same file with no guard), so a raise here costs nothing that was still working.
|
| 181 |
+
"""
|
| 182 |
+
global _SHARED_KEYS
|
| 183 |
+
if _SHARED_KEYS is None:
|
| 184 |
+
doc = json.loads(_GRID_FIELDS_PATH.read_text(encoding='utf-8'))
|
| 185 |
+
_SHARED_KEYS = tuple(f['key'] for f in (doc.get('product_data') or {}).get('fields') or []
|
| 186 |
+
if f.get('shared'))
|
| 187 |
+
return _SHARED_KEYS
|
| 188 |
+
|
| 189 |
+
|
| 190 |
+
class _ProductTableStore(table_store.TableStore):
|
| 191 |
+
"""The product workspace, with the SHARED columns routed to the tenant-wide stratum.
|
| 192 |
+
|
| 193 |
+
ββ THIS SUBCLASS IS THE WHOLE OF W30-T36's WRITE PATH, AND THE REASON IT LIVES HERE RATHER
|
| 194 |
+
THAN AT A ROUTE IS THAT **THE BROWSER NEVER CALLS `PATCH /products/{pid}`** β measured, zero
|
| 195 |
+
call sites in `aios-web/web/src`. A cell edit travels `POST /grid/events` β `grid_events.
|
| 196 |
+
handle_one` β `_tops(ctx).patch_overlay(...)`, and `_tops` returns `ctx.table`, which
|
| 197 |
+
`routes_grid._ctx` sets to `TABLE_OPS` for the product scope. So this object IS the seam both
|
| 198 |
+
doors pass through; intercepting at either route would have left the other one writing a
|
| 199 |
+
per-user value that only its author could see.
|
| 200 |
+
|
| 201 |
+
β `st=self.st`, NEVER the module default. The shared stratum must resolve to the SAME store
|
| 202 |
+
handle as the per-user one it sits beside β `_tops`' own comment explains that a split, where
|
| 203 |
+
one side is tenant-scoped and the other is not, is worse than a stated residency error because
|
| 204 |
+
a user's value would vanish the moment they saved it. Reading `self.st` means both strata move
|
| 205 |
+
together the day that singleton gains a tenant handle.
|
| 206 |
+
"""
|
| 207 |
+
|
| 208 |
+
def patch_overlay(self, username, pid, updates):
|
| 209 |
+
clean = dict(updates or {})
|
| 210 |
+
if not clean:
|
| 211 |
+
return
|
| 212 |
+
keys = set(SHARED_KEYS())
|
| 213 |
+
shared = {k: v for k, v in clean.items() if k in keys}
|
| 214 |
+
personal = {k: v for k, v in clean.items() if k not in keys}
|
| 215 |
+
if shared:
|
| 216 |
+
shared_overlay.put_cells(TABLE_KEY, pid, shared, st=self.st)
|
| 217 |
+
if personal:
|
| 218 |
+
super().patch_overlay(username, pid, personal)
|
| 219 |
+
|
| 220 |
+
|
| 221 |
+
TABLE_OPS = _ProductTableStore(TABLE_KEY)
|
| 222 |
+
|
| 223 |
+
|
| 224 |
+
def shared_cells(pids, st=None):
|
| 225 |
+
"""`{"<pid>": {key: value}}` for the SHARED columns of the rows named by `pids`.
|
| 226 |
+
|
| 227 |
+
β `pids` is required and positional all the way down β `shared_overlay.cells` refuses to serve
|
| 228 |
+
"everything" by design, and the caller here always holds an already-scoped pool.
|
| 229 |
+
"""
|
| 230 |
+
return shared_overlay.cells(TABLE_KEY, pids, st=st if st is not None else TABLE_OPS.st)
|
| 231 |
+
|
| 232 |
+
|
| 233 |
+
def sku_pid(code):
|
| 234 |
+
"""A stable integer id for a SKU code. CRC32, masked to 31 bits so it is always positive and
|
| 235 |
+
always JSON-safe. Stable across processes and pulls, which an enumeration index is not."""
|
| 236 |
+
return zlib.crc32(str(code).encode("utf-8")) & 0x7FFFFFFF
|
| 237 |
+
|
| 238 |
+
|
| 239 |
+
def _assert_no_pid_collision(rows):
|
| 240 |
+
"""Two SKUs sharing a pid would MERGE in every set operation downstream β cohort membership,
|
| 241 |
+
allowed_pids, the measure channel β and nothing would report it. Fail the build instead."""
|
| 242 |
+
seen = {}
|
| 243 |
+
for r in rows:
|
| 244 |
+
prior = seen.get(r["pid"])
|
| 245 |
+
if prior is not None and prior != r["code"]:
|
| 246 |
+
raise ValueError(
|
| 247 |
+
f"product_data: pid collision β {prior!r} and {r['code']!r} both hash to "
|
| 248 |
+
f"{r['pid']}. Downstream set operations would merge them silently; widen the id "
|
| 249 |
+
f"before shipping this catalogue.")
|
| 250 |
+
seen[r["pid"]] = r["code"]
|
| 251 |
+
|
| 252 |
+
|
| 253 |
+
def _inventory_by_code(t):
|
| 254 |
+
"""`{code: {...}}` from the inventory module, or `{}` if it cannot be read.
|
| 255 |
+
|
| 256 |
+
Degrades to empty rather than raising, matching `customer_data._pool_build`'s treatment of
|
| 257 |
+
its own slow families: a product table that will not render because inventory is momentarily
|
| 258 |
+
unreachable is a worse failure than one with blank stock columns.
|
| 259 |
+
"""
|
| 260 |
+
try:
|
| 261 |
+
import modules.inventory as inventory
|
| 262 |
+
return inventory.sku_inventory(t=t) or {}
|
| 263 |
+
except Exception:
|
| 264 |
+
return {}
|
| 265 |
+
|
| 266 |
+
|
| 267 |
+
def _bu_ltm_share(t, team_id):
|
| 268 |
+
"""`{code: 0.0..1.0}` β this unit's SHARE of the SKU's last-twelve-months units.
|
| 269 |
+
|
| 270 |
+
β OWNER 2026-08-11: *"scope any inventory with Sales from Fisch"*. The velocity half of the
|
| 271 |
+
inventory block has to be re-shaped by a BU fact, and this is that fact.
|
| 272 |
+
|
| 273 |
+
β A SHARE, NOT THE UNIT COUNT ITSELF β AND THE REASON IS A MEASURED IMPOSSIBILITY. My first
|
| 274 |
+
version returned `products._sku_rev(...)['qty']` and used it directly as the scoped `qty_ltm`,
|
| 275 |
+
leaving the consolidated column on `inventory.sku_inventory`'s own figure. Two readers of one
|
| 276 |
+
question ([[one-question-two-normalizers]]): they disagree, and on 5 SKUs of 5,871 the live
|
| 277 |
+
check found **Fisch's LTM units EXCEEDING the company's** β a subset larger than its superset,
|
| 278 |
+
which no reader could explain and no reconciliation could survive.
|
| 279 |
+
|
| 280 |
+
Both halves of the ratio come from the SAME read here, so the share is in [0, 1] by
|
| 281 |
+
construction and the scoped figure can never exceed the consolidated one. It also leaves the
|
| 282 |
+
consolidated column exactly as it was β the Inventory page and the Product grid still agree
|
| 283 |
+
about company-wide units, which is what "leave the rest" asked for.
|
| 284 |
+
|
| 285 |
+
`all_qty == 0` implies `bu_qty == 0` (same reader), so a share of 0 is the honest answer for
|
| 286 |
+
"this unit never sold it": `_bucket` turns that into 'No recent sales', not zero cover.
|
| 287 |
+
|
| 288 |
+
Degrades to `{}` on any failure, matching `_inventory_by_code` β and a missing code then takes
|
| 289 |
+
the `_DEFAULT_SHARE` below rather than a silent 0.
|
| 290 |
+
"""
|
| 291 |
+
try:
|
| 292 |
+
f, to = P.ltm(t)
|
| 293 |
+
allq = {code: (r.get('qty') or 0.0)
|
| 294 |
+
for code, r in (products._sku_rev(f, to, None) or {}).items()}
|
| 295 |
+
buq = {code: (r.get('qty') or 0.0)
|
| 296 |
+
for code, r in (products._sku_rev(f, to, team_id) or {}).items()}
|
| 297 |
+
except Exception:
|
| 298 |
+
return {}
|
| 299 |
+
out = {}
|
| 300 |
+
for code, total in allq.items():
|
| 301 |
+
out[code] = min(1.0, max(0.0, (buq.get(code, 0.0) / total))) if total > 0 else 0.0
|
| 302 |
+
return out
|
| 303 |
+
|
| 304 |
+
|
| 305 |
+
#: What a SKU absent from the LTM sales read is worth to a business unit. ZERO β it did not sell
|
| 306 |
+
#: in anybody's book over the window, so no unit can claim its velocity. Named rather than
|
| 307 |
+
#: inlined so the choice is visible: the alternative (1.0, "assume it is all ours") would print
|
| 308 |
+
#: company-wide cover on a BU grid, which is the mixed-scope defect this whole rule avoids.
|
| 309 |
+
_DEFAULT_SHARE = 0.0
|
| 310 |
+
|
| 311 |
+
|
| 312 |
+
def _rescope_inventory(e, share):
|
| 313 |
+
"""`(qty_ltm, dos, bucket)` recomputed for ONE business unit's sales rate.
|
| 314 |
+
|
| 315 |
+
The formulas are `inventory.sku_inventory`'s, applied to a BU-shaped numerator β NOT a second
|
| 316 |
+
idea of what days-of-supply means. `_bucket` is imported from there for the same reason: two
|
| 317 |
+
copies of a threshold table is how the Product grid and the Inventory page start disagreeing
|
| 318 |
+
about which SKUs are dead.
|
| 319 |
+
|
| 320 |
+
β `on_hand` is whatever the warehouse holds, unscoped β so a Fisch reader's `dos` answers
|
| 321 |
+
"how long does ALL our stock last at Fisch's rate", which is the question a Fisch salesperson
|
| 322 |
+
actually has. It is deliberately NOT a pro-rated share of the shelf: there is no such shelf,
|
| 323 |
+
and inventing one would put a number on screen that no Odoo query could reproduce.
|
| 324 |
+
"""
|
| 325 |
+
on_hand = e.get("on_hand")
|
| 326 |
+
if on_hand is None:
|
| 327 |
+
return None, None, None # no inventory row for this SKU: blank, never zero
|
| 328 |
+
qty = float(e.get("qty_ltm") or 0.0) * float(share or 0.0)
|
| 329 |
+
daily = qty / 365.0
|
| 330 |
+
if daily > 0:
|
| 331 |
+
dos_raw = on_hand / daily
|
| 332 |
+
else:
|
| 333 |
+
dos_raw = float('inf') if on_hand > 0 else 0.0
|
| 334 |
+
try:
|
| 335 |
+
import modules.inventory as inventory
|
| 336 |
+
bucket = inventory._bucket(dos_raw, on_hand, qty)
|
| 337 |
+
except Exception:
|
| 338 |
+
bucket = None
|
| 339 |
+
return qty, (None if dos_raw == float('inf') else round(float(dos_raw), 0)), bucket
|
| 340 |
+
|
| 341 |
+
|
| 342 |
+
def _catalogue_by_code():
|
| 343 |
+
"""`{code: {'product', 'category'}}` β the CATALOGUE universe this pool is built from.
|
| 344 |
+
|
| 345 |
+
β UNLIKE `_inventory_by_code`, THIS ONE RAISES, and the asymmetry is the whole point.
|
| 346 |
+
Inventory degrades to `{}` because a product table with blank stock columns beats one that
|
| 347 |
+
will not render. The CATALOGUE is not a column β it is the ROW SET. A catalogue read that
|
| 348 |
+
failed quietly would drop the grid straight back to the sold-only 2,717 with every gate
|
| 349 |
+
green and nothing on screen saying so, which is the defect this seam exists to end
|
| 350 |
+
([[gate-can-report-green-on-nothing]]). `routes_products._pool_for` already turns the raise
|
| 351 |
+
into a 503 that names the cause, so the loud failure has somewhere honest to land.
|
| 352 |
+
|
| 353 |
+
It exists as a `pd`-level function rather than an inline `products.catalogue()` call for the
|
| 354 |
+
same reason `_inventory_by_code` does: it is the seam `verify_perm_scope`'s section H stubs
|
| 355 |
+
to build a pool without Odoo.
|
| 356 |
+
"""
|
| 357 |
+
return products.catalogue()
|
| 358 |
+
|
| 359 |
+
|
| 360 |
+
def _pricelist_by_code():
|
| 361 |
+
"""`({code: {price_*: price}}, report)` from `products.pricelist_by_code`, or `({}, β¦)`.
|
| 362 |
+
|
| 363 |
+
A SEAM for the same two reasons `_inventory_by_code` is one: it degrades rather than raises
|
| 364 |
+
(these are columns, not the row set), and `verify_perm_scope`'s section H stubs it to build a
|
| 365 |
+
pool without Odoo. β Unstubbed there, section H would reach live Odoo through the back door
|
| 366 |
+
and the whole file would stop being runnable offline.
|
| 367 |
+
"""
|
| 368 |
+
try:
|
| 369 |
+
return products.pricelist_by_code()
|
| 370 |
+
except Exception as e:
|
| 371 |
+
return {}, {"error": f"{type(e).__name__}: {str(e)[:200]}"}
|
| 372 |
+
|
| 373 |
+
|
| 374 |
+
def pool(team_id=None, t=None):
|
| 375 |
+
"""One row per ACTIVE product β the PRODUCT analogue of `customer_data.pool`.
|
| 376 |
+
|
| 377 |
+
CATALOGUE-FIRST, REVENUE LEFT-JOINED (R12 β see decision 4 in the module header). The row set
|
| 378 |
+
is `products.catalogue()`; `products.directory()` supplies the revenue columns for the SKUs
|
| 379 |
+
that sold in its window and contributes NO rows of its own.
|
| 380 |
+
|
| 381 |
+
`team_id` shapes the revenue columns exactly as it does for customers (`products.directory`
|
| 382 |
+
passes it into `_sku_rev`), which is why `core.perm_scope.derive_pool_scope` must keep
|
| 383 |
+
driving it rather than a post-filter deciding the BU. It does NOT shape the row set: a
|
| 384 |
+
catalogue has no team.
|
| 385 |
+
"""
|
| 386 |
+
t = t or P.today()
|
| 387 |
+
consolidated = team_id is None
|
| 388 |
+
# β OWNER 2026-08-11: read inventory on EVERY pull, not just a consolidated one. The stock
|
| 389 |
+
# itself is company-wide; only the sales-derived half is re-scoped, by `_rescope_inventory`.
|
| 390 |
+
inv = _inventory_by_code(t)
|
| 391 |
+
bu_share = {} if consolidated else _bu_ltm_share(t, team_id)
|
| 392 |
+
sup = supplier_master()
|
| 393 |
+
prices, _price_report = _pricelist_by_code()
|
| 394 |
+
cat = _catalogue_by_code()
|
| 395 |
+
# The LEFT side of the join, indexed by the same code key. β A code here that the catalogue
|
| 396 |
+
# does not carry belongs to a product no ACTIVE record claims β archived, and R12 keeps those
|
| 397 |
+
# out. `validate()` asserts that of Odoo rather than assuming it, and reports the revenue
|
| 398 |
+
# that therefore sits outside the grid (MEASURED 2026-08-11: 3 codes, $0.00 YTD / $294.50 LY).
|
| 399 |
+
rev = {r["code"]: r for r in products.directory(t=t, team_id=team_id)}
|
| 400 |
+
|
| 401 |
+
rows = []
|
| 402 |
+
for code, meta in cat.items():
|
| 403 |
+
r = rev.get(code)
|
| 404 |
+
row = {
|
| 405 |
+
"pid": sku_pid(code),
|
| 406 |
+
"code": code,
|
| 407 |
+
# β ONE source for the name and the category, the CATALOGUE β not the sale line's
|
| 408 |
+
# m2o. For a re-SKUed code the line's name can be the ARCHIVED record's; the active
|
| 409 |
+
# record's `display_name` is the current truth, and it is the same string for every
|
| 410 |
+
# SKU that is not re-SKUed. `directory()` derives the category identically.
|
| 411 |
+
"product": meta.get("product") or code,
|
| 412 |
+
"category": meta.get("category") or "(uncategorized)",
|
| 413 |
+
# ββ W33-T43 (R2 / amendment A2) β ODOO'S OWN PRODUCT ID, beside the hashed `pid`.
|
| 414 |
+
# R2 retires `ut_odoo_products` onto this key and keeps every data column it had; this
|
| 415 |
+
# is that column. β NOT DERIVABLE DOWNSTREAM: `pid` is `crc32(default_code)` here,
|
| 416 |
+
# unlike a customer row whose `pid` IS the partner id, so `aios_grid.py` cannot recover
|
| 417 |
+
# it and it has to be carried from `products.catalogue()`.
|
| 418 |
+
# β `None`, never 0, when the catalogue somehow has no id β 0 is a real Odoo id.
|
| 419 |
+
"product_id": meta.get("id"),
|
| 420 |
+
# β R12 β BLANK, NEVER $0, on a SKU no revenue read ever saw. A row that IS in `rev`
|
| 421 |
+
# keeps its measured numbers including a real 0.0 (it sold last year, not this one).
|
| 422 |
+
"rev_ytd": r.get("rev_ytd", 0.0) if r else None,
|
| 423 |
+
"rev_ly": r.get("rev_ly", 0.0) if r else None,
|
| 424 |
+
"yoy_pct": r.get("yoy_pct") if r else None,
|
| 425 |
+
"qty_ytd": r.get("qty_ytd", 0.0) if r else None,
|
| 426 |
+
"orders_ytd": r.get("orders_ytd", 0) if r else None,
|
| 427 |
+
}
|
| 428 |
+
# β WAVE 30 W30-T34 β the PRICELIST stratum, one column per declared list. Catalogue
|
| 429 |
+
# data like the supplier block, so it rides every pull, scoped or not: a price book is
|
| 430 |
+
# not a thing a business unit owns a slice of, and a Fisch reader who cannot see the
|
| 431 |
+
# Fisch price is exactly who this column is for.
|
| 432 |
+
#
|
| 433 |
+
# β `None`, NEVER 0 AND NEVER A FALLBACK, for the 1,883 SKUs no list prices. That is
|
| 434 |
+
# W29-T52's own negative control: the single `3_global` rule computes over `list_price`,
|
| 435 |
+
# which is 1.00 on 5,817 of 5,871 products, so the fallback is not a cheaper answer β
|
| 436 |
+
# it is a wrong one wearing a currency sign.
|
| 437 |
+
p = prices.get(code) or {}
|
| 438 |
+
for _col, _name in products.PRICELIST_COLUMNS:
|
| 439 |
+
row[_col] = p.get(_col)
|
| 440 |
+
# Wave 17 R3 β the supplier master, on every pull (it is catalogue data, not stock).
|
| 441 |
+
s = sup.get(code) or {}
|
| 442 |
+
row.update({
|
| 443 |
+
"supplier": s.get("supplier") or "(none)",
|
| 444 |
+
"lead_days": s.get("lead_days"),
|
| 445 |
+
"origin_country": s.get("origin_country") or "(none)",
|
| 446 |
+
"first_cost": s.get("first_cost"),
|
| 447 |
+
})
|
| 448 |
+
e = inv.get(code) or {}
|
| 449 |
+
if consolidated:
|
| 450 |
+
qty_ltm, dos, bucket = e.get("qty_ltm"), e.get("dos"), e.get("bucket")
|
| 451 |
+
else:
|
| 452 |
+
qty_ltm, dos, bucket = _rescope_inventory(e, bu_share.get(code, _DEFAULT_SHARE))
|
| 453 |
+
row.update({
|
| 454 |
+
# UNSCOPED on purpose (owner 2026-08-11): one warehouse, no per-brand shelf.
|
| 455 |
+
"on_hand": e.get("on_hand"),
|
| 456 |
+
"unit_cost": e.get("unit_cost"),
|
| 457 |
+
"inv_value": e.get("inv_value"),
|
| 458 |
+
# SCOPED: these three are functions of how fast THIS unit sells the SKU.
|
| 459 |
+
"qty_ltm": qty_ltm,
|
| 460 |
+
"dos": dos,
|
| 461 |
+
"stock_bucket": bucket,
|
| 462 |
+
})
|
| 463 |
+
# THE BUY TRIGGER (R3), ported from `modules/procurement`'s lead-time-cover rule:
|
| 464 |
+
# buy when the shelf runs out before a reorder could land.
|
| 465 |
+
#
|
| 466 |
+
# β BLANK, NOT "OK", WHEN EITHER INPUT IS MISSING. 339 SKUs have no supplier and 400
|
| 467 |
+
# no lead time; `dos` is null for anything that never sells through. "We don't know"
|
| 468 |
+
# and "you're fine" are different sentences, and only one of them is safe to print
|
| 469 |
+
# beside a purchasing decision.
|
| 470 |
+
#
|
| 471 |
+
# β `buy_now` SHIPS NOWHERE ANY MORE, and is computed anyway. OWNER 2026-08-03: it
|
| 472 |
+
# is not a preset field β it is a formula over the two columns beside it, and the
|
| 473 |
+
# platform has a formula field type for exactly that. Its Field is gone from
|
| 474 |
+
# `aios_grid_fields.json`, and `rows_from_pool` projects rows STRICTLY through that
|
| 475 |
+
# contract, so no Field means no cell on the wire. Nothing renders this key.
|
| 476 |
+
#
|
| 477 |
+
# It stays computed because deleting it would delete the PROOF. `validate()` below
|
| 478 |
+
# reconciles the formula's predicate against this one, row for row, which is the only
|
| 479 |
+
# thing that makes "the same exact figures" a claim rather than an assertion β and
|
| 480 |
+
# this repo has the scar already (wave 17: archiving `ar` "would have DELETED the
|
| 481 |
+
# proof"). Two comparisons per SKU is what that costs.
|
| 482 |
+
#
|
| 483 |
+
# β READS THE SCOPED `dos`, NOT `e['dos']`. On a Fisch pull the cover gap must answer
|
| 484 |
+
# "does the shelf outlast a reorder AT FISCH'S RATE" β reading the consolidated figure
|
| 485 |
+
# here would print a buy signal computed from both units' velocity beside a days-of-supply
|
| 486 |
+
# computed from one, and the two columns would disagree on the same row.
|
| 487 |
+
lead = s.get("lead_days")
|
| 488 |
+
if isinstance(dos, (int, float)) and isinstance(lead, (int, float)) and lead > 0:
|
| 489 |
+
row["cover_gap_d"] = int(round(dos - lead))
|
| 490 |
+
row["buy_now"] = "Buy now" if dos < lead else "OK"
|
| 491 |
+
else:
|
| 492 |
+
row["cover_gap_d"] = None
|
| 493 |
+
row["buy_now"] = None
|
| 494 |
+
rows.append(row)
|
| 495 |
+
|
| 496 |
+
# `or 0.0` reads a BLANK as zero FOR SORTING ONLY β never for the cell. A never-sold SKU
|
| 497 |
+
# settles among the zero-revenue ones at the bottom, which is where it belongs; the stored
|
| 498 |
+
# value stays None so nothing downstream can mistake "we never saw a sale" for "we measured
|
| 499 |
+
# nothing sold".
|
| 500 |
+
rows.sort(key=lambda r: -(r["rev_ytd"] or 0.0))
|
| 501 |
+
_assert_no_pid_collision(rows)
|
| 502 |
+
return rows
|
| 503 |
+
|
| 504 |
+
|
| 505 |
+
#: β THE BUY SIGNAL, AS THE FORMULA FIELD EVALUATES IT (owner, 2026-08-03).
|
| 506 |
+
#:
|
| 507 |
+
#: The owner's ruling was that "Buy signal" is not a preset field β it is a formula over two
|
| 508 |
+
#: columns the product table already carries, and the platform has a formula field type for it.
|
| 509 |
+
#: The formula, verbatim, is what `_seed_wave17.BUY_SIGNAL_FORMULA` creates and what the JSON
|
| 510 |
+
#: contract's `_product_removed_buy_now` note records:
|
| 511 |
+
#:
|
| 512 |
+
#: IF({lead_days} > 0, IF({dos} < {lead_days}, "Buy now", "OK"), "")
|
| 513 |
+
#:
|
| 514 |
+
#: This function is a PORT of how `customer-grid/formulaEngine.ts` evaluates that tree, not a
|
| 515 |
+
#: restatement of the business rule β that is the whole point, because the two could drift and
|
| 516 |
+
#: `validate()` is where the drift must show. Three engine behaviours it reproduces exactly:
|
| 517 |
+
#:
|
| 518 |
+
#: Β· a `ref` to a missing/non-numeric cell is None (`case "ref"` returns null for anything
|
| 519 |
+
#: that is not a finite number or a string);
|
| 520 |
+
#: Β· `cmp` returns BLANK unless BOTH sides read as numbers β it never coerces a blank to 0,
|
| 521 |
+
#: which is the difference between this and a filter engine's `toNum(null) === 0`;
|
| 522 |
+
#: Β· `IF` with a non-boolean condition returns BLANK ("no truthiness"), so a blank comparison
|
| 523 |
+
#: propagates out as a blank cell rather than taking the false branch.
|
| 524 |
+
BUY_SIGNAL_FORMULA = 'IF({lead_days} > 0, IF({dos} < {lead_days}, "Buy now", "OK"), "")'
|
| 525 |
+
|
| 526 |
+
|
| 527 |
+
def _buy_signal_formula(row):
|
| 528 |
+
"""Evaluate `BUY_SIGNAL_FORMULA` over one pool row -> 'Buy now' | 'OK' | '' (blank)."""
|
| 529 |
+
def num(v):
|
| 530 |
+
# The engine's `ref` + `asNumber`: booleans are not values here, and a non-finite
|
| 531 |
+
# number is null. `isinstance(True, int)` is True in Python, so bool is excluded first.
|
| 532 |
+
if isinstance(v, bool) or not isinstance(v, (int, float)):
|
| 533 |
+
return None
|
| 534 |
+
return v if v == v and v not in (float('inf'), float('-inf')) else None
|
| 535 |
+
|
| 536 |
+
lead, dos = num(row.get("lead_days")), num(row.get("dos"))
|
| 537 |
+
if lead is None: # `{lead_days} > 0` is blank -> IF(blank, β¦) is blank
|
| 538 |
+
return ""
|
| 539 |
+
if not lead > 0:
|
| 540 |
+
return "" # the formula's own else-branch
|
| 541 |
+
if dos is None: # `{dos} < {lead_days}` is blank -> IF(blank, β¦) is blank
|
| 542 |
+
return ""
|
| 543 |
+
return "Buy now" if dos < lead else "OK"
|
| 544 |
+
|
| 545 |
+
|
| 546 |
+
def validate(team_id=None, t=None):
|
| 547 |
+
"""Reconcile what SHIPS to an independent aggregate β the platform's own rule that a number
|
| 548 |
+
which does not tie to Odoo does not ship.
|
| 549 |
+
|
| 550 |
+
ββ THE POPULATION LEG IS NEW, AND IT IS HERE BECAUSE THIS FUNCTION USED TO BE SELF-SEALING
|
| 551 |
+
(wave 29, contract C8). It reconciled Ξ£ per-SKU YTD revenue against `products._sku_rev` β the
|
| 552 |
+
same function `pool()` built its rows from. BOTH SIDES CAME FROM THE JOIN, so the oracle could
|
| 553 |
+
not see a missing row, and a grid holding 2,717 of 5,875 active products passed for a wave.
|
| 554 |
+
No gate anywhere asserted a pool row COUNT against an independent Odoo `search_count` either.
|
| 555 |
+
That is the leg below, and it is the control that would have caught it
|
| 556 |
+
([[no-unverifiable-aggregates]]).
|
| 557 |
+
|
| 558 |
+
β The revenue leg was NOT deleted with it β it answers a different question ("did the join
|
| 559 |
+
lose money?") and is still the right shape for that one. What changed is that it now has to
|
| 560 |
+
account for revenue belonging to codes NO ACTIVE PRODUCT CARRIES, because R12 keeps archived
|
| 561 |
+
products out of the grid; the total is decomposed rather than compared loosely, so a growing
|
| 562 |
+
"outside the grid" figure shows up as a number rather than as slack in a tolerance.
|
| 563 |
+
|
| 564 |
+
Still deliberately NOT covered, and said rather than implied: the inventory columns are a
|
| 565 |
+
different module's reconciliation, and they are absent on a scoped pull anyway.
|
| 566 |
+
"""
|
| 567 |
+
t = t or P.today()
|
| 568 |
+
rows = pool(team_id=team_id, t=t)
|
| 569 |
+
|
| 570 |
+
# ββ THE POPULATION, AGAINST AN ORACLE THAT CANNOT SEE OUR JOIN. A bare Odoo count of active
|
| 571 |
+
# products, asked fresh β not a `len()` over anything this module or `products.catalogue()`
|
| 572 |
+
# built. NC: drop one row from `pool()` and this goes red; that is the whole point of it.
|
| 573 |
+
n_active = products.catalogue_count()
|
| 574 |
+
checks = [{
|
| 575 |
+
"check": "the pool holds one row per ACTIVE product (R12) β row count == an INDEPENDENT "
|
| 576 |
+
"Odoo search_count('product.product', active=True)",
|
| 577 |
+
"ours": len(rows), "theirs": n_active, "ok": len(rows) == n_active,
|
| 578 |
+
}]
|
| 579 |
+
|
| 580 |
+
yf, yt = P.ytd(t)
|
| 581 |
+
sku_rev = products._sku_rev(yf, yt, team_id) or {}
|
| 582 |
+
in_pool = {r["code"] for r in rows}
|
| 583 |
+
outside = {c: v for c, v in sku_rev.items() if c not in in_pool}
|
| 584 |
+
ours = round(sum(r["rev_ytd"] or 0.0 for r in rows), 2)
|
| 585 |
+
theirs = round(sum(v.get("rev", 0.0) for v in sku_rev.values()), 2)
|
| 586 |
+
dropped = round(sum(v.get("rev", 0.0) for v in outside.values()), 2)
|
| 587 |
+
checks.append({
|
| 588 |
+
"check": "Ξ£ per-SKU YTD revenue in the grid + Ξ£ YTD revenue of codes no ACTIVE product "
|
| 589 |
+
"carries == modules.products._sku_rev over the same window",
|
| 590 |
+
"ours": round(ours + dropped, 2), "theirs": theirs,
|
| 591 |
+
"ok": abs((ours + dropped) - theirs) < 0.01,
|
| 592 |
+
"detail": {"in_the_grid": ours, "outside_the_grid": dropped,
|
| 593 |
+
"codes_outside": len(outside)},
|
| 594 |
+
})
|
| 595 |
+
# β AND THE EXCLUSION IS ASSERTED, NOT ASSUMED. "Everything I dropped was archived" is
|
| 596 |
+
# trivially true when checked against the dict that did the dropping β that is the
|
| 597 |
+
# self-sealing shape all over again. So it is asked of ODOO. A code here that a live ACTIVE
|
| 598 |
+
# product carries means `products.catalogue()` missed a row that has revenue, which is the
|
| 599 |
+
# 2,717 defect returning in a smaller costume.
|
| 600 |
+
# β `pid:N` keys are not `default_code`s and are skipped: an UNCODED active product is in the
|
| 601 |
+
# catalogue under its own `pid:N` key and therefore cannot be in `outside` at all.
|
| 602 |
+
coded_outside = sorted(c for c in outside if not str(c).startswith("pid:"))
|
| 603 |
+
still_active = (O.get_odoo().search_count(
|
| 604 |
+
'product.product', [('active', '=', True), ('default_code', 'in', coded_outside)])
|
| 605 |
+
if coded_outside else 0)
|
| 606 |
+
checks.append({
|
| 607 |
+
"check": "every SKU with revenue but NO grid row is genuinely ARCHIVED (R12: archived "
|
| 608 |
+
"stay out) β asked of Odoo, never of the catalogue that did the dropping",
|
| 609 |
+
"ours": still_active, "theirs": 0, "ok": still_active == 0,
|
| 610 |
+
"detail": {"codes": coded_outside[:10], "n_codes": len(outside)},
|
| 611 |
+
})
|
| 612 |
+
# ββ ββ WAVE 30 W30-T34 β THE PRICELIST COLUMNS, AGAINST ORACLES THAT CANNOT SEE OUR JOIN ββ
|
| 613 |
+
#
|
| 614 |
+
# Three legs, because the column can fail in three different ways and only one of them is a
|
| 615 |
+
# count. `products.pricelist_by_code` builds a `{code: {col: price}}` map by expanding each
|
| 616 |
+
# rule over the products it names, IN PYTHON β that expansion is the fragile part, so every
|
| 617 |
+
# leg below re-asks ODOO instead of re-reading the map.
|
| 618 |
+
price_report = _pricelist_by_code()[1]
|
| 619 |
+
_pl_rows = O.search_read('product.pricelist', [], ['id', 'name'])
|
| 620 |
+
_pl_id = {}
|
| 621 |
+
for _p in _pl_rows:
|
| 622 |
+
_pl_id.setdefault(str(_p.get('name') or '').strip(), _p['id'])
|
| 623 |
+
_today = P.today().isoformat()
|
| 624 |
+
|
| 625 |
+
_targets_memo = {}
|
| 626 |
+
|
| 627 |
+
def _rule_targets(list_name):
|
| 628 |
+
"""(variant_ids, template_ids) a pricelist prices today β read FRESH from Odoo.
|
| 629 |
+
|
| 630 |
+
Memoised for the LIFE OF THIS CALL only: legs 1 and 2 both need all three lists, and
|
| 631 |
+
without this `validate()` makes six identical round trips instead of three (measured:
|
| 632 |
+
~48s of the run). Deliberately NOT an `lru_cache` β an oracle that survives the process
|
| 633 |
+
is an oracle reading yesterday's Odoo.
|
| 634 |
+
"""
|
| 635 |
+
if list_name in _targets_memo:
|
| 636 |
+
return _targets_memo[list_name]
|
| 637 |
+
var, tmpl = set(), set()
|
| 638 |
+
plid = _pl_id.get(list_name)
|
| 639 |
+
if plid is None:
|
| 640 |
+
return var, tmpl
|
| 641 |
+
for r in O.search_read(
|
| 642 |
+
'product.pricelist.item',
|
| 643 |
+
[('pricelist_id', '=', plid), ('compute_price', '=', 'fixed'),
|
| 644 |
+
('applied_on', 'in', ['0_product_variant', '1_product'])],
|
| 645 |
+
['product_id', 'product_tmpl_id', 'applied_on', 'fixed_price',
|
| 646 |
+
'date_start', 'date_end']):
|
| 647 |
+
ds, de = str(r.get('date_start') or '')[:10], str(r.get('date_end') or '')[:10]
|
| 648 |
+
if (ds and ds > _today) or (de and de < _today):
|
| 649 |
+
continue
|
| 650 |
+
fp = r.get('fixed_price')
|
| 651 |
+
if not isinstance(fp, (int, float)) or fp <= 0:
|
| 652 |
+
continue
|
| 653 |
+
if r.get('applied_on') == '0_product_variant' and r.get('product_id'):
|
| 654 |
+
var.add(O.m2o_id(r['product_id']))
|
| 655 |
+
elif r.get('product_tmpl_id'):
|
| 656 |
+
tmpl.add(O.m2o_id(r['product_tmpl_id']))
|
| 657 |
+
_targets_memo[list_name] = (var, tmpl)
|
| 658 |
+
return var, tmpl
|
| 659 |
+
|
| 660 |
+
# LEG 1 β COVERAGE PER LIST. Ours: cells we filled. Theirs: a bare Odoo `search_count` of
|
| 661 |
+
# ACTIVE products a fresh read of that list's rules reaches. The rule set is shared (it IS
|
| 662 |
+
# the data) but the EXPANSION is not, and the expansion is what breaks.
|
| 663 |
+
for _col, _name in products.PRICELIST_COLUMNS:
|
| 664 |
+
_var, _tmpl = _rule_targets(_name)
|
| 665 |
+
_dom = [('active', '=', True), '|', ('id', 'in', sorted(_var)),
|
| 666 |
+
('product_tmpl_id', 'in', sorted(_tmpl))]
|
| 667 |
+
theirs = O.get_odoo().search_count('product.product', _dom) if (_var or _tmpl) else 0
|
| 668 |
+
ours = sum(1 for r in rows if isinstance(r.get(_col), (int, float)))
|
| 669 |
+
checks.append({
|
| 670 |
+
"check": f"{_name} pricelist: SKUs priced in the grid == an INDEPENDENT Odoo count "
|
| 671 |
+
f"of ACTIVE products its date-valid fixed rules reach",
|
| 672 |
+
"ours": ours, "theirs": theirs, "ok": ours == theirs,
|
| 673 |
+
"detail": {"column": _col, "variant_rules": len(_var), "template_rules": len(_tmpl)},
|
| 674 |
+
})
|
| 675 |
+
|
| 676 |
+
# LEG 2 β β THE TICKET'S OWN NEGATIVE CONTROL, AS A LEG. A SKU with no specific item must
|
| 677 |
+
# render BLANK. Asked of ODOO, never of the map that produced the blank β "everything I left
|
| 678 |
+
# empty was genuinely unpriced" is trivially true when checked against the dict that emptied
|
| 679 |
+
# it, which is the self-sealing shape the population leg above exists to end.
|
| 680 |
+
_unpriced = [r for r in rows
|
| 681 |
+
if not any(isinstance(r.get(c), (int, float))
|
| 682 |
+
for c, _n in products.PRICELIST_COLUMNS)]
|
| 683 |
+
_codes = {r["code"] for r in _unpriced}
|
| 684 |
+
_stray = 0
|
| 685 |
+
if _codes:
|
| 686 |
+
# Resolve those codes back to Odoo ids and ask whether ANY declared list prices them.
|
| 687 |
+
_ids, _tmpls = set(), set()
|
| 688 |
+
for _p in O.search_read('product.product',
|
| 689 |
+
[('active', '=', True),
|
| 690 |
+
('default_code', 'in', sorted(c for c in _codes
|
| 691 |
+
if not c.startswith("pid:")))],
|
| 692 |
+
['id', 'product_tmpl_id']):
|
| 693 |
+
_ids.add(_p['id'])
|
| 694 |
+
_tmpls.add(O.m2o_id(_p.get('product_tmpl_id')))
|
| 695 |
+
for _col, _name in products.PRICELIST_COLUMNS:
|
| 696 |
+
_v, _t = _rule_targets(_name)
|
| 697 |
+
_stray += len((_v & _ids) | ({t for t in _tmpls if t in _t}))
|
| 698 |
+
checks.append({
|
| 699 |
+
"check": "every SKU rendering a BLANK price is genuinely unpriced on all three declared "
|
| 700 |
+
"lists (W29-T52's NC: no fallback dressed as a price) β asked of Odoo",
|
| 701 |
+
"ours": _stray, "theirs": 0, "ok": _stray == 0,
|
| 702 |
+
"detail": {"blank_skus": len(_unpriced), "priced_skus": len(rows) - len(_unpriced)},
|
| 703 |
+
})
|
| 704 |
+
|
| 705 |
+
# LEG 3 β β R6's SECOND SENTENCE, AS A NUMBER. *"If there is lag or it can't be done, you
|
| 706 |
+
# need to explicitly tell me why and recommend a fix."* Everything the reader cannot see is
|
| 707 |
+
# counted here rather than dropped. The ASSERTION is the one thing that must never be true β
|
| 708 |
+
# a price cell that is present and not a positive number, i.e. a "free" SKU β while the
|
| 709 |
+
# rest rides `detail` so it is reported without reddening an honest day's data.
|
| 710 |
+
_bad = [r["code"] for r in rows
|
| 711 |
+
for c, _n in products.PRICELIST_COLUMNS
|
| 712 |
+
if r.get(c) is not None and not (isinstance(r.get(c), (int, float)) and r[c] > 0)]
|
| 713 |
+
# β MEASURED 2026-08-12 AND NOT FIXABLE FROM THIS FENCE: `aios_grid.rows_from_pool` sends
|
| 714 |
+
# every non-text field through `_round(v)` = `round(v)` with no ndigits, i.e. to a whole
|
| 715 |
+
# dollar. 52.1% of fixed prices carry cents and the median relative error is 5.26%, so this
|
| 716 |
+
# column ties to the cent HERE and ships rounded. Reported, never silently enforced.
|
| 717 |
+
_cents = sum(1 for r in rows for c, _n in products.PRICELIST_COLUMNS
|
| 718 |
+
if isinstance(r.get(c), (int, float)) and abs(r[c] - round(r[c])) > 1e-9)
|
| 719 |
+
_to_zero = sum(1 for r in rows for c, _n in products.PRICELIST_COLUMNS
|
| 720 |
+
if isinstance(r.get(c), (int, float)) and round(r[c]) == 0)
|
| 721 |
+
checks.append({
|
| 722 |
+
"check": "no price cell is present-but-not-a-positive-number (a 0 would read as FREE; "
|
| 723 |
+
"an unset rule must be blank), and what the reader cannot see is COUNTED",
|
| 724 |
+
"ours": len(_bad), "theirs": 0, "ok": not _bad,
|
| 725 |
+
"detail": {"bad_cells": _bad[:10], "reader_report": price_report,
|
| 726 |
+
"wire_rounding_loses_cents_on": _cents,
|
| 727 |
+
"wire_rounding_to_zero_dollars": _to_zero},
|
| 728 |
+
})
|
| 729 |
+
|
| 730 |
+
# WAVE 17 R3 β the BUY SIGNAL must be a total, exact partition of the catalogue, and every
|
| 731 |
+
# member of it must be re-derivable from the two columns beside it. A signal somebody buys
|
| 732 |
+
# stock on cannot be "mostly right": the failure that matters is a row that says OK because
|
| 733 |
+
# an input was missing, so the blank leg is checked as hard as the other two.
|
| 734 |
+
if team_id is None: # consolidated only β the inputs exist only there
|
| 735 |
+
buy = [r for r in rows if r.get("buy_now") == "Buy now"]
|
| 736 |
+
ok_rows = [r for r in rows if r.get("buy_now") == "OK"]
|
| 737 |
+
blank = [r for r in rows if r.get("buy_now") is None]
|
| 738 |
+
mis = sum(1 for r in buy if not (r["dos"] < r["lead_days"]))
|
| 739 |
+
mis += sum(1 for r in ok_rows if not (r["dos"] >= r["lead_days"]))
|
| 740 |
+
# A blank must be UNKNOWN β never a row we could have answered and quietly did not.
|
| 741 |
+
mis += sum(1 for r in blank
|
| 742 |
+
if isinstance(r.get("dos"), (int, float))
|
| 743 |
+
and isinstance(r.get("lead_days"), (int, float)) and r["lead_days"] > 0)
|
| 744 |
+
checks.append({
|
| 745 |
+
"check": "Buy signal partitions the catalogue (buy + ok + unknown == rows, none "
|
| 746 |
+
"misclassified)",
|
| 747 |
+
"ours": len(buy) + len(ok_rows) + len(blank) - mis, "theirs": len(rows),
|
| 748 |
+
"ok": mis == 0 and len(buy) + len(ok_rows) + len(blank) == len(rows),
|
| 749 |
+
"detail": {"buy_now": len(buy), "ok": len(ok_rows), "unknown": len(blank),
|
| 750 |
+
"misclassified": mis},
|
| 751 |
+
})
|
| 752 |
+
# β OWNER 2026-08-03 β "use the formula fields to come up to the same EXACT figures".
|
| 753 |
+
#
|
| 754 |
+
# This is that sentence, as a check. The retired preset column and the formula that
|
| 755 |
+
# replaces it must agree on every SKU, in all three states, or the replacement is not a
|
| 756 |
+
# replacement. `_buy_signal_formula` is a line-by-line port of the client formula
|
| 757 |
+
# engine's evaluation of
|
| 758 |
+
#
|
| 759 |
+
# IF({lead_days} > 0, IF({dos} < {lead_days}, "Buy now", "OK"), "")
|
| 760 |
+
#
|
| 761 |
+
# including the part that is easy to get wrong: a comparison against a BLANK is blank,
|
| 762 |
+
# never a coerced 0. (The client engine's `cmp` returns null unless both sides read as
|
| 763 |
+
# numbers, and `IF` refuses a non-boolean condition β so a missing `dos` yields "" and
|
| 764 |
+
# not "Buy now". A filter engine would have said `0 < 30` and swept in every SKU that
|
| 765 |
+
# never sells through; the formula engine does not, and this check is what holds it.)
|
| 766 |
+
#
|
| 767 |
+
# β It compares SETS OF SKUs, not counts. Two different partitions can share a shape.
|
| 768 |
+
disagree = sorted(r["code"] for r in rows
|
| 769 |
+
if (r.get("buy_now") or "") != _buy_signal_formula(r))
|
| 770 |
+
checks.append({
|
| 771 |
+
"check": 'Buy signal as a FORMULA field == the retired preset column, per SKU '
|
| 772 |
+
'(IF({lead_days} > 0, IF({dos} < {lead_days}, "Buy now", "OK"), ""))',
|
| 773 |
+
"ours": len(rows) - len(disagree), "theirs": len(rows),
|
| 774 |
+
"ok": not disagree,
|
| 775 |
+
"detail": {"disagreeing_skus": disagree[:10], "n_disagree": len(disagree)},
|
| 776 |
+
})
|
| 777 |
+
# The other half of "the same figures": the SHARED Buy list view can no longer filter on
|
| 778 |
+
# the retired column, and its replacement conditions must select the same SKUs. They are
|
| 779 |
+
# `dos isNotEmpty AND lead_days isNotEmpty AND lead_days > 0 AND dos < lead_days`
|
| 780 |
+
# (_seed_wave17.views), so this reproduces exactly that conjunction.
|
| 781 |
+
#
|
| 782 |
+
# β THE NEGATIVE CONTROL IS WHY THIS IS NOT `cover_gap_d < 0`, which reads like the
|
| 783 |
+
# obvious filter and is WRONG: `cover_gap_d` is `int(round(dos - lead))`, so a genuine
|
| 784 |
+
# gap of -0.4 days rounds to 0 and that SKU drops off a buy list it belongs on.
|
| 785 |
+
view_rows = {r["code"] for r in rows
|
| 786 |
+
if isinstance(r.get("dos"), (int, float))
|
| 787 |
+
and isinstance(r.get("lead_days"), (int, float))
|
| 788 |
+
and r["lead_days"] > 0 and r["dos"] < r["lead_days"]}
|
| 789 |
+
signal_rows = {r["code"] for r in buy}
|
| 790 |
+
rounding_would_miss = sorted(
|
| 791 |
+
c for c in signal_rows
|
| 792 |
+
if next((r for r in rows if r["code"] == c), {}).get("cover_gap_d") == 0)
|
| 793 |
+
checks.append({
|
| 794 |
+
"check": "Buy list view conditions (dos/lead_days, no retired column) select "
|
| 795 |
+
"exactly the Buy-now SKUs",
|
| 796 |
+
"ours": len(view_rows), "theirs": len(signal_rows),
|
| 797 |
+
"ok": view_rows == signal_rows,
|
| 798 |
+
"detail": {"only_in_view": sorted(view_rows - signal_rows)[:10],
|
| 799 |
+
"only_in_signal": sorted(signal_rows - view_rows)[:10],
|
| 800 |
+
# Reported, not asserted: how many SKUs a `cover_gap_d < 0` filter would
|
| 801 |
+
# have silently dropped. 0 today does not make that filter correct.
|
| 802 |
+
"cover_gap_rounds_to_zero": len(rounding_would_miss)},
|
| 803 |
+
})
|
| 804 |
+
return checks
|
platform/modules/products.py
CHANGED
|
@@ -1,737 +1,737 @@
|
|
| 1 |
-
"""Products / SKU module β SKU health: YoY movers (risers & decliners), zombie SKUs
|
| 2 |
-
(catalog rot β sellable, formerly selling, now dead), new winners, coverage collapse
|
| 3 |
-
(SKUs losing customer breadth β the FFS-recovery early-warning signal), and velocity leaders.
|
| 4 |
-
|
| 5 |
-
Per-SKU margin already lives in the Financial module (low_margin_skus); not duplicated here.
|
| 6 |
-
Basket / co-purchase (MBA) is intentionally deferred β the existing client app computes it
|
| 7 |
-
runtime-side, and a local co-occurrence pull over 74k LTM lines is the kind of heavy job the
|
| 8 |
-
project guardrails keep off the local PC. (See BACKLOG.)
|
| 9 |
-
|
| 10 |
-
All line-level (sale.order.line), RI+FFS scope, excluded accounts removed β reusing sale_line_domain.
|
| 11 |
-
Coverage (distinct customers per SKU) uses a 2-level read_group and is a touch slow (~15s/
|
| 12 |
-
window), so it's its own function the app calls lazily and caches.
|
| 13 |
-
"""
|
| 14 |
-
import sys
|
| 15 |
-
import datetime as dt
|
| 16 |
-
from pathlib import Path
|
| 17 |
-
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
| 18 |
-
from functools import lru_cache
|
| 19 |
-
import core.odoo as O
|
| 20 |
-
import core.periods as P
|
| 21 |
-
import modules.sales as sales_mod
|
| 22 |
-
|
| 23 |
-
# SKU-health views are about real products β exclude service/delivery pseudo-SKUs
|
| 24 |
-
# (Delivery Charges otherwise dominate movers/winners). Dot-path FILTER works (groupby
|
| 25 |
-
# on the dot-path does not). Validation reconciles on this same non-service universe.
|
| 26 |
-
_NO_SVC = [('product_id.type', '!=', 'service')]
|
| 27 |
-
|
| 28 |
-
|
| 29 |
-
@lru_cache(maxsize=1)
|
| 30 |
-
def _code_map():
|
| 31 |
-
"""product_id β SKU code. Multiple product records can share a default_code (re-SKUing
|
| 32 |
-
/duplicates); keying by code merges them so a re-coded item doesn't read as a fake
|
| 33 |
-
decliner + fake riser. Products without a code fall back to a per-id key. Archived
|
| 34 |
-
products are INCLUDED β re-SKUing typically archives the old record and creates a new
|
| 35 |
-
one under the same code, and the old record still carries last-year sales."""
|
| 36 |
-
prods = O.search_read('product.product', [('active', 'in', [True, False])], ['id', 'default_code'],
|
| 37 |
-
limit=50000)
|
| 38 |
-
return {p['id']: (str(p['default_code']).strip() if p.get('default_code') else f"pid:{p['id']}")
|
| 39 |
-
for p in prods}
|
| 40 |
-
|
| 41 |
-
|
| 42 |
-
def _sku_rev(date_from, date_to, team_id=None):
|
| 43 |
-
"""{sku_code: {'name','rev','qty','orders'}} over a window (services excluded,
|
| 44 |
-
duplicate product records merged by SKU code)."""
|
| 45 |
-
g = O.read_group('sale.order.line', O.sale_line_domain(date_from, date_to, team_id, extra=_NO_SVC),
|
| 46 |
-
['price_subtotal:sum', 'product_uom_qty:sum'], ['product_id'], lazy=False)
|
| 47 |
-
codes = _code_map()
|
| 48 |
-
out = {}
|
| 49 |
-
for r in g:
|
| 50 |
-
pid = O.m2o_id(r.get('product_id'))
|
| 51 |
-
if not pid:
|
| 52 |
-
continue
|
| 53 |
-
key = codes.get(pid, f"pid:{pid}")
|
| 54 |
-
e = out.setdefault(key, {'name': O.m2o_name(r.get('product_id')),
|
| 55 |
-
'rev': 0.0, 'qty': 0.0, 'orders': 0})
|
| 56 |
-
e['rev'] += r.get('price_subtotal') or 0.0
|
| 57 |
-
e['qty'] += r.get('product_uom_qty') or 0.0
|
| 58 |
-
e['orders'] += r.get('__count') or 0
|
| 59 |
-
return out
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
def yoy_movers(t=None, limit=20, team_id=None):
|
| 63 |
-
"""Top SKU risers and decliners by YTD-vs-same-period-LY revenue change."""
|
| 64 |
-
t = t or P.today()
|
| 65 |
-
yf, yt = P.ytd(t)
|
| 66 |
-
lf, lt = P.ytd_last_year(t)
|
| 67 |
-
this = _sku_rev(yf, yt, team_id)
|
| 68 |
-
last = _sku_rev(lf, lt, team_id)
|
| 69 |
-
rows = []
|
| 70 |
-
for pid in set(this) | set(last):
|
| 71 |
-
tr = this.get(pid, {'rev': 0.0, 'name': last.get(pid, {}).get('name', '')})
|
| 72 |
-
lr = last.get(pid, {'rev': 0.0})
|
| 73 |
-
name = this.get(pid, {}).get('name') or last.get(pid, {}).get('name') or ''
|
| 74 |
-
rows.append({'code': pid, 'product': name, 'rev_ytd': tr['rev'], 'rev_ly': lr['rev'],
|
| 75 |
-
'change': tr['rev'] - lr['rev']})
|
| 76 |
-
risers = sorted([r for r in rows if r['change'] > 0], key=lambda x: -x['change'])[:limit]
|
| 77 |
-
decliners = sorted([r for r in rows if r['change'] < 0], key=lambda x: x['change'])[:limit]
|
| 78 |
-
return {'risers': risers, 'decliners': decliners}
|
| 79 |
-
|
| 80 |
-
|
| 81 |
-
def zombie_skus(t=None, limit=30, min_prior=1500.0, team_id=None):
|
| 82 |
-
"""Catalog rot: SKUs that sold materially last year but are ~dead this year."""
|
| 83 |
-
t = t or P.today()
|
| 84 |
-
yf, yt = P.ytd(t)
|
| 85 |
-
lf, lt = P.ytd_last_year(t)
|
| 86 |
-
this = _sku_rev(yf, yt, team_id)
|
| 87 |
-
last = _sku_rev(lf, lt, team_id)
|
| 88 |
-
rows = []
|
| 89 |
-
for pid, lr in last.items():
|
| 90 |
-
if lr['rev'] < min_prior:
|
| 91 |
-
continue
|
| 92 |
-
tr = this.get(pid, {'rev': 0.0})
|
| 93 |
-
if tr['rev'] > 0.05 * lr['rev']: # still selling at >5% of prior β not a zombie
|
| 94 |
-
continue
|
| 95 |
-
rows.append({'code': pid, 'product': lr['name'], 'rev_ly': lr['rev'], 'rev_ytd': tr['rev'],
|
| 96 |
-
'lost': lr['rev'] - tr['rev']})
|
| 97 |
-
rows.sort(key=lambda x: -x['lost'])
|
| 98 |
-
return rows[:limit]
|
| 99 |
-
|
| 100 |
-
|
| 101 |
-
def new_winners(t=None, limit=20, min_this=1500.0, team_id=None):
|
| 102 |
-
"""SKUs that barely sold last year but are selling well this year."""
|
| 103 |
-
t = t or P.today()
|
| 104 |
-
yf, yt = P.ytd(t)
|
| 105 |
-
lf, lt = P.ytd_last_year(t)
|
| 106 |
-
this = _sku_rev(yf, yt, team_id)
|
| 107 |
-
last = _sku_rev(lf, lt, team_id)
|
| 108 |
-
rows = []
|
| 109 |
-
for pid, tr in this.items():
|
| 110 |
-
if tr['rev'] < min_this:
|
| 111 |
-
continue
|
| 112 |
-
lr = last.get(pid, {'rev': 0.0})
|
| 113 |
-
if lr['rev'] > 0.05 * tr['rev']:
|
| 114 |
-
continue
|
| 115 |
-
rows.append({'code': pid, 'product': tr['name'], 'rev_ytd': tr['rev'], 'rev_ly': lr['rev'],
|
| 116 |
-
'gained': tr['rev'] - lr['rev']})
|
| 117 |
-
rows.sort(key=lambda x: -x['gained'])
|
| 118 |
-
return rows[:limit]
|
| 119 |
-
|
| 120 |
-
|
| 121 |
-
def velocity_leaders(t=None, limit=25, team_id=None):
|
| 122 |
-
"""Top SKUs by LTM unit velocity (units/month) and revenue."""
|
| 123 |
-
t = t or P.today()
|
| 124 |
-
lf, lt = P.ltm(t)
|
| 125 |
-
sku = _sku_rev(lf, lt, team_id)
|
| 126 |
-
rows = [{'code': k, 'product': v['name'], 'units_ltm': v['qty'], 'units_per_mo': v['qty'] / 12.0,
|
| 127 |
-
'rev_ltm': v['rev'], 'orders_ltm': v['orders']} for k, v in sku.items()]
|
| 128 |
-
rows.sort(key=lambda x: -x['units_ltm'])
|
| 129 |
-
return rows[:limit]
|
| 130 |
-
|
| 131 |
-
|
| 132 |
-
def _code_category():
|
| 133 |
-
"""code -> category name. Built from _code_map (pid->code) + the sales category map
|
| 134 |
-
(pid->category); the first record carrying a code sets that code's category."""
|
| 135 |
-
codes = _code_map()
|
| 136 |
-
cats = sales_mod._product_cat()
|
| 137 |
-
out = {}
|
| 138 |
-
for pid, code in codes.items():
|
| 139 |
-
if code not in out and pid in cats:
|
| 140 |
-
out[code] = cats[pid]
|
| 141 |
-
return out
|
| 142 |
-
|
| 143 |
-
|
| 144 |
-
def directory(t=None, team_id=None):
|
| 145 |
-
"""Every SKU that sold this YTD or the same period last year, with the fields the SKU
|
| 146 |
-
drill-down filters/sorts on: category, YTD/LY revenue, YoY %, units and orders. Mirrors the
|
| 147 |
-
customer directory β the full list (filtering happens in the UI), sorted by YTD revenue."""
|
| 148 |
-
t = t or P.today()
|
| 149 |
-
yf, yt = P.ytd(t)
|
| 150 |
-
lf, lt = P.ytd_last_year(t)
|
| 151 |
-
this = _sku_rev(yf, yt, team_id)
|
| 152 |
-
last = _sku_rev(lf, lt, team_id)
|
| 153 |
-
code_cat = _code_category()
|
| 154 |
-
rows = []
|
| 155 |
-
for code in set(this) | set(last):
|
| 156 |
-
tr = this.get(code, {})
|
| 157 |
-
lr = last.get(code, {})
|
| 158 |
-
rev_ytd = tr.get('rev', 0.0)
|
| 159 |
-
rev_ly = lr.get('rev', 0.0)
|
| 160 |
-
rows.append({'code': code, 'product': tr.get('name') or lr.get('name') or code,
|
| 161 |
-
'category': code_cat.get(code, '(uncategorized)'),
|
| 162 |
-
'rev_ytd': rev_ytd, 'rev_ly': rev_ly, 'change': rev_ytd - rev_ly,
|
| 163 |
-
'yoy_pct': P.yoy_pct(rev_ytd, rev_ly),
|
| 164 |
-
'qty_ytd': tr.get('qty', 0.0), 'orders_ytd': tr.get('orders', 0)})
|
| 165 |
-
rows.sort(key=lambda r: -r['rev_ytd'])
|
| 166 |
-
return rows
|
| 167 |
-
|
| 168 |
-
|
| 169 |
-
def catalogue():
|
| 170 |
-
"""`{sku_code: {'product', 'category', 'id'}}` β EVERY ACTIVE product, sold or not.
|
| 171 |
-
|
| 172 |
-
β THIS IS THE CATALOGUE UNIVERSE, AND IT IS DELIBERATELY NOT `directory()`. `directory()`'s
|
| 173 |
-
row set IS the union of two revenue `read_group`s over `sale.order.line` (`:155`), so a SKU
|
| 174 |
-
that never sold cannot exist in it. That is CORRECT for its own callers β `yoy_movers`,
|
| 175 |
-
`zombie_skus`, `new_winners` and `categories` all legitimately want a sales-window universe β
|
| 176 |
-
and it is wrong for the PRODUCT GRID, which is a catalogue and was therefore showing 2,717 of
|
| 177 |
-
5,875 SKUs (wave 29, owner item 22 / ruling R12). The fix is this function plus a LEFT JOIN in
|
| 178 |
-
the consumer, never a window removed from `directory()`: removing the window alone lands at
|
| 179 |
-
3,327 (all-time-sold), because ~2,550 active SKUs have never sold in wholesale scope at all.
|
| 180 |
-
|
| 181 |
-
β NOT BU-SHAPED, and it cannot be: `product.product` carries no team. A catalogue is one
|
| 182 |
-
catalogue. `directory()` stays the BU-shaped half, which is why the two are joined rather than
|
| 183 |
-
merged β a scoped caller gets every SKU with ITS OWN revenue, blank where that BU never sold.
|
| 184 |
-
|
| 185 |
-
Keyed exactly like `_code_map()` β the `default_code`, or a `pid:N` fallback for the 33 active
|
| 186 |
-
records that carry none β so the join against `directory()` is code-for-code with no
|
| 187 |
-
normaliser. Names come from `display_name` (`[CODE] NAME`), which is what `O.m2o_name` yields
|
| 188 |
-
off a sale line, so a never-sold row wears the same format as a sold one.
|
| 189 |
-
|
| 190 |
-
β RAISES on a truncated read rather than returning a short catalogue. A silently short pull
|
| 191 |
-
would put the grid back at a plausible wrong number with every gate green β the exact failure
|
| 192 |
-
this function exists to end ([[no-unverifiable-aggregates]], and the same truncation guard
|
| 193 |
-
`modules/backorders.py:161` already uses).
|
| 194 |
-
"""
|
| 195 |
-
dom = [('active', '=', True)]
|
| 196 |
-
prods = O.search_read('product.product', dom, ['id', 'default_code', 'display_name', 'name'],
|
| 197 |
-
limit=50000)
|
| 198 |
-
n = O.get_odoo().search_count('product.product', dom)
|
| 199 |
-
if len(prods) != n:
|
| 200 |
-
raise ValueError(
|
| 201 |
-
f"products.catalogue: the product pull is TRUNCATED β read {len(prods)} rows against "
|
| 202 |
-
f"a search_count of {n}. A short catalogue renders as a plausible smaller grid with "
|
| 203 |
-
f"nothing reporting it; raise the limit before shipping this.")
|
| 204 |
-
code_cat = _code_category()
|
| 205 |
-
out = {}
|
| 206 |
-
for p in prods:
|
| 207 |
-
code = (str(p['default_code']).strip() if p.get('default_code') else f"pid:{p['id']}")
|
| 208 |
-
# First record wins, matching `_code_category`'s own convention. MEASURED 2026-08-11: zero
|
| 209 |
-
# active products share a code, so this branch is a guard, not a merge policy.
|
| 210 |
-
# ββ W33-T43 (R2 / amendment A2) β `id` RIDES THE ROW. Odoo's `product.product` id is
|
| 211 |
-
# already in hand (the `pid:{id}` fallback above uses it and then threw it away), and it is
|
| 212 |
-
# the ONE column `ut_odoo_products` had that `product_data` lacked. R2 merges onto the
|
| 213 |
-
# legacy key and keeps every data column, so retiring the twin without this would lose a
|
| 214 |
-
# real identifier rather than a navigational link.
|
| 215 |
-
# β A product's grid pid is `crc32(default_code)`, NOT the Odoo id β unlike a CUSTOMER row,
|
| 216 |
-
# whose pid IS the partner id. That asymmetry is exactly why this cannot be derived
|
| 217 |
-
# downstream in `aios_grid.py` and has to be carried from the source read.
|
| 218 |
-
out.setdefault(code, {'product': p.get('display_name') or p.get('name') or code,
|
| 219 |
-
'category': code_cat.get(code, '(uncategorized)'),
|
| 220 |
-
'id': p['id']})
|
| 221 |
-
return out
|
| 222 |
-
|
| 223 |
-
|
| 224 |
-
#: ββ WAVE 30 / W30-T34 (the carried W29-T52) β THE PRICELIST STRATUM, PER LIST.
|
| 225 |
-
#:
|
| 226 |
-
#: MEASURED LIVE 2026-08-12, and every one of these numbers shaped the design rather than
|
| 227 |
-
#: decorating it:
|
| 228 |
-
#:
|
| 229 |
-
#: Β· **Five pricelists, three of them material.** Date-valid fixed rules: Fisch 5,139,
|
| 230 |
-
#: Royal 1 2,702, Royal 2 2,605, Public Pricelist 17, Giftware Deals 1. At SKU level that
|
| 231 |
-
#: is Fisch 3,971 Β· Royal 1 1,920 Β· Royal 2 1,857 Β· Public 6 Β· Giftware 0.
|
| 232 |
-
#: Β· **3,988 of 5,871 active products carry a usable fixed rule; 1,883 carry NONE.**
|
| 233 |
-
#: Β· **1,853 SKUs are priced on all three lists AND THE LISTS DISAGREE** β median relative
|
| 234 |
-
#: spread 16%, max 82%. That is why this is three columns rather than one "the price" or a
|
| 235 |
-
#: low/high pair: neither can tell a Fisch rep what Fisch sells the SKU for, which is the
|
| 236 |
-
#: only question the column exists to answer.
|
| 237 |
-
#: Β· β **`list_price` is 1.00 on 5,817 of the 5,871** β so the single `3_global` rule
|
| 238 |
-
#: computed over it is meaningless, exactly as W29-T52 said. **This reader never falls
|
| 239 |
-
#: through to it**, and that is the ticket's own negative control: a SKU with no specific
|
| 240 |
-
#: item renders BLANK, never a fallback dressed as a price.
|
| 241 |
-
#:
|
| 242 |
-
#: β RESOLVED BY NAME, NOT BY ID. A pricelist renamed or deleted in Odoo makes its column go
|
| 243 |
-
#: blank and makes `report["lists_missing"]` name it β a hard-coded id would keep pointing at
|
| 244 |
-
#: whatever inherited it and mislabel every cell silently.
|
| 245 |
-
PRICELIST_COLUMNS = (
|
| 246 |
-
("price_fisch", "Fisch"),
|
| 247 |
-
("price_royal_1", "Royal 1"),
|
| 248 |
-
("price_royal_2", "Royal 2"),
|
| 249 |
-
)
|
| 250 |
-
|
| 251 |
-
|
| 252 |
-
def pricelist_by_code():
|
| 253 |
-
"""`({code: {column_key: price}}, report)` β the date-valid FIXED base-tier price for each
|
| 254 |
-
declared pricelist, keyed EXACTLY as `catalogue()` keys its rows.
|
| 255 |
-
|
| 256 |
-
β THE KEYING IS LOAD-BEARING, not a detail. `catalogue()` keys on `default_code` with a
|
| 257 |
-
`pid:{id}` fallback for the ~33 active records that carry none. Keying this map any other
|
| 258 |
-
way would leave those SKUs permanently blank while an oracle counting active products
|
| 259 |
-
counted them β a red gate on a working build, or worse, a silent hole nobody counts.
|
| 260 |
-
|
| 261 |
-
**The base tier, deliberately.** `pricecomp._tier_for` picks the highest `min_quantity` at
|
| 262 |
-
or below an order's quantity, because it is pricing a LINE that has one. A catalogue column
|
| 263 |
-
has no quantity in hand, so it takes the LOWEST `min_quantity` β the price at qty 1. Rules
|
| 264 |
-
above that break are a bulk price, and `report["qty_break_only"]` counts the SKUs whose only
|
| 265 |
-
rule sits on one (MEASURED: 10 rules of 10,464 carry a break at all).
|
| 266 |
-
|
| 267 |
-
**Variant rules beat template rules**, matching `pricecomp._tier_for`: a `0_product_variant`
|
| 268 |
-
rule is the more specific statement about this exact SKU.
|
| 269 |
-
|
| 270 |
-
β DEGRADES TO `({}, report)` on a read failure, matching `_inventory_by_code` rather than
|
| 271 |
-
`catalogue()`: these are COLUMNS, and a product grid that will not render because pricing is
|
| 272 |
-
momentarily unreachable is a worse failure than one with blank price columns. The blanks are
|
| 273 |
-
not silent β `product_data.validate()`'s coverage leg reconciles each column against a fresh
|
| 274 |
-
Odoo count and goes red at zero.
|
| 275 |
-
|
| 276 |
-
β **The report exists because R6's second sentence is law** (*"if it can't be done, you need
|
| 277 |
-
to explicitly tell me why and recommend a fix"*). Everything this reader CANNOT see is
|
| 278 |
-
counted rather than dropped: rules on pricelists the contract does not declare, non-fixed
|
| 279 |
-
(`formula`/`percent`) rules, the `3_global` fallback, and prices that only exist above a
|
| 280 |
-
quantity break.
|
| 281 |
-
"""
|
| 282 |
-
# β THE COUNTERS ARE DISJOINT AND ORDER-DEPENDENT, and saying so is the difference between a
|
| 283 |
-
# report and a misleading one. A rule is classified ONCE, by the first reason it is skipped:
|
| 284 |
-
# undeclared list β out of date β not fixed β global β zero price. So `rules_not_fixed: 0`
|
| 285 |
-
# means "no formula rule on a list we declare", NOT "this Odoo has no formula rules"
|
| 286 |
-
# (MEASURED 2026-08-12: it has exactly one, and it sits on Public Pricelist, which the
|
| 287 |
-
# contract does not declare β so it lands in `rules_undeclared_list`).
|
| 288 |
-
report = {"lists_missing": [], "rules_total": 0, "rules_undeclared_list": 0,
|
| 289 |
-
"rules_not_fixed": 0, "rules_global": 0, "rules_out_of_date": 0,
|
| 290 |
-
"rules_zero_price": 0, "qty_break_only": 0}
|
| 291 |
-
try:
|
| 292 |
-
pl_rows = O.search_read('product.pricelist', [], ['id', 'name'])
|
| 293 |
-
by_name = {}
|
| 294 |
-
for p in pl_rows:
|
| 295 |
-
by_name.setdefault(str(p.get('name') or '').strip(), p['id'])
|
| 296 |
-
wanted = {}
|
| 297 |
-
for col, name in PRICELIST_COLUMNS:
|
| 298 |
-
pid = by_name.get(name)
|
| 299 |
-
if pid is None:
|
| 300 |
-
report["lists_missing"].append(name)
|
| 301 |
-
else:
|
| 302 |
-
wanted[pid] = col
|
| 303 |
-
|
| 304 |
-
# ββ FILTERED SERVER-SIDE, AND THE REASON IS A MEASUREMENT, NOT A STYLE PREFERENCE.
|
| 305 |
-
# Reading all 10,464 rules with 9 fields and sorting them in Python costs **30.1s** on
|
| 306 |
-
# this connection; the same rules under a server-side domain with 6 fields cost
|
| 307 |
-
# **13.6s** (measured 2026-08-12, back to back). `pool()` runs this on a scope's
|
| 308 |
-
# first-ever build, so that 16.5s is 16.5s of somebody's page load.
|
| 309 |
-
#
|
| 310 |
-
# β The domain reproduces the Python predicate EXACTLY, and the null legs are the part
|
| 311 |
-
# that is easy to get wrong: an absent `date_start` is `False`, not a past date, so
|
| 312 |
-
# `('date_start','<=',today)` ALONE would drop every open-ended rule β which is almost
|
| 313 |
-
# all of them.
|
| 314 |
-
today = P.today().isoformat()
|
| 315 |
-
_declared = sorted(wanted)
|
| 316 |
-
_kinds = ['0_product_variant', '1_product']
|
| 317 |
-
_live = [('pricelist_id', 'in', _declared), ('compute_price', '=', 'fixed'),
|
| 318 |
-
('applied_on', 'in', _kinds),
|
| 319 |
-
'|', ('date_start', '=', False), ('date_start', '<=', today),
|
| 320 |
-
'|', ('date_end', '=', False), ('date_end', '>=', today),
|
| 321 |
-
('fixed_price', '>', 0)]
|
| 322 |
-
rules = O.search_read(
|
| 323 |
-
'product.pricelist.item', _live,
|
| 324 |
-
['pricelist_id', 'product_id', 'product_tmpl_id', 'applied_on', 'fixed_price',
|
| 325 |
-
'min_quantity']) if _declared else []
|
| 326 |
-
|
| 327 |
-
# β R6's SECOND SENTENCE, PAID FOR WITH `search_count` RATHER THAN A WIDER READ. Six
|
| 328 |
-
# counts cost ~1.2s together; the rules they count would cost 16s to read. What this
|
| 329 |
-
# reader cannot see is still REPORTED β it is just no longer transferred.
|
| 330 |
-
def _n(extra):
|
| 331 |
-
try:
|
| 332 |
-
return O.get_odoo().search_count('product.pricelist.item', extra)
|
| 333 |
-
except Exception:
|
| 334 |
-
return -1 # -1 reads as "not measured", never as zero
|
| 335 |
-
_dated = ['|', ('date_start', '=', False), ('date_start', '<=', today),
|
| 336 |
-
'|', ('date_end', '=', False), ('date_end', '>=', today)]
|
| 337 |
-
report["rules_total"] = _n([])
|
| 338 |
-
report["rules_undeclared_list"] = _n([('pricelist_id', 'not in', _declared)]) \
|
| 339 |
-
if _declared else report["rules_total"]
|
| 340 |
-
if _declared:
|
| 341 |
-
_on = [('pricelist_id', 'in', _declared)]
|
| 342 |
-
# `percent_price` and formula rules are read NOWHERE in this repo. A rule we cannot
|
| 343 |
-
# price is one the operator is TOLD about, never one that quietly becomes a blank.
|
| 344 |
-
report["rules_not_fixed"] = _n(_on + [('compute_price', '!=', 'fixed')])
|
| 345 |
-
report["rules_global"] = _n(_on + [('compute_price', '=', 'fixed'),
|
| 346 |
-
('applied_on', 'not in', _kinds)])
|
| 347 |
-
report["rules_out_of_date"] = (
|
| 348 |
-
_n(_on + [('compute_price', '=', 'fixed'), ('applied_on', 'in', _kinds)])
|
| 349 |
-
- _n(_on + [('compute_price', '=', 'fixed'),
|
| 350 |
-
('applied_on', 'in', _kinds)] + _dated))
|
| 351 |
-
# A zero price is not "free" β it is an unset rule. Blank says so; 0 does not.
|
| 352 |
-
report["rules_zero_price"] = _n(
|
| 353 |
-
_on + [('compute_price', '=', 'fixed'), ('applied_on', 'in', _kinds)]
|
| 354 |
-
+ _dated + [('fixed_price', '<=', 0)])
|
| 355 |
-
|
| 356 |
-
by_var, by_tmpl = {}, {}
|
| 357 |
-
for r in rules:
|
| 358 |
-
col = wanted[O.m2o_id(r.get('pricelist_id'))]
|
| 359 |
-
if r.get('applied_on') == '0_product_variant' and r.get('product_id'):
|
| 360 |
-
by_var.setdefault((col, O.m2o_id(r['product_id'])), []).append(r)
|
| 361 |
-
elif r.get('product_tmpl_id'):
|
| 362 |
-
by_tmpl.setdefault((col, O.m2o_id(r['product_tmpl_id'])), []).append(r)
|
| 363 |
-
|
| 364 |
-
dom = [('active', '=', True)]
|
| 365 |
-
prods = O.search_read('product.product', dom,
|
| 366 |
-
['id', 'default_code', 'product_tmpl_id'], limit=50000)
|
| 367 |
-
n = O.get_odoo().search_count('product.product', dom)
|
| 368 |
-
if len(prods) != n:
|
| 369 |
-
# Same guard, same reason as `catalogue()`: a short pull renders as a plausible
|
| 370 |
-
# smaller set of priced SKUs with nothing reporting it.
|
| 371 |
-
raise ValueError(
|
| 372 |
-
f"products.pricelist_by_code: the product pull is TRUNCATED β read {len(prods)} "
|
| 373 |
-
f"rows against a search_count of {n}.")
|
| 374 |
-
except Exception as e:
|
| 375 |
-
report["error"] = f"{type(e).__name__}: {str(e)[:200]}"
|
| 376 |
-
return {}, report
|
| 377 |
-
|
| 378 |
-
out = {}
|
| 379 |
-
for p in prods:
|
| 380 |
-
code = (str(p['default_code']).strip() if p.get('default_code') else f"pid:{p['id']}")
|
| 381 |
-
tmpl = O.m2o_id(p.get('product_tmpl_id'))
|
| 382 |
-
cells = {}
|
| 383 |
-
for col, _name in PRICELIST_COLUMNS:
|
| 384 |
-
cands = by_var.get((col, p['id'])) or by_tmpl.get((col, tmpl))
|
| 385 |
-
if not cands:
|
| 386 |
-
continue
|
| 387 |
-
base = min(cands, key=lambda r: r.get('min_quantity') or 0.0)
|
| 388 |
-
if (base.get('min_quantity') or 0.0) > 1.0:
|
| 389 |
-
report["qty_break_only"] += 1
|
| 390 |
-
cells[col] = base.get('fixed_price')
|
| 391 |
-
if cells:
|
| 392 |
-
out.setdefault(code, {}).update(cells)
|
| 393 |
-
return out, report
|
| 394 |
-
|
| 395 |
-
|
| 396 |
-
def catalogue_count():
|
| 397 |
-
"""The INDEPENDENT population oracle: Odoo's own count of active products.
|
| 398 |
-
|
| 399 |
-
Deliberately a bare `search_count` and not a `len()` over anything this module built β the
|
| 400 |
-
2,717 defect shipped silently for a wave because `product_data.validate()` derived BOTH sides
|
| 401 |
-
of its reconciliation from `_sku_rev`, so the oracle could never see a missing row.
|
| 402 |
-
"""
|
| 403 |
-
return O.get_odoo().search_count('product.product', [('active', '=', True)])
|
| 404 |
-
|
| 405 |
-
|
| 406 |
-
def categories(t=None, team_id=None):
|
| 407 |
-
"""Sorted distinct category names present in the SKU directory (for the drill-down filter)."""
|
| 408 |
-
return sorted({r['category'] for r in directory(t, team_id)})
|
| 409 |
-
|
| 410 |
-
|
| 411 |
-
def _coverage(date_from, date_to, team_id=None):
|
| 412 |
-
"""{sku_code: distinct_customer_count} via 2-level read_group (slow-ish), merged by code.
|
| 413 |
-
(A customer buying two records sharing a code can count twice; duplicates are rare, and
|
| 414 |
-
code-level avoids the bigger error of a re-SKUed product reading as full coverage loss.)"""
|
| 415 |
-
g = O.read_group('sale.order.line', O.sale_line_domain(date_from, date_to, team_id, extra=_NO_SVC),
|
| 416 |
-
['__count'], ['product_id', 'order_partner_id'], lazy=False)
|
| 417 |
-
codes = _code_map()
|
| 418 |
-
cov = {}
|
| 419 |
-
names = {}
|
| 420 |
-
for r in g:
|
| 421 |
-
pid = O.m2o_id(r.get('product_id'))
|
| 422 |
-
if not pid:
|
| 423 |
-
continue
|
| 424 |
-
key = codes.get(pid, f"pid:{pid}")
|
| 425 |
-
cov[key] = cov.get(key, 0) + 1
|
| 426 |
-
names.setdefault(key, O.m2o_name(r.get('product_id')))
|
| 427 |
-
return cov, names
|
| 428 |
-
|
| 429 |
-
|
| 430 |
-
def coverage_collapse(t=None, limit=25, min_prior_custs=8, team_id=None):
|
| 431 |
-
"""SKUs that lost the most customer breadth YoY β early warning a SKU is dying even if
|
| 432 |
-
revenue hasn't fully cratered yet."""
|
| 433 |
-
t = t or P.today()
|
| 434 |
-
yf, yt = P.ytd(t)
|
| 435 |
-
lf, lt = P.ytd_last_year(t)
|
| 436 |
-
cov_t, names_t = _coverage(yf, yt, team_id)
|
| 437 |
-
cov_l, names_l = _coverage(lf, lt, team_id)
|
| 438 |
-
rows = []
|
| 439 |
-
for pid, lc in cov_l.items():
|
| 440 |
-
if lc < min_prior_custs:
|
| 441 |
-
continue
|
| 442 |
-
tc = cov_t.get(pid, 0)
|
| 443 |
-
drop = lc - tc
|
| 444 |
-
if drop <= 0:
|
| 445 |
-
continue
|
| 446 |
-
rows.append({'code': pid, 'product': names_l.get(pid) or names_t.get(pid) or '',
|
| 447 |
-
'custs_ly': lc, 'custs_ytd': tc, 'lost_custs': drop,
|
| 448 |
-
'pct_drop': drop / lc * 100})
|
| 449 |
-
rows.sort(key=lambda x: (-x['lost_custs'], -x['pct_drop']))
|
| 450 |
-
return rows[:limit]
|
| 451 |
-
|
| 452 |
-
|
| 453 |
-
# ====================================================================== SKU DRAWER (mirror of customer drawer)
|
| 454 |
-
def _sku_product_ids(code):
|
| 455 |
-
"""All product.product ids sharing this SKU code (merged variants / archived records)."""
|
| 456 |
-
return [pid for pid, c in _code_map().items() if c == code]
|
| 457 |
-
|
| 458 |
-
|
| 459 |
-
def _sku_dom(pids, date_from, date_to, team_id=None):
|
| 460 |
-
return O.sale_line_domain(date_from, date_to, team_id, extra=[('product_id', 'in', pids)] + _NO_SVC)
|
| 461 |
-
|
| 462 |
-
|
| 463 |
-
def _sku_name_category(pids):
|
| 464 |
-
rows = O.search_read('product.product', [('id', 'in', pids)], ['name', 'categ_id'])
|
| 465 |
-
name = rows[0]['name'] if rows else '(unknown)'
|
| 466 |
-
catmap = sales_mod._product_cat()
|
| 467 |
-
main = next((catmap.get(p) for p in pids if catmap.get(p)), '(uncategorized)')
|
| 468 |
-
return name, main
|
| 469 |
-
|
| 470 |
-
|
| 471 |
-
def _sku_buyers(pids, date_from, date_to, team_id=None):
|
| 472 |
-
"""{partner_id: {'name','rev','qty'}} for buyers of this SKU over a window."""
|
| 473 |
-
g = O.read_group('sale.order.line', _sku_dom(pids, date_from, date_to, team_id),
|
| 474 |
-
['price_subtotal:sum', 'product_uom_qty:sum'], ['order_partner_id'], lazy=False)
|
| 475 |
-
out = {}
|
| 476 |
-
for r in g:
|
| 477 |
-
pid = O.m2o_id(r.get('order_partner_id'))
|
| 478 |
-
if pid:
|
| 479 |
-
out[pid] = {'name': O.m2o_name(r.get('order_partner_id')),
|
| 480 |
-
'rev': r.get('price_subtotal') or 0.0, 'qty': r.get('product_uom_qty') or 0.0}
|
| 481 |
-
return out
|
| 482 |
-
|
| 483 |
-
|
| 484 |
-
def sku_detail(code, t=None, team_id=None, n_months=13, allsku=None):
|
| 485 |
-
"""KPIs (rev/qty/buyers YoY, GM%), monthly trend, rank & % of BU for one SKU code.
|
| 486 |
-
Pass `allsku` (a cached _sku_rev YTD map) to skip the ~all-SKU rank read."""
|
| 487 |
-
t = t or P.today()
|
| 488 |
-
pids = _sku_product_ids(code)
|
| 489 |
-
if not pids:
|
| 490 |
-
return None
|
| 491 |
-
yf, yt = P.ytd(t)
|
| 492 |
-
lf, lt = P.ytd_last_year(t)
|
| 493 |
-
mf, mt = P.ltm(t)
|
| 494 |
-
name, category = _sku_name_category(pids)
|
| 495 |
-
|
| 496 |
-
def s(df, dtt, field='price_subtotal'):
|
| 497 |
-
return O.sum_field('sale.order.line', _sku_dom(pids, df, dtt, team_id), field)
|
| 498 |
-
rev_ytd, rev_ly = s(yf, yt), s(lf, lt)
|
| 499 |
-
qty_ytd, qty_ly = s(yf, yt, 'product_uom_qty'), s(lf, lt, 'product_uom_qty')
|
| 500 |
-
buyers_ytd = len(_sku_buyers(pids, yf, yt, team_id))
|
| 501 |
-
buyers_ly = len(_sku_buyers(pids, lf, lt, team_id))
|
| 502 |
-
g = O.read_group('sale.order.line', _sku_dom(pids, mf, mt, team_id),
|
| 503 |
-
['price_subtotal:sum', 'margin:sum'], [], lazy=False)
|
| 504 |
-
line_rev = (g[0].get('price_subtotal') if g else 0) or 0.0
|
| 505 |
-
margin = (g[0].get('margin') if g else 0) or 0.0
|
| 506 |
-
|
| 507 |
-
# Monthly trend: bin lines by their order's month in Python (dot-path month groupby is rejected
|
| 508 |
-
# on sale.order.line), in 2 reads instead of 26 point queries.
|
| 509 |
-
range_start = dt.date(t.year - 2, t.month, 1).isoformat()
|
| 510 |
-
lines = O.search_read('sale.order.line', _sku_dom(pids, range_start, t.isoformat(), team_id),
|
| 511 |
-
['price_subtotal', 'order_id'])
|
| 512 |
-
oids = list({O.m2o_id(line['order_id']) for line in lines if line.get('order_id')})
|
| 513 |
-
odate = {}
|
| 514 |
-
for i in range(0, len(oids), 1000):
|
| 515 |
-
for o in O.search_read('sale.order', [('id', 'in', oids[i:i + 1000])], ['date_order']):
|
| 516 |
-
if o.get('date_order'):
|
| 517 |
-
odate[o['id']] = str(o['date_order'])[:7]
|
| 518 |
-
mrev = {}
|
| 519 |
-
for line in lines:
|
| 520 |
-
ym = odate.get(O.m2o_id(line.get('order_id')))
|
| 521 |
-
if ym:
|
| 522 |
-
mrev[ym] = mrev.get(ym, 0.0) + (line.get('price_subtotal') or 0.0)
|
| 523 |
-
monthly = []
|
| 524 |
-
for ym, start, end in P.month_starts(n_months, t):
|
| 525 |
-
y, m = int(ym[:4]) - 1, int(ym[5:7])
|
| 526 |
-
monthly.append({'month': ym, 'revenue': mrev.get(ym, 0.0),
|
| 527 |
-
'revenue_ly': mrev.get(f'{y:04d}-{m:02d}', 0.0)})
|
| 528 |
-
|
| 529 |
-
allsku = allsku if allsku is not None else _sku_rev(yf, yt, team_id)
|
| 530 |
-
total = sum(v['rev'] for v in allsku.values()) or 1.0
|
| 531 |
-
rank = next((i + 1 for i, (c, _v) in enumerate(sorted(allsku.items(), key=lambda kv: -kv[1]['rev']))
|
| 532 |
-
if c == code), None)
|
| 533 |
-
return {
|
| 534 |
-
'code': code, 'name': name, 'category': category,
|
| 535 |
-
'rev_ytd': rev_ytd, 'rev_ly': rev_ly, 'rev_yoy_pct': P.yoy_pct(rev_ytd, rev_ly),
|
| 536 |
-
'qty_ytd': qty_ytd, 'qty_ly': qty_ly, 'qty_yoy_pct': P.yoy_pct(qty_ytd, qty_ly),
|
| 537 |
-
'buyers_ytd': buyers_ytd, 'buyers_ly': buyers_ly, 'buyers_delta': buyers_ytd - buyers_ly,
|
| 538 |
-
'gm_pct': (margin / line_rev * 100) if line_rev else 0.0, 'gm_dollars': margin,
|
| 539 |
-
'rank': rank, 'n_skus': len(allsku), 'pct_of_bu': rev_ytd / total * 100,
|
| 540 |
-
'monthly': monthly,
|
| 541 |
-
}
|
| 542 |
-
|
| 543 |
-
|
| 544 |
-
def sku_buyer_bridge(code, t=None, team_id=None, top=12):
|
| 545 |
-
"""Who drives the SKU's YoY: retained / new / churned buyers + a $-ranked churned call list."""
|
| 546 |
-
t = t or P.today()
|
| 547 |
-
pids = _sku_product_ids(code)
|
| 548 |
-
yf, yt = P.ytd(t)
|
| 549 |
-
lf, lt = P.ytd_last_year(t)
|
| 550 |
-
this, last = _sku_buyers(pids, yf, yt, team_id), _sku_buyers(pids, lf, lt, team_id)
|
| 551 |
-
tset, lset = set(this), set(last)
|
| 552 |
-
retained, new, churned = tset & lset, tset - lset, lset - tset
|
| 553 |
-
churned_list = sorted([{'pid': p, 'customer': last[p]['name'], 'ly_rev': last[p]['rev']}
|
| 554 |
-
for p in churned], key=lambda x: -x['ly_rev'])[:top]
|
| 555 |
-
return {'retained': {'n': len(retained), 'rev': sum(this[p]['rev'] for p in retained)},
|
| 556 |
-
'new': {'n': len(new), 'rev': sum(this[p]['rev'] for p in new)},
|
| 557 |
-
'churned': {'n': len(churned), 'rev': sum(last[p]['rev'] for p in churned)},
|
| 558 |
-
'buyer_retention_pct': (len(retained) / len(lset) * 100) if lset else 0.0,
|
| 559 |
-
'churned_buyers': churned_list, 'buyers_this': len(tset), 'buyers_last': len(lset),
|
| 560 |
-
'this_total': sum(v['rev'] for v in this.values())}
|
| 561 |
-
|
| 562 |
-
|
| 563 |
-
def sku_concentration(code, t=None, team_id=None):
|
| 564 |
-
"""Buyer-concentration risk: top-1/top-3 share, Herfindahl index, effective buyer count (LTM)."""
|
| 565 |
-
t = t or P.today()
|
| 566 |
-
mf, mt = P.ltm(t)
|
| 567 |
-
buyers = _sku_buyers(_sku_product_ids(code), mf, mt, team_id)
|
| 568 |
-
revs = sorted([v['rev'] for v in buyers.values()], reverse=True)
|
| 569 |
-
total = sum(revs) or 1.0
|
| 570 |
-
hhi = sum((r / total) ** 2 for r in revs)
|
| 571 |
-
return {'n_buyers': len(revs), 'top1_pct': (revs[0] / total * 100) if revs else 0.0,
|
| 572 |
-
'top3_pct': (sum(revs[:3]) / total * 100) if revs else 0.0,
|
| 573 |
-
'hhi': hhi, 'eff_buyers': (1 / hhi) if hhi else 0.0}
|
| 574 |
-
|
| 575 |
-
|
| 576 |
-
def sku_price_dispersion(code, t=None, team_id=None, cap=60):
|
| 577 |
-
"""Realized $/unit per buyer (LTM) vs the volume-weighted average; recoverable $ on below-VWAP
|
| 578 |
-
accounts. list_price is unreliable, so the dispersion among actual buyers is the margin lever."""
|
| 579 |
-
t = t or P.today()
|
| 580 |
-
mf, mt = P.ltm(t)
|
| 581 |
-
g = O.read_group('sale.order.line', _sku_dom(_sku_product_ids(code), mf, mt, team_id),
|
| 582 |
-
['price_subtotal:sum', 'product_uom_qty:sum'], ['order_partner_id'], lazy=False)
|
| 583 |
-
rows, tot_rev, tot_qty = [], 0.0, 0.0
|
| 584 |
-
for r in g:
|
| 585 |
-
pid = O.m2o_id(r.get('order_partner_id'))
|
| 586 |
-
rev = r.get('price_subtotal') or 0.0
|
| 587 |
-
qty = r.get('product_uom_qty') or 0.0
|
| 588 |
-
if not pid or qty <= 0:
|
| 589 |
-
continue
|
| 590 |
-
rows.append({'pid': pid, 'customer': O.m2o_name(r.get('order_partner_id')),
|
| 591 |
-
'price': rev / qty, 'qty': qty, 'rev': rev})
|
| 592 |
-
tot_rev += rev
|
| 593 |
-
tot_qty += qty
|
| 594 |
-
vwap = (tot_rev / tot_qty) if tot_qty else 0.0
|
| 595 |
-
for r in rows:
|
| 596 |
-
r['recoverable'] = max(0.0, vwap - r['price']) * r['qty']
|
| 597 |
-
rows.sort(key=lambda x: -x['recoverable'])
|
| 598 |
-
return {'vwap': vwap, 'n_buyers': len(rows),
|
| 599 |
-
'recoverable_total': sum(r['recoverable'] for r in rows), 'rows': rows[:cap]}
|
| 600 |
-
|
| 601 |
-
|
| 602 |
-
def sku_top_buyers(code, t=None, team_id=None, top=15):
|
| 603 |
-
"""Ranked buyers of this SKU (YTD) with YoY β clickable to open the customer drawer."""
|
| 604 |
-
t = t or P.today()
|
| 605 |
-
pids = _sku_product_ids(code)
|
| 606 |
-
yf, yt = P.ytd(t)
|
| 607 |
-
lf, lt = P.ytd_last_year(t)
|
| 608 |
-
this, last = _sku_buyers(pids, yf, yt, team_id), _sku_buyers(pids, lf, lt, team_id)
|
| 609 |
-
rows = [{'pid': p, 'customer': v['name'], 'rev': v['rev'], 'qty': v['qty'],
|
| 610 |
-
'yoy_pct': P.yoy_pct(v['rev'], last.get(p, {}).get('rev', 0.0))} for p, v in this.items()]
|
| 611 |
-
rows.sort(key=lambda x: -x['rev'])
|
| 612 |
-
return rows[:top]
|
| 613 |
-
|
| 614 |
-
|
| 615 |
-
_BUYER_TIERS = [('Whale (β₯$25k)', 25000.0), ('Large ($10β25k)', 10000.0),
|
| 616 |
-
('Mid ($2β10k)', 2000.0), ('Small (<$2k)', 0.0)]
|
| 617 |
-
|
| 618 |
-
|
| 619 |
-
def sku_customer_analysis(code, t=None, team_id=None, top=15):
|
| 620 |
-
"""WHO buys this SKU (LTM), as customers: the value-tier mix of its buyers (by each buyer's TOTAL
|
| 621 |
-
spend), and the accounts most DEPENDENT on it (this SKU as a share of their spend β who gets hurt
|
| 622 |
-
most if it stocks out / who to protect)."""
|
| 623 |
-
t = t or P.today()
|
| 624 |
-
lf, lt = P.ltm(t)
|
| 625 |
-
pids = _sku_product_ids(code)
|
| 626 |
-
empty = {'segments': [], 'dependency': [], 'n_buyers': 0, 'avg_dependency': 0.0}
|
| 627 |
-
if not pids:
|
| 628 |
-
return empty
|
| 629 |
-
buyers = _sku_buyers(pids, lf, lt, team_id) # {pid: {name, rev(this SKU), qty}}
|
| 630 |
-
if not buyers:
|
| 631 |
-
return empty
|
| 632 |
-
bpids = list(buyers)
|
| 633 |
-
g = O.read_group('sale.order', sales_mod.order_domain(lf, lt, team_id) + [('partner_id', 'in', bpids)],
|
| 634 |
-
['amount_untaxed:sum'], ['partner_id'], lazy=False)
|
| 635 |
-
total = {O.m2o_id(r['partner_id']): (r.get('amount_untaxed') or 0.0) for r in g if r.get('partner_id')}
|
| 636 |
-
|
| 637 |
-
def tier(rev):
|
| 638 |
-
for nm, lo in _BUYER_TIERS:
|
| 639 |
-
if rev >= lo:
|
| 640 |
-
return nm
|
| 641 |
-
return _BUYER_TIERS[-1][0]
|
| 642 |
-
seg = {nm: {'tier': nm, 'buyers': 0, 'sku_rev': 0.0} for nm, _ in _BUYER_TIERS}
|
| 643 |
-
dep = []
|
| 644 |
-
for p, v in buyers.items():
|
| 645 |
-
ct = total.get(p, v['rev']) or v['rev']
|
| 646 |
-
s = seg[tier(ct)]
|
| 647 |
-
s['buyers'] += 1
|
| 648 |
-
s['sku_rev'] += v['rev']
|
| 649 |
-
dep.append({'customer': v['name'], 'pid': p, 'sku_rev': v['rev'], 'cust_total': ct,
|
| 650 |
-
'dependency_pct': (v['rev'] / ct * 100) if ct else None})
|
| 651 |
-
tot = sum(s['sku_rev'] for s in seg.values()) or 1.0
|
| 652 |
-
segments = []
|
| 653 |
-
for nm, _ in _BUYER_TIERS:
|
| 654 |
-
s = seg[nm]
|
| 655 |
-
s['rev_share'] = s['sku_rev'] / tot * 100
|
| 656 |
-
s['avg_per_buyer'] = (s['sku_rev'] / s['buyers']) if s['buyers'] else 0.0
|
| 657 |
-
segments.append(s)
|
| 658 |
-
dep.sort(key=lambda x: -x['sku_rev'])
|
| 659 |
-
deps_known = [d['dependency_pct'] for d in dep if d['dependency_pct'] is not None]
|
| 660 |
-
return {'segments': segments, 'dependency': dep[:top], 'n_buyers': len(buyers),
|
| 661 |
-
'avg_dependency': (sum(deps_known) / len(deps_known)) if deps_known else 0.0}
|
| 662 |
-
|
| 663 |
-
|
| 664 |
-
def sku_whitespace(code, t=None, team_id=None, top=15):
|
| 665 |
-
"""Customers who buy this SKU's category but NOT this SKU β ranked prospect list."""
|
| 666 |
-
import modules.customers as cust_mod
|
| 667 |
-
t = t or P.today()
|
| 668 |
-
pids = _sku_product_ids(code)
|
| 669 |
-
_name, category = _sku_name_category(pids)
|
| 670 |
-
cat_buyers = cust_mod.category_buyers(category, team_id=team_id) or set()
|
| 671 |
-
mf, mt = P.ltm(t)
|
| 672 |
-
prospects = list(cat_buyers - set(_sku_buyers(pids, mf, mt, team_id)))
|
| 673 |
-
if not prospects:
|
| 674 |
-
return {'category': category, 'rows': []}
|
| 675 |
-
g = O.read_group('sale.order', sales_mod.order_domain(mf, mt, team_id) + [('partner_id', 'in', prospects)],
|
| 676 |
-
['amount_untaxed:sum'], ['partner_id'], lazy=False)
|
| 677 |
-
rows = [{'pid': O.m2o_id(r['partner_id']), 'customer': O.m2o_name(r['partner_id']),
|
| 678 |
-
'total_spend': r.get('amount_untaxed') or 0.0} for r in g if r.get('partner_id')]
|
| 679 |
-
rows.sort(key=lambda x: -x['total_spend'])
|
| 680 |
-
return {'category': category, 'rows': rows[:top]}
|
| 681 |
-
|
| 682 |
-
|
| 683 |
-
def sku_drawer_bundle(code, t=None, team_id=None, allsku=None):
|
| 684 |
-
"""The whole SKU drawer's first paint in one cached unit: sku_detail first (the not-found gate),
|
| 685 |
-
then the other six pulls CONCURRENTLY (O.parallel). Cuts a ~6-call cold open to ~max(call)."""
|
| 686 |
-
detail = sku_detail(code, t=t, team_id=team_id, allsku=allsku)
|
| 687 |
-
if detail is None:
|
| 688 |
-
return {'detail': None}
|
| 689 |
-
bridge, conc, price, buyers, white, ca = O.parallel([
|
| 690 |
-
lambda: sku_buyer_bridge(code, t, team_id),
|
| 691 |
-
lambda: sku_concentration(code, t, team_id),
|
| 692 |
-
lambda: sku_price_dispersion(code, t, team_id),
|
| 693 |
-
lambda: sku_top_buyers(code, t, team_id),
|
| 694 |
-
lambda: sku_whitespace(code, t, team_id),
|
| 695 |
-
lambda: sku_customer_analysis(code, t, team_id),
|
| 696 |
-
])
|
| 697 |
-
return {'detail': detail, 'bridge': bridge, 'conc': conc, 'price': price,
|
| 698 |
-
'buyers': buyers, 'white': white, 'ca': ca}
|
| 699 |
-
|
| 700 |
-
|
| 701 |
-
def validate(t=None, team_id=None):
|
| 702 |
-
"""Reconcile SKU metrics to independent Odoo aggregates. When team_id is set (a single BU
|
| 703 |
-
selected) every check runs SCOPED to that BU, so the validation panel never reconciles
|
| 704 |
-
against β or exposes β the other BU's numbers. All checks here are BU-scopeable (no cross-BU
|
| 705 |
-
mirror), so team_id threads straight through."""
|
| 706 |
-
t = t or P.today()
|
| 707 |
-
yf, yt = P.ytd(t)
|
| 708 |
-
checks = []
|
| 709 |
-
sku = _sku_rev(yf, yt, team_id)
|
| 710 |
-
sku_sum = sum(v['rev'] for v in sku.values())
|
| 711 |
-
line_total = O.sum_field('sale.order.line',
|
| 712 |
-
O.sale_line_domain(yf, yt, team_id, extra=_NO_SVC), 'price_subtotal')
|
| 713 |
-
checks.append({'check': 'SKU rev: Ξ£(per-SKU) == total line revenue, ex-services (YTD)',
|
| 714 |
-
'a': round(sku_sum, 2), 'b': round(line_total, 2),
|
| 715 |
-
'gap': round(sku_sum - line_total, 2),
|
| 716 |
-
'ok': abs(sku_sum - line_total) <= 1.0})
|
| 717 |
-
|
| 718 |
-
m = yoy_movers(t, limit=10**9, team_id=team_id)
|
| 719 |
-
movers_sum = sum(r['change'] for r in m['risers']) + sum(r['change'] for r in m['decliners'])
|
| 720 |
-
lf, lt = P.ytd_last_year(t)
|
| 721 |
-
last_total = O.sum_field('sale.order.line',
|
| 722 |
-
O.sale_line_domain(lf, lt, team_id, extra=_NO_SVC), 'price_subtotal')
|
| 723 |
-
checks.append({'check': 'SKU movers: Ξ£(Ξ) == (YTD β LY) total',
|
| 724 |
-
'a': round(movers_sum, 2), 'b': round(line_total - last_total, 2),
|
| 725 |
-
'gap': round(movers_sum - (line_total - last_total), 2),
|
| 726 |
-
'ok': abs(movers_sum - (line_total - last_total)) <= 1.0})
|
| 727 |
-
|
| 728 |
-
# SKU drawer: buyer-bridge reconciles β retained + new buyer revenue == this-period SKU revenue
|
| 729 |
-
top_code = max(sku.items(), key=lambda kv: kv[1]['rev'])[0] if sku else None
|
| 730 |
-
if top_code:
|
| 731 |
-
bb = sku_buyer_bridge(top_code, t, team_id=team_id)
|
| 732 |
-
recon = bb['retained']['rev'] + bb['new']['rev']
|
| 733 |
-
checks.append({'check': 'SKU drawer: retained+new buyer rev == SKU YTD revenue',
|
| 734 |
-
'a': round(recon, 2), 'b': round(bb['this_total'], 2),
|
| 735 |
-
'gap': round(recon - bb['this_total'], 2),
|
| 736 |
-
'ok': abs(recon - bb['this_total']) <= 1.0})
|
| 737 |
-
return checks
|
|
|
|
| 1 |
+
"""Products / SKU module β SKU health: YoY movers (risers & decliners), zombie SKUs
|
| 2 |
+
(catalog rot β sellable, formerly selling, now dead), new winners, coverage collapse
|
| 3 |
+
(SKUs losing customer breadth β the FFS-recovery early-warning signal), and velocity leaders.
|
| 4 |
+
|
| 5 |
+
Per-SKU margin already lives in the Financial module (low_margin_skus); not duplicated here.
|
| 6 |
+
Basket / co-purchase (MBA) is intentionally deferred β the existing client app computes it
|
| 7 |
+
runtime-side, and a local co-occurrence pull over 74k LTM lines is the kind of heavy job the
|
| 8 |
+
project guardrails keep off the local PC. (See BACKLOG.)
|
| 9 |
+
|
| 10 |
+
All line-level (sale.order.line), RI+FFS scope, excluded accounts removed β reusing sale_line_domain.
|
| 11 |
+
Coverage (distinct customers per SKU) uses a 2-level read_group and is a touch slow (~15s/
|
| 12 |
+
window), so it's its own function the app calls lazily and caches.
|
| 13 |
+
"""
|
| 14 |
+
import sys
|
| 15 |
+
import datetime as dt
|
| 16 |
+
from pathlib import Path
|
| 17 |
+
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
| 18 |
+
from functools import lru_cache
|
| 19 |
+
import core.odoo as O
|
| 20 |
+
import core.periods as P
|
| 21 |
+
import modules.sales as sales_mod
|
| 22 |
+
|
| 23 |
+
# SKU-health views are about real products β exclude service/delivery pseudo-SKUs
|
| 24 |
+
# (Delivery Charges otherwise dominate movers/winners). Dot-path FILTER works (groupby
|
| 25 |
+
# on the dot-path does not). Validation reconciles on this same non-service universe.
|
| 26 |
+
_NO_SVC = [('product_id.type', '!=', 'service')]
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
@lru_cache(maxsize=1)
|
| 30 |
+
def _code_map():
|
| 31 |
+
"""product_id β SKU code. Multiple product records can share a default_code (re-SKUing
|
| 32 |
+
/duplicates); keying by code merges them so a re-coded item doesn't read as a fake
|
| 33 |
+
decliner + fake riser. Products without a code fall back to a per-id key. Archived
|
| 34 |
+
products are INCLUDED β re-SKUing typically archives the old record and creates a new
|
| 35 |
+
one under the same code, and the old record still carries last-year sales."""
|
| 36 |
+
prods = O.search_read('product.product', [('active', 'in', [True, False])], ['id', 'default_code'],
|
| 37 |
+
limit=50000)
|
| 38 |
+
return {p['id']: (str(p['default_code']).strip() if p.get('default_code') else f"pid:{p['id']}")
|
| 39 |
+
for p in prods}
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def _sku_rev(date_from, date_to, team_id=None):
|
| 43 |
+
"""{sku_code: {'name','rev','qty','orders'}} over a window (services excluded,
|
| 44 |
+
duplicate product records merged by SKU code)."""
|
| 45 |
+
g = O.read_group('sale.order.line', O.sale_line_domain(date_from, date_to, team_id, extra=_NO_SVC),
|
| 46 |
+
['price_subtotal:sum', 'product_uom_qty:sum'], ['product_id'], lazy=False)
|
| 47 |
+
codes = _code_map()
|
| 48 |
+
out = {}
|
| 49 |
+
for r in g:
|
| 50 |
+
pid = O.m2o_id(r.get('product_id'))
|
| 51 |
+
if not pid:
|
| 52 |
+
continue
|
| 53 |
+
key = codes.get(pid, f"pid:{pid}")
|
| 54 |
+
e = out.setdefault(key, {'name': O.m2o_name(r.get('product_id')),
|
| 55 |
+
'rev': 0.0, 'qty': 0.0, 'orders': 0})
|
| 56 |
+
e['rev'] += r.get('price_subtotal') or 0.0
|
| 57 |
+
e['qty'] += r.get('product_uom_qty') or 0.0
|
| 58 |
+
e['orders'] += r.get('__count') or 0
|
| 59 |
+
return out
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
def yoy_movers(t=None, limit=20, team_id=None):
|
| 63 |
+
"""Top SKU risers and decliners by YTD-vs-same-period-LY revenue change."""
|
| 64 |
+
t = t or P.today()
|
| 65 |
+
yf, yt = P.ytd(t)
|
| 66 |
+
lf, lt = P.ytd_last_year(t)
|
| 67 |
+
this = _sku_rev(yf, yt, team_id)
|
| 68 |
+
last = _sku_rev(lf, lt, team_id)
|
| 69 |
+
rows = []
|
| 70 |
+
for pid in set(this) | set(last):
|
| 71 |
+
tr = this.get(pid, {'rev': 0.0, 'name': last.get(pid, {}).get('name', '')})
|
| 72 |
+
lr = last.get(pid, {'rev': 0.0})
|
| 73 |
+
name = this.get(pid, {}).get('name') or last.get(pid, {}).get('name') or ''
|
| 74 |
+
rows.append({'code': pid, 'product': name, 'rev_ytd': tr['rev'], 'rev_ly': lr['rev'],
|
| 75 |
+
'change': tr['rev'] - lr['rev']})
|
| 76 |
+
risers = sorted([r for r in rows if r['change'] > 0], key=lambda x: -x['change'])[:limit]
|
| 77 |
+
decliners = sorted([r for r in rows if r['change'] < 0], key=lambda x: x['change'])[:limit]
|
| 78 |
+
return {'risers': risers, 'decliners': decliners}
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
def zombie_skus(t=None, limit=30, min_prior=1500.0, team_id=None):
|
| 82 |
+
"""Catalog rot: SKUs that sold materially last year but are ~dead this year."""
|
| 83 |
+
t = t or P.today()
|
| 84 |
+
yf, yt = P.ytd(t)
|
| 85 |
+
lf, lt = P.ytd_last_year(t)
|
| 86 |
+
this = _sku_rev(yf, yt, team_id)
|
| 87 |
+
last = _sku_rev(lf, lt, team_id)
|
| 88 |
+
rows = []
|
| 89 |
+
for pid, lr in last.items():
|
| 90 |
+
if lr['rev'] < min_prior:
|
| 91 |
+
continue
|
| 92 |
+
tr = this.get(pid, {'rev': 0.0})
|
| 93 |
+
if tr['rev'] > 0.05 * lr['rev']: # still selling at >5% of prior β not a zombie
|
| 94 |
+
continue
|
| 95 |
+
rows.append({'code': pid, 'product': lr['name'], 'rev_ly': lr['rev'], 'rev_ytd': tr['rev'],
|
| 96 |
+
'lost': lr['rev'] - tr['rev']})
|
| 97 |
+
rows.sort(key=lambda x: -x['lost'])
|
| 98 |
+
return rows[:limit]
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
def new_winners(t=None, limit=20, min_this=1500.0, team_id=None):
|
| 102 |
+
"""SKUs that barely sold last year but are selling well this year."""
|
| 103 |
+
t = t or P.today()
|
| 104 |
+
yf, yt = P.ytd(t)
|
| 105 |
+
lf, lt = P.ytd_last_year(t)
|
| 106 |
+
this = _sku_rev(yf, yt, team_id)
|
| 107 |
+
last = _sku_rev(lf, lt, team_id)
|
| 108 |
+
rows = []
|
| 109 |
+
for pid, tr in this.items():
|
| 110 |
+
if tr['rev'] < min_this:
|
| 111 |
+
continue
|
| 112 |
+
lr = last.get(pid, {'rev': 0.0})
|
| 113 |
+
if lr['rev'] > 0.05 * tr['rev']:
|
| 114 |
+
continue
|
| 115 |
+
rows.append({'code': pid, 'product': tr['name'], 'rev_ytd': tr['rev'], 'rev_ly': lr['rev'],
|
| 116 |
+
'gained': tr['rev'] - lr['rev']})
|
| 117 |
+
rows.sort(key=lambda x: -x['gained'])
|
| 118 |
+
return rows[:limit]
|
| 119 |
+
|
| 120 |
+
|
| 121 |
+
def velocity_leaders(t=None, limit=25, team_id=None):
|
| 122 |
+
"""Top SKUs by LTM unit velocity (units/month) and revenue."""
|
| 123 |
+
t = t or P.today()
|
| 124 |
+
lf, lt = P.ltm(t)
|
| 125 |
+
sku = _sku_rev(lf, lt, team_id)
|
| 126 |
+
rows = [{'code': k, 'product': v['name'], 'units_ltm': v['qty'], 'units_per_mo': v['qty'] / 12.0,
|
| 127 |
+
'rev_ltm': v['rev'], 'orders_ltm': v['orders']} for k, v in sku.items()]
|
| 128 |
+
rows.sort(key=lambda x: -x['units_ltm'])
|
| 129 |
+
return rows[:limit]
|
| 130 |
+
|
| 131 |
+
|
| 132 |
+
def _code_category():
|
| 133 |
+
"""code -> category name. Built from _code_map (pid->code) + the sales category map
|
| 134 |
+
(pid->category); the first record carrying a code sets that code's category."""
|
| 135 |
+
codes = _code_map()
|
| 136 |
+
cats = sales_mod._product_cat()
|
| 137 |
+
out = {}
|
| 138 |
+
for pid, code in codes.items():
|
| 139 |
+
if code not in out and pid in cats:
|
| 140 |
+
out[code] = cats[pid]
|
| 141 |
+
return out
|
| 142 |
+
|
| 143 |
+
|
| 144 |
+
def directory(t=None, team_id=None):
|
| 145 |
+
"""Every SKU that sold this YTD or the same period last year, with the fields the SKU
|
| 146 |
+
drill-down filters/sorts on: category, YTD/LY revenue, YoY %, units and orders. Mirrors the
|
| 147 |
+
customer directory β the full list (filtering happens in the UI), sorted by YTD revenue."""
|
| 148 |
+
t = t or P.today()
|
| 149 |
+
yf, yt = P.ytd(t)
|
| 150 |
+
lf, lt = P.ytd_last_year(t)
|
| 151 |
+
this = _sku_rev(yf, yt, team_id)
|
| 152 |
+
last = _sku_rev(lf, lt, team_id)
|
| 153 |
+
code_cat = _code_category()
|
| 154 |
+
rows = []
|
| 155 |
+
for code in set(this) | set(last):
|
| 156 |
+
tr = this.get(code, {})
|
| 157 |
+
lr = last.get(code, {})
|
| 158 |
+
rev_ytd = tr.get('rev', 0.0)
|
| 159 |
+
rev_ly = lr.get('rev', 0.0)
|
| 160 |
+
rows.append({'code': code, 'product': tr.get('name') or lr.get('name') or code,
|
| 161 |
+
'category': code_cat.get(code, '(uncategorized)'),
|
| 162 |
+
'rev_ytd': rev_ytd, 'rev_ly': rev_ly, 'change': rev_ytd - rev_ly,
|
| 163 |
+
'yoy_pct': P.yoy_pct(rev_ytd, rev_ly),
|
| 164 |
+
'qty_ytd': tr.get('qty', 0.0), 'orders_ytd': tr.get('orders', 0)})
|
| 165 |
+
rows.sort(key=lambda r: -r['rev_ytd'])
|
| 166 |
+
return rows
|
| 167 |
+
|
| 168 |
+
|
| 169 |
+
def catalogue():
|
| 170 |
+
"""`{sku_code: {'product', 'category', 'id'}}` β EVERY ACTIVE product, sold or not.
|
| 171 |
+
|
| 172 |
+
β THIS IS THE CATALOGUE UNIVERSE, AND IT IS DELIBERATELY NOT `directory()`. `directory()`'s
|
| 173 |
+
row set IS the union of two revenue `read_group`s over `sale.order.line` (`:155`), so a SKU
|
| 174 |
+
that never sold cannot exist in it. That is CORRECT for its own callers β `yoy_movers`,
|
| 175 |
+
`zombie_skus`, `new_winners` and `categories` all legitimately want a sales-window universe β
|
| 176 |
+
and it is wrong for the PRODUCT GRID, which is a catalogue and was therefore showing 2,717 of
|
| 177 |
+
5,875 SKUs (wave 29, owner item 22 / ruling R12). The fix is this function plus a LEFT JOIN in
|
| 178 |
+
the consumer, never a window removed from `directory()`: removing the window alone lands at
|
| 179 |
+
3,327 (all-time-sold), because ~2,550 active SKUs have never sold in wholesale scope at all.
|
| 180 |
+
|
| 181 |
+
β NOT BU-SHAPED, and it cannot be: `product.product` carries no team. A catalogue is one
|
| 182 |
+
catalogue. `directory()` stays the BU-shaped half, which is why the two are joined rather than
|
| 183 |
+
merged β a scoped caller gets every SKU with ITS OWN revenue, blank where that BU never sold.
|
| 184 |
+
|
| 185 |
+
Keyed exactly like `_code_map()` β the `default_code`, or a `pid:N` fallback for the 33 active
|
| 186 |
+
records that carry none β so the join against `directory()` is code-for-code with no
|
| 187 |
+
normaliser. Names come from `display_name` (`[CODE] NAME`), which is what `O.m2o_name` yields
|
| 188 |
+
off a sale line, so a never-sold row wears the same format as a sold one.
|
| 189 |
+
|
| 190 |
+
β RAISES on a truncated read rather than returning a short catalogue. A silently short pull
|
| 191 |
+
would put the grid back at a plausible wrong number with every gate green β the exact failure
|
| 192 |
+
this function exists to end ([[no-unverifiable-aggregates]], and the same truncation guard
|
| 193 |
+
`modules/backorders.py:161` already uses).
|
| 194 |
+
"""
|
| 195 |
+
dom = [('active', '=', True)]
|
| 196 |
+
prods = O.search_read('product.product', dom, ['id', 'default_code', 'display_name', 'name'],
|
| 197 |
+
limit=50000)
|
| 198 |
+
n = O.get_odoo().search_count('product.product', dom)
|
| 199 |
+
if len(prods) != n:
|
| 200 |
+
raise ValueError(
|
| 201 |
+
f"products.catalogue: the product pull is TRUNCATED β read {len(prods)} rows against "
|
| 202 |
+
f"a search_count of {n}. A short catalogue renders as a plausible smaller grid with "
|
| 203 |
+
f"nothing reporting it; raise the limit before shipping this.")
|
| 204 |
+
code_cat = _code_category()
|
| 205 |
+
out = {}
|
| 206 |
+
for p in prods:
|
| 207 |
+
code = (str(p['default_code']).strip() if p.get('default_code') else f"pid:{p['id']}")
|
| 208 |
+
# First record wins, matching `_code_category`'s own convention. MEASURED 2026-08-11: zero
|
| 209 |
+
# active products share a code, so this branch is a guard, not a merge policy.
|
| 210 |
+
# ββ W33-T43 (R2 / amendment A2) β `id` RIDES THE ROW. Odoo's `product.product` id is
|
| 211 |
+
# already in hand (the `pid:{id}` fallback above uses it and then threw it away), and it is
|
| 212 |
+
# the ONE column `ut_odoo_products` had that `product_data` lacked. R2 merges onto the
|
| 213 |
+
# legacy key and keeps every data column, so retiring the twin without this would lose a
|
| 214 |
+
# real identifier rather than a navigational link.
|
| 215 |
+
# β A product's grid pid is `crc32(default_code)`, NOT the Odoo id β unlike a CUSTOMER row,
|
| 216 |
+
# whose pid IS the partner id. That asymmetry is exactly why this cannot be derived
|
| 217 |
+
# downstream in `aios_grid.py` and has to be carried from the source read.
|
| 218 |
+
out.setdefault(code, {'product': p.get('display_name') or p.get('name') or code,
|
| 219 |
+
'category': code_cat.get(code, '(uncategorized)'),
|
| 220 |
+
'id': p['id']})
|
| 221 |
+
return out
|
| 222 |
+
|
| 223 |
+
|
| 224 |
+
#: ββ WAVE 30 / W30-T34 (the carried W29-T52) β THE PRICELIST STRATUM, PER LIST.
|
| 225 |
+
#:
|
| 226 |
+
#: MEASURED LIVE 2026-08-12, and every one of these numbers shaped the design rather than
|
| 227 |
+
#: decorating it:
|
| 228 |
+
#:
|
| 229 |
+
#: Β· **Five pricelists, three of them material.** Date-valid fixed rules: Fisch 5,139,
|
| 230 |
+
#: Royal 1 2,702, Royal 2 2,605, Public Pricelist 17, Giftware Deals 1. At SKU level that
|
| 231 |
+
#: is Fisch 3,971 Β· Royal 1 1,920 Β· Royal 2 1,857 Β· Public 6 Β· Giftware 0.
|
| 232 |
+
#: Β· **3,988 of 5,871 active products carry a usable fixed rule; 1,883 carry NONE.**
|
| 233 |
+
#: Β· **1,853 SKUs are priced on all three lists AND THE LISTS DISAGREE** β median relative
|
| 234 |
+
#: spread 16%, max 82%. That is why this is three columns rather than one "the price" or a
|
| 235 |
+
#: low/high pair: neither can tell a Fisch rep what Fisch sells the SKU for, which is the
|
| 236 |
+
#: only question the column exists to answer.
|
| 237 |
+
#: Β· β **`list_price` is 1.00 on 5,817 of the 5,871** β so the single `3_global` rule
|
| 238 |
+
#: computed over it is meaningless, exactly as W29-T52 said. **This reader never falls
|
| 239 |
+
#: through to it**, and that is the ticket's own negative control: a SKU with no specific
|
| 240 |
+
#: item renders BLANK, never a fallback dressed as a price.
|
| 241 |
+
#:
|
| 242 |
+
#: β RESOLVED BY NAME, NOT BY ID. A pricelist renamed or deleted in Odoo makes its column go
|
| 243 |
+
#: blank and makes `report["lists_missing"]` name it β a hard-coded id would keep pointing at
|
| 244 |
+
#: whatever inherited it and mislabel every cell silently.
|
| 245 |
+
PRICELIST_COLUMNS = (
|
| 246 |
+
("price_fisch", "Fisch"),
|
| 247 |
+
("price_royal_1", "Royal 1"),
|
| 248 |
+
("price_royal_2", "Royal 2"),
|
| 249 |
+
)
|
| 250 |
+
|
| 251 |
+
|
| 252 |
+
def pricelist_by_code():
|
| 253 |
+
"""`({code: {column_key: price}}, report)` β the date-valid FIXED base-tier price for each
|
| 254 |
+
declared pricelist, keyed EXACTLY as `catalogue()` keys its rows.
|
| 255 |
+
|
| 256 |
+
β THE KEYING IS LOAD-BEARING, not a detail. `catalogue()` keys on `default_code` with a
|
| 257 |
+
`pid:{id}` fallback for the ~33 active records that carry none. Keying this map any other
|
| 258 |
+
way would leave those SKUs permanently blank while an oracle counting active products
|
| 259 |
+
counted them β a red gate on a working build, or worse, a silent hole nobody counts.
|
| 260 |
+
|
| 261 |
+
**The base tier, deliberately.** `pricecomp._tier_for` picks the highest `min_quantity` at
|
| 262 |
+
or below an order's quantity, because it is pricing a LINE that has one. A catalogue column
|
| 263 |
+
has no quantity in hand, so it takes the LOWEST `min_quantity` β the price at qty 1. Rules
|
| 264 |
+
above that break are a bulk price, and `report["qty_break_only"]` counts the SKUs whose only
|
| 265 |
+
rule sits on one (MEASURED: 10 rules of 10,464 carry a break at all).
|
| 266 |
+
|
| 267 |
+
**Variant rules beat template rules**, matching `pricecomp._tier_for`: a `0_product_variant`
|
| 268 |
+
rule is the more specific statement about this exact SKU.
|
| 269 |
+
|
| 270 |
+
β DEGRADES TO `({}, report)` on a read failure, matching `_inventory_by_code` rather than
|
| 271 |
+
`catalogue()`: these are COLUMNS, and a product grid that will not render because pricing is
|
| 272 |
+
momentarily unreachable is a worse failure than one with blank price columns. The blanks are
|
| 273 |
+
not silent β `product_data.validate()`'s coverage leg reconciles each column against a fresh
|
| 274 |
+
Odoo count and goes red at zero.
|
| 275 |
+
|
| 276 |
+
β **The report exists because R6's second sentence is law** (*"if it can't be done, you need
|
| 277 |
+
to explicitly tell me why and recommend a fix"*). Everything this reader CANNOT see is
|
| 278 |
+
counted rather than dropped: rules on pricelists the contract does not declare, non-fixed
|
| 279 |
+
(`formula`/`percent`) rules, the `3_global` fallback, and prices that only exist above a
|
| 280 |
+
quantity break.
|
| 281 |
+
"""
|
| 282 |
+
# β THE COUNTERS ARE DISJOINT AND ORDER-DEPENDENT, and saying so is the difference between a
|
| 283 |
+
# report and a misleading one. A rule is classified ONCE, by the first reason it is skipped:
|
| 284 |
+
# undeclared list β out of date β not fixed β global β zero price. So `rules_not_fixed: 0`
|
| 285 |
+
# means "no formula rule on a list we declare", NOT "this Odoo has no formula rules"
|
| 286 |
+
# (MEASURED 2026-08-12: it has exactly one, and it sits on Public Pricelist, which the
|
| 287 |
+
# contract does not declare β so it lands in `rules_undeclared_list`).
|
| 288 |
+
report = {"lists_missing": [], "rules_total": 0, "rules_undeclared_list": 0,
|
| 289 |
+
"rules_not_fixed": 0, "rules_global": 0, "rules_out_of_date": 0,
|
| 290 |
+
"rules_zero_price": 0, "qty_break_only": 0}
|
| 291 |
+
try:
|
| 292 |
+
pl_rows = O.search_read('product.pricelist', [], ['id', 'name'])
|
| 293 |
+
by_name = {}
|
| 294 |
+
for p in pl_rows:
|
| 295 |
+
by_name.setdefault(str(p.get('name') or '').strip(), p['id'])
|
| 296 |
+
wanted = {}
|
| 297 |
+
for col, name in PRICELIST_COLUMNS:
|
| 298 |
+
pid = by_name.get(name)
|
| 299 |
+
if pid is None:
|
| 300 |
+
report["lists_missing"].append(name)
|
| 301 |
+
else:
|
| 302 |
+
wanted[pid] = col
|
| 303 |
+
|
| 304 |
+
# ββ FILTERED SERVER-SIDE, AND THE REASON IS A MEASUREMENT, NOT A STYLE PREFERENCE.
|
| 305 |
+
# Reading all 10,464 rules with 9 fields and sorting them in Python costs **30.1s** on
|
| 306 |
+
# this connection; the same rules under a server-side domain with 6 fields cost
|
| 307 |
+
# **13.6s** (measured 2026-08-12, back to back). `pool()` runs this on a scope's
|
| 308 |
+
# first-ever build, so that 16.5s is 16.5s of somebody's page load.
|
| 309 |
+
#
|
| 310 |
+
# β The domain reproduces the Python predicate EXACTLY, and the null legs are the part
|
| 311 |
+
# that is easy to get wrong: an absent `date_start` is `False`, not a past date, so
|
| 312 |
+
# `('date_start','<=',today)` ALONE would drop every open-ended rule β which is almost
|
| 313 |
+
# all of them.
|
| 314 |
+
today = P.today().isoformat()
|
| 315 |
+
_declared = sorted(wanted)
|
| 316 |
+
_kinds = ['0_product_variant', '1_product']
|
| 317 |
+
_live = [('pricelist_id', 'in', _declared), ('compute_price', '=', 'fixed'),
|
| 318 |
+
('applied_on', 'in', _kinds),
|
| 319 |
+
'|', ('date_start', '=', False), ('date_start', '<=', today),
|
| 320 |
+
'|', ('date_end', '=', False), ('date_end', '>=', today),
|
| 321 |
+
('fixed_price', '>', 0)]
|
| 322 |
+
rules = O.search_read(
|
| 323 |
+
'product.pricelist.item', _live,
|
| 324 |
+
['pricelist_id', 'product_id', 'product_tmpl_id', 'applied_on', 'fixed_price',
|
| 325 |
+
'min_quantity']) if _declared else []
|
| 326 |
+
|
| 327 |
+
# β R6's SECOND SENTENCE, PAID FOR WITH `search_count` RATHER THAN A WIDER READ. Six
|
| 328 |
+
# counts cost ~1.2s together; the rules they count would cost 16s to read. What this
|
| 329 |
+
# reader cannot see is still REPORTED β it is just no longer transferred.
|
| 330 |
+
def _n(extra):
|
| 331 |
+
try:
|
| 332 |
+
return O.get_odoo().search_count('product.pricelist.item', extra)
|
| 333 |
+
except Exception:
|
| 334 |
+
return -1 # -1 reads as "not measured", never as zero
|
| 335 |
+
_dated = ['|', ('date_start', '=', False), ('date_start', '<=', today),
|
| 336 |
+
'|', ('date_end', '=', False), ('date_end', '>=', today)]
|
| 337 |
+
report["rules_total"] = _n([])
|
| 338 |
+
report["rules_undeclared_list"] = _n([('pricelist_id', 'not in', _declared)]) \
|
| 339 |
+
if _declared else report["rules_total"]
|
| 340 |
+
if _declared:
|
| 341 |
+
_on = [('pricelist_id', 'in', _declared)]
|
| 342 |
+
# `percent_price` and formula rules are read NOWHERE in this repo. A rule we cannot
|
| 343 |
+
# price is one the operator is TOLD about, never one that quietly becomes a blank.
|
| 344 |
+
report["rules_not_fixed"] = _n(_on + [('compute_price', '!=', 'fixed')])
|
| 345 |
+
report["rules_global"] = _n(_on + [('compute_price', '=', 'fixed'),
|
| 346 |
+
('applied_on', 'not in', _kinds)])
|
| 347 |
+
report["rules_out_of_date"] = (
|
| 348 |
+
_n(_on + [('compute_price', '=', 'fixed'), ('applied_on', 'in', _kinds)])
|
| 349 |
+
- _n(_on + [('compute_price', '=', 'fixed'),
|
| 350 |
+
('applied_on', 'in', _kinds)] + _dated))
|
| 351 |
+
# A zero price is not "free" β it is an unset rule. Blank says so; 0 does not.
|
| 352 |
+
report["rules_zero_price"] = _n(
|
| 353 |
+
_on + [('compute_price', '=', 'fixed'), ('applied_on', 'in', _kinds)]
|
| 354 |
+
+ _dated + [('fixed_price', '<=', 0)])
|
| 355 |
+
|
| 356 |
+
by_var, by_tmpl = {}, {}
|
| 357 |
+
for r in rules:
|
| 358 |
+
col = wanted[O.m2o_id(r.get('pricelist_id'))]
|
| 359 |
+
if r.get('applied_on') == '0_product_variant' and r.get('product_id'):
|
| 360 |
+
by_var.setdefault((col, O.m2o_id(r['product_id'])), []).append(r)
|
| 361 |
+
elif r.get('product_tmpl_id'):
|
| 362 |
+
by_tmpl.setdefault((col, O.m2o_id(r['product_tmpl_id'])), []).append(r)
|
| 363 |
+
|
| 364 |
+
dom = [('active', '=', True)]
|
| 365 |
+
prods = O.search_read('product.product', dom,
|
| 366 |
+
['id', 'default_code', 'product_tmpl_id'], limit=50000)
|
| 367 |
+
n = O.get_odoo().search_count('product.product', dom)
|
| 368 |
+
if len(prods) != n:
|
| 369 |
+
# Same guard, same reason as `catalogue()`: a short pull renders as a plausible
|
| 370 |
+
# smaller set of priced SKUs with nothing reporting it.
|
| 371 |
+
raise ValueError(
|
| 372 |
+
f"products.pricelist_by_code: the product pull is TRUNCATED β read {len(prods)} "
|
| 373 |
+
f"rows against a search_count of {n}.")
|
| 374 |
+
except Exception as e:
|
| 375 |
+
report["error"] = f"{type(e).__name__}: {str(e)[:200]}"
|
| 376 |
+
return {}, report
|
| 377 |
+
|
| 378 |
+
out = {}
|
| 379 |
+
for p in prods:
|
| 380 |
+
code = (str(p['default_code']).strip() if p.get('default_code') else f"pid:{p['id']}")
|
| 381 |
+
tmpl = O.m2o_id(p.get('product_tmpl_id'))
|
| 382 |
+
cells = {}
|
| 383 |
+
for col, _name in PRICELIST_COLUMNS:
|
| 384 |
+
cands = by_var.get((col, p['id'])) or by_tmpl.get((col, tmpl))
|
| 385 |
+
if not cands:
|
| 386 |
+
continue
|
| 387 |
+
base = min(cands, key=lambda r: r.get('min_quantity') or 0.0)
|
| 388 |
+
if (base.get('min_quantity') or 0.0) > 1.0:
|
| 389 |
+
report["qty_break_only"] += 1
|
| 390 |
+
cells[col] = base.get('fixed_price')
|
| 391 |
+
if cells:
|
| 392 |
+
out.setdefault(code, {}).update(cells)
|
| 393 |
+
return out, report
|
| 394 |
+
|
| 395 |
+
|
| 396 |
+
def catalogue_count():
|
| 397 |
+
"""The INDEPENDENT population oracle: Odoo's own count of active products.
|
| 398 |
+
|
| 399 |
+
Deliberately a bare `search_count` and not a `len()` over anything this module built β the
|
| 400 |
+
2,717 defect shipped silently for a wave because `product_data.validate()` derived BOTH sides
|
| 401 |
+
of its reconciliation from `_sku_rev`, so the oracle could never see a missing row.
|
| 402 |
+
"""
|
| 403 |
+
return O.get_odoo().search_count('product.product', [('active', '=', True)])
|
| 404 |
+
|
| 405 |
+
|
| 406 |
+
def categories(t=None, team_id=None):
|
| 407 |
+
"""Sorted distinct category names present in the SKU directory (for the drill-down filter)."""
|
| 408 |
+
return sorted({r['category'] for r in directory(t, team_id)})
|
| 409 |
+
|
| 410 |
+
|
| 411 |
+
def _coverage(date_from, date_to, team_id=None):
|
| 412 |
+
"""{sku_code: distinct_customer_count} via 2-level read_group (slow-ish), merged by code.
|
| 413 |
+
(A customer buying two records sharing a code can count twice; duplicates are rare, and
|
| 414 |
+
code-level avoids the bigger error of a re-SKUed product reading as full coverage loss.)"""
|
| 415 |
+
g = O.read_group('sale.order.line', O.sale_line_domain(date_from, date_to, team_id, extra=_NO_SVC),
|
| 416 |
+
['__count'], ['product_id', 'order_partner_id'], lazy=False)
|
| 417 |
+
codes = _code_map()
|
| 418 |
+
cov = {}
|
| 419 |
+
names = {}
|
| 420 |
+
for r in g:
|
| 421 |
+
pid = O.m2o_id(r.get('product_id'))
|
| 422 |
+
if not pid:
|
| 423 |
+
continue
|
| 424 |
+
key = codes.get(pid, f"pid:{pid}")
|
| 425 |
+
cov[key] = cov.get(key, 0) + 1
|
| 426 |
+
names.setdefault(key, O.m2o_name(r.get('product_id')))
|
| 427 |
+
return cov, names
|
| 428 |
+
|
| 429 |
+
|
| 430 |
+
def coverage_collapse(t=None, limit=25, min_prior_custs=8, team_id=None):
|
| 431 |
+
"""SKUs that lost the most customer breadth YoY β early warning a SKU is dying even if
|
| 432 |
+
revenue hasn't fully cratered yet."""
|
| 433 |
+
t = t or P.today()
|
| 434 |
+
yf, yt = P.ytd(t)
|
| 435 |
+
lf, lt = P.ytd_last_year(t)
|
| 436 |
+
cov_t, names_t = _coverage(yf, yt, team_id)
|
| 437 |
+
cov_l, names_l = _coverage(lf, lt, team_id)
|
| 438 |
+
rows = []
|
| 439 |
+
for pid, lc in cov_l.items():
|
| 440 |
+
if lc < min_prior_custs:
|
| 441 |
+
continue
|
| 442 |
+
tc = cov_t.get(pid, 0)
|
| 443 |
+
drop = lc - tc
|
| 444 |
+
if drop <= 0:
|
| 445 |
+
continue
|
| 446 |
+
rows.append({'code': pid, 'product': names_l.get(pid) or names_t.get(pid) or '',
|
| 447 |
+
'custs_ly': lc, 'custs_ytd': tc, 'lost_custs': drop,
|
| 448 |
+
'pct_drop': drop / lc * 100})
|
| 449 |
+
rows.sort(key=lambda x: (-x['lost_custs'], -x['pct_drop']))
|
| 450 |
+
return rows[:limit]
|
| 451 |
+
|
| 452 |
+
|
| 453 |
+
# ====================================================================== SKU DRAWER (mirror of customer drawer)
|
| 454 |
+
def _sku_product_ids(code):
|
| 455 |
+
"""All product.product ids sharing this SKU code (merged variants / archived records)."""
|
| 456 |
+
return [pid for pid, c in _code_map().items() if c == code]
|
| 457 |
+
|
| 458 |
+
|
| 459 |
+
def _sku_dom(pids, date_from, date_to, team_id=None):
|
| 460 |
+
return O.sale_line_domain(date_from, date_to, team_id, extra=[('product_id', 'in', pids)] + _NO_SVC)
|
| 461 |
+
|
| 462 |
+
|
| 463 |
+
def _sku_name_category(pids):
|
| 464 |
+
rows = O.search_read('product.product', [('id', 'in', pids)], ['name', 'categ_id'])
|
| 465 |
+
name = rows[0]['name'] if rows else '(unknown)'
|
| 466 |
+
catmap = sales_mod._product_cat()
|
| 467 |
+
main = next((catmap.get(p) for p in pids if catmap.get(p)), '(uncategorized)')
|
| 468 |
+
return name, main
|
| 469 |
+
|
| 470 |
+
|
| 471 |
+
def _sku_buyers(pids, date_from, date_to, team_id=None):
|
| 472 |
+
"""{partner_id: {'name','rev','qty'}} for buyers of this SKU over a window."""
|
| 473 |
+
g = O.read_group('sale.order.line', _sku_dom(pids, date_from, date_to, team_id),
|
| 474 |
+
['price_subtotal:sum', 'product_uom_qty:sum'], ['order_partner_id'], lazy=False)
|
| 475 |
+
out = {}
|
| 476 |
+
for r in g:
|
| 477 |
+
pid = O.m2o_id(r.get('order_partner_id'))
|
| 478 |
+
if pid:
|
| 479 |
+
out[pid] = {'name': O.m2o_name(r.get('order_partner_id')),
|
| 480 |
+
'rev': r.get('price_subtotal') or 0.0, 'qty': r.get('product_uom_qty') or 0.0}
|
| 481 |
+
return out
|
| 482 |
+
|
| 483 |
+
|
| 484 |
+
def sku_detail(code, t=None, team_id=None, n_months=13, allsku=None):
|
| 485 |
+
"""KPIs (rev/qty/buyers YoY, GM%), monthly trend, rank & % of BU for one SKU code.
|
| 486 |
+
Pass `allsku` (a cached _sku_rev YTD map) to skip the ~all-SKU rank read."""
|
| 487 |
+
t = t or P.today()
|
| 488 |
+
pids = _sku_product_ids(code)
|
| 489 |
+
if not pids:
|
| 490 |
+
return None
|
| 491 |
+
yf, yt = P.ytd(t)
|
| 492 |
+
lf, lt = P.ytd_last_year(t)
|
| 493 |
+
mf, mt = P.ltm(t)
|
| 494 |
+
name, category = _sku_name_category(pids)
|
| 495 |
+
|
| 496 |
+
def s(df, dtt, field='price_subtotal'):
|
| 497 |
+
return O.sum_field('sale.order.line', _sku_dom(pids, df, dtt, team_id), field)
|
| 498 |
+
rev_ytd, rev_ly = s(yf, yt), s(lf, lt)
|
| 499 |
+
qty_ytd, qty_ly = s(yf, yt, 'product_uom_qty'), s(lf, lt, 'product_uom_qty')
|
| 500 |
+
buyers_ytd = len(_sku_buyers(pids, yf, yt, team_id))
|
| 501 |
+
buyers_ly = len(_sku_buyers(pids, lf, lt, team_id))
|
| 502 |
+
g = O.read_group('sale.order.line', _sku_dom(pids, mf, mt, team_id),
|
| 503 |
+
['price_subtotal:sum', 'margin:sum'], [], lazy=False)
|
| 504 |
+
line_rev = (g[0].get('price_subtotal') if g else 0) or 0.0
|
| 505 |
+
margin = (g[0].get('margin') if g else 0) or 0.0
|
| 506 |
+
|
| 507 |
+
# Monthly trend: bin lines by their order's month in Python (dot-path month groupby is rejected
|
| 508 |
+
# on sale.order.line), in 2 reads instead of 26 point queries.
|
| 509 |
+
range_start = dt.date(t.year - 2, t.month, 1).isoformat()
|
| 510 |
+
lines = O.search_read('sale.order.line', _sku_dom(pids, range_start, t.isoformat(), team_id),
|
| 511 |
+
['price_subtotal', 'order_id'])
|
| 512 |
+
oids = list({O.m2o_id(line['order_id']) for line in lines if line.get('order_id')})
|
| 513 |
+
odate = {}
|
| 514 |
+
for i in range(0, len(oids), 1000):
|
| 515 |
+
for o in O.search_read('sale.order', [('id', 'in', oids[i:i + 1000])], ['date_order']):
|
| 516 |
+
if o.get('date_order'):
|
| 517 |
+
odate[o['id']] = str(o['date_order'])[:7]
|
| 518 |
+
mrev = {}
|
| 519 |
+
for line in lines:
|
| 520 |
+
ym = odate.get(O.m2o_id(line.get('order_id')))
|
| 521 |
+
if ym:
|
| 522 |
+
mrev[ym] = mrev.get(ym, 0.0) + (line.get('price_subtotal') or 0.0)
|
| 523 |
+
monthly = []
|
| 524 |
+
for ym, start, end in P.month_starts(n_months, t):
|
| 525 |
+
y, m = int(ym[:4]) - 1, int(ym[5:7])
|
| 526 |
+
monthly.append({'month': ym, 'revenue': mrev.get(ym, 0.0),
|
| 527 |
+
'revenue_ly': mrev.get(f'{y:04d}-{m:02d}', 0.0)})
|
| 528 |
+
|
| 529 |
+
allsku = allsku if allsku is not None else _sku_rev(yf, yt, team_id)
|
| 530 |
+
total = sum(v['rev'] for v in allsku.values()) or 1.0
|
| 531 |
+
rank = next((i + 1 for i, (c, _v) in enumerate(sorted(allsku.items(), key=lambda kv: -kv[1]['rev']))
|
| 532 |
+
if c == code), None)
|
| 533 |
+
return {
|
| 534 |
+
'code': code, 'name': name, 'category': category,
|
| 535 |
+
'rev_ytd': rev_ytd, 'rev_ly': rev_ly, 'rev_yoy_pct': P.yoy_pct(rev_ytd, rev_ly),
|
| 536 |
+
'qty_ytd': qty_ytd, 'qty_ly': qty_ly, 'qty_yoy_pct': P.yoy_pct(qty_ytd, qty_ly),
|
| 537 |
+
'buyers_ytd': buyers_ytd, 'buyers_ly': buyers_ly, 'buyers_delta': buyers_ytd - buyers_ly,
|
| 538 |
+
'gm_pct': (margin / line_rev * 100) if line_rev else 0.0, 'gm_dollars': margin,
|
| 539 |
+
'rank': rank, 'n_skus': len(allsku), 'pct_of_bu': rev_ytd / total * 100,
|
| 540 |
+
'monthly': monthly,
|
| 541 |
+
}
|
| 542 |
+
|
| 543 |
+
|
| 544 |
+
def sku_buyer_bridge(code, t=None, team_id=None, top=12):
|
| 545 |
+
"""Who drives the SKU's YoY: retained / new / churned buyers + a $-ranked churned call list."""
|
| 546 |
+
t = t or P.today()
|
| 547 |
+
pids = _sku_product_ids(code)
|
| 548 |
+
yf, yt = P.ytd(t)
|
| 549 |
+
lf, lt = P.ytd_last_year(t)
|
| 550 |
+
this, last = _sku_buyers(pids, yf, yt, team_id), _sku_buyers(pids, lf, lt, team_id)
|
| 551 |
+
tset, lset = set(this), set(last)
|
| 552 |
+
retained, new, churned = tset & lset, tset - lset, lset - tset
|
| 553 |
+
churned_list = sorted([{'pid': p, 'customer': last[p]['name'], 'ly_rev': last[p]['rev']}
|
| 554 |
+
for p in churned], key=lambda x: -x['ly_rev'])[:top]
|
| 555 |
+
return {'retained': {'n': len(retained), 'rev': sum(this[p]['rev'] for p in retained)},
|
| 556 |
+
'new': {'n': len(new), 'rev': sum(this[p]['rev'] for p in new)},
|
| 557 |
+
'churned': {'n': len(churned), 'rev': sum(last[p]['rev'] for p in churned)},
|
| 558 |
+
'buyer_retention_pct': (len(retained) / len(lset) * 100) if lset else 0.0,
|
| 559 |
+
'churned_buyers': churned_list, 'buyers_this': len(tset), 'buyers_last': len(lset),
|
| 560 |
+
'this_total': sum(v['rev'] for v in this.values())}
|
| 561 |
+
|
| 562 |
+
|
| 563 |
+
def sku_concentration(code, t=None, team_id=None):
|
| 564 |
+
"""Buyer-concentration risk: top-1/top-3 share, Herfindahl index, effective buyer count (LTM)."""
|
| 565 |
+
t = t or P.today()
|
| 566 |
+
mf, mt = P.ltm(t)
|
| 567 |
+
buyers = _sku_buyers(_sku_product_ids(code), mf, mt, team_id)
|
| 568 |
+
revs = sorted([v['rev'] for v in buyers.values()], reverse=True)
|
| 569 |
+
total = sum(revs) or 1.0
|
| 570 |
+
hhi = sum((r / total) ** 2 for r in revs)
|
| 571 |
+
return {'n_buyers': len(revs), 'top1_pct': (revs[0] / total * 100) if revs else 0.0,
|
| 572 |
+
'top3_pct': (sum(revs[:3]) / total * 100) if revs else 0.0,
|
| 573 |
+
'hhi': hhi, 'eff_buyers': (1 / hhi) if hhi else 0.0}
|
| 574 |
+
|
| 575 |
+
|
| 576 |
+
def sku_price_dispersion(code, t=None, team_id=None, cap=60):
|
| 577 |
+
"""Realized $/unit per buyer (LTM) vs the volume-weighted average; recoverable $ on below-VWAP
|
| 578 |
+
accounts. list_price is unreliable, so the dispersion among actual buyers is the margin lever."""
|
| 579 |
+
t = t or P.today()
|
| 580 |
+
mf, mt = P.ltm(t)
|
| 581 |
+
g = O.read_group('sale.order.line', _sku_dom(_sku_product_ids(code), mf, mt, team_id),
|
| 582 |
+
['price_subtotal:sum', 'product_uom_qty:sum'], ['order_partner_id'], lazy=False)
|
| 583 |
+
rows, tot_rev, tot_qty = [], 0.0, 0.0
|
| 584 |
+
for r in g:
|
| 585 |
+
pid = O.m2o_id(r.get('order_partner_id'))
|
| 586 |
+
rev = r.get('price_subtotal') or 0.0
|
| 587 |
+
qty = r.get('product_uom_qty') or 0.0
|
| 588 |
+
if not pid or qty <= 0:
|
| 589 |
+
continue
|
| 590 |
+
rows.append({'pid': pid, 'customer': O.m2o_name(r.get('order_partner_id')),
|
| 591 |
+
'price': rev / qty, 'qty': qty, 'rev': rev})
|
| 592 |
+
tot_rev += rev
|
| 593 |
+
tot_qty += qty
|
| 594 |
+
vwap = (tot_rev / tot_qty) if tot_qty else 0.0
|
| 595 |
+
for r in rows:
|
| 596 |
+
r['recoverable'] = max(0.0, vwap - r['price']) * r['qty']
|
| 597 |
+
rows.sort(key=lambda x: -x['recoverable'])
|
| 598 |
+
return {'vwap': vwap, 'n_buyers': len(rows),
|
| 599 |
+
'recoverable_total': sum(r['recoverable'] for r in rows), 'rows': rows[:cap]}
|
| 600 |
+
|
| 601 |
+
|
| 602 |
+
def sku_top_buyers(code, t=None, team_id=None, top=15):
|
| 603 |
+
"""Ranked buyers of this SKU (YTD) with YoY β clickable to open the customer drawer."""
|
| 604 |
+
t = t or P.today()
|
| 605 |
+
pids = _sku_product_ids(code)
|
| 606 |
+
yf, yt = P.ytd(t)
|
| 607 |
+
lf, lt = P.ytd_last_year(t)
|
| 608 |
+
this, last = _sku_buyers(pids, yf, yt, team_id), _sku_buyers(pids, lf, lt, team_id)
|
| 609 |
+
rows = [{'pid': p, 'customer': v['name'], 'rev': v['rev'], 'qty': v['qty'],
|
| 610 |
+
'yoy_pct': P.yoy_pct(v['rev'], last.get(p, {}).get('rev', 0.0))} for p, v in this.items()]
|
| 611 |
+
rows.sort(key=lambda x: -x['rev'])
|
| 612 |
+
return rows[:top]
|
| 613 |
+
|
| 614 |
+
|
| 615 |
+
_BUYER_TIERS = [('Whale (β₯$25k)', 25000.0), ('Large ($10β25k)', 10000.0),
|
| 616 |
+
('Mid ($2β10k)', 2000.0), ('Small (<$2k)', 0.0)]
|
| 617 |
+
|
| 618 |
+
|
| 619 |
+
def sku_customer_analysis(code, t=None, team_id=None, top=15):
|
| 620 |
+
"""WHO buys this SKU (LTM), as customers: the value-tier mix of its buyers (by each buyer's TOTAL
|
| 621 |
+
spend), and the accounts most DEPENDENT on it (this SKU as a share of their spend β who gets hurt
|
| 622 |
+
most if it stocks out / who to protect)."""
|
| 623 |
+
t = t or P.today()
|
| 624 |
+
lf, lt = P.ltm(t)
|
| 625 |
+
pids = _sku_product_ids(code)
|
| 626 |
+
empty = {'segments': [], 'dependency': [], 'n_buyers': 0, 'avg_dependency': 0.0}
|
| 627 |
+
if not pids:
|
| 628 |
+
return empty
|
| 629 |
+
buyers = _sku_buyers(pids, lf, lt, team_id) # {pid: {name, rev(this SKU), qty}}
|
| 630 |
+
if not buyers:
|
| 631 |
+
return empty
|
| 632 |
+
bpids = list(buyers)
|
| 633 |
+
g = O.read_group('sale.order', sales_mod.order_domain(lf, lt, team_id) + [('partner_id', 'in', bpids)],
|
| 634 |
+
['amount_untaxed:sum'], ['partner_id'], lazy=False)
|
| 635 |
+
total = {O.m2o_id(r['partner_id']): (r.get('amount_untaxed') or 0.0) for r in g if r.get('partner_id')}
|
| 636 |
+
|
| 637 |
+
def tier(rev):
|
| 638 |
+
for nm, lo in _BUYER_TIERS:
|
| 639 |
+
if rev >= lo:
|
| 640 |
+
return nm
|
| 641 |
+
return _BUYER_TIERS[-1][0]
|
| 642 |
+
seg = {nm: {'tier': nm, 'buyers': 0, 'sku_rev': 0.0} for nm, _ in _BUYER_TIERS}
|
| 643 |
+
dep = []
|
| 644 |
+
for p, v in buyers.items():
|
| 645 |
+
ct = total.get(p, v['rev']) or v['rev']
|
| 646 |
+
s = seg[tier(ct)]
|
| 647 |
+
s['buyers'] += 1
|
| 648 |
+
s['sku_rev'] += v['rev']
|
| 649 |
+
dep.append({'customer': v['name'], 'pid': p, 'sku_rev': v['rev'], 'cust_total': ct,
|
| 650 |
+
'dependency_pct': (v['rev'] / ct * 100) if ct else None})
|
| 651 |
+
tot = sum(s['sku_rev'] for s in seg.values()) or 1.0
|
| 652 |
+
segments = []
|
| 653 |
+
for nm, _ in _BUYER_TIERS:
|
| 654 |
+
s = seg[nm]
|
| 655 |
+
s['rev_share'] = s['sku_rev'] / tot * 100
|
| 656 |
+
s['avg_per_buyer'] = (s['sku_rev'] / s['buyers']) if s['buyers'] else 0.0
|
| 657 |
+
segments.append(s)
|
| 658 |
+
dep.sort(key=lambda x: -x['sku_rev'])
|
| 659 |
+
deps_known = [d['dependency_pct'] for d in dep if d['dependency_pct'] is not None]
|
| 660 |
+
return {'segments': segments, 'dependency': dep[:top], 'n_buyers': len(buyers),
|
| 661 |
+
'avg_dependency': (sum(deps_known) / len(deps_known)) if deps_known else 0.0}
|
| 662 |
+
|
| 663 |
+
|
| 664 |
+
def sku_whitespace(code, t=None, team_id=None, top=15):
|
| 665 |
+
"""Customers who buy this SKU's category but NOT this SKU β ranked prospect list."""
|
| 666 |
+
import modules.customers as cust_mod
|
| 667 |
+
t = t or P.today()
|
| 668 |
+
pids = _sku_product_ids(code)
|
| 669 |
+
_name, category = _sku_name_category(pids)
|
| 670 |
+
cat_buyers = cust_mod.category_buyers(category, team_id=team_id) or set()
|
| 671 |
+
mf, mt = P.ltm(t)
|
| 672 |
+
prospects = list(cat_buyers - set(_sku_buyers(pids, mf, mt, team_id)))
|
| 673 |
+
if not prospects:
|
| 674 |
+
return {'category': category, 'rows': []}
|
| 675 |
+
g = O.read_group('sale.order', sales_mod.order_domain(mf, mt, team_id) + [('partner_id', 'in', prospects)],
|
| 676 |
+
['amount_untaxed:sum'], ['partner_id'], lazy=False)
|
| 677 |
+
rows = [{'pid': O.m2o_id(r['partner_id']), 'customer': O.m2o_name(r['partner_id']),
|
| 678 |
+
'total_spend': r.get('amount_untaxed') or 0.0} for r in g if r.get('partner_id')]
|
| 679 |
+
rows.sort(key=lambda x: -x['total_spend'])
|
| 680 |
+
return {'category': category, 'rows': rows[:top]}
|
| 681 |
+
|
| 682 |
+
|
| 683 |
+
def sku_drawer_bundle(code, t=None, team_id=None, allsku=None):
|
| 684 |
+
"""The whole SKU drawer's first paint in one cached unit: sku_detail first (the not-found gate),
|
| 685 |
+
then the other six pulls CONCURRENTLY (O.parallel). Cuts a ~6-call cold open to ~max(call)."""
|
| 686 |
+
detail = sku_detail(code, t=t, team_id=team_id, allsku=allsku)
|
| 687 |
+
if detail is None:
|
| 688 |
+
return {'detail': None}
|
| 689 |
+
bridge, conc, price, buyers, white, ca = O.parallel([
|
| 690 |
+
lambda: sku_buyer_bridge(code, t, team_id),
|
| 691 |
+
lambda: sku_concentration(code, t, team_id),
|
| 692 |
+
lambda: sku_price_dispersion(code, t, team_id),
|
| 693 |
+
lambda: sku_top_buyers(code, t, team_id),
|
| 694 |
+
lambda: sku_whitespace(code, t, team_id),
|
| 695 |
+
lambda: sku_customer_analysis(code, t, team_id),
|
| 696 |
+
])
|
| 697 |
+
return {'detail': detail, 'bridge': bridge, 'conc': conc, 'price': price,
|
| 698 |
+
'buyers': buyers, 'white': white, 'ca': ca}
|
| 699 |
+
|
| 700 |
+
|
| 701 |
+
def validate(t=None, team_id=None):
|
| 702 |
+
"""Reconcile SKU metrics to independent Odoo aggregates. When team_id is set (a single BU
|
| 703 |
+
selected) every check runs SCOPED to that BU, so the validation panel never reconciles
|
| 704 |
+
against β or exposes β the other BU's numbers. All checks here are BU-scopeable (no cross-BU
|
| 705 |
+
mirror), so team_id threads straight through."""
|
| 706 |
+
t = t or P.today()
|
| 707 |
+
yf, yt = P.ytd(t)
|
| 708 |
+
checks = []
|
| 709 |
+
sku = _sku_rev(yf, yt, team_id)
|
| 710 |
+
sku_sum = sum(v['rev'] for v in sku.values())
|
| 711 |
+
line_total = O.sum_field('sale.order.line',
|
| 712 |
+
O.sale_line_domain(yf, yt, team_id, extra=_NO_SVC), 'price_subtotal')
|
| 713 |
+
checks.append({'check': 'SKU rev: Ξ£(per-SKU) == total line revenue, ex-services (YTD)',
|
| 714 |
+
'a': round(sku_sum, 2), 'b': round(line_total, 2),
|
| 715 |
+
'gap': round(sku_sum - line_total, 2),
|
| 716 |
+
'ok': abs(sku_sum - line_total) <= 1.0})
|
| 717 |
+
|
| 718 |
+
m = yoy_movers(t, limit=10**9, team_id=team_id)
|
| 719 |
+
movers_sum = sum(r['change'] for r in m['risers']) + sum(r['change'] for r in m['decliners'])
|
| 720 |
+
lf, lt = P.ytd_last_year(t)
|
| 721 |
+
last_total = O.sum_field('sale.order.line',
|
| 722 |
+
O.sale_line_domain(lf, lt, team_id, extra=_NO_SVC), 'price_subtotal')
|
| 723 |
+
checks.append({'check': 'SKU movers: Ξ£(Ξ) == (YTD β LY) total',
|
| 724 |
+
'a': round(movers_sum, 2), 'b': round(line_total - last_total, 2),
|
| 725 |
+
'gap': round(movers_sum - (line_total - last_total), 2),
|
| 726 |
+
'ok': abs(movers_sum - (line_total - last_total)) <= 1.0})
|
| 727 |
+
|
| 728 |
+
# SKU drawer: buyer-bridge reconciles β retained + new buyer revenue == this-period SKU revenue
|
| 729 |
+
top_code = max(sku.items(), key=lambda kv: kv[1]['rev'])[0] if sku else None
|
| 730 |
+
if top_code:
|
| 731 |
+
bb = sku_buyer_bridge(top_code, t, team_id=team_id)
|
| 732 |
+
recon = bb['retained']['rev'] + bb['new']['rev']
|
| 733 |
+
checks.append({'check': 'SKU drawer: retained+new buyer rev == SKU YTD revenue',
|
| 734 |
+
'a': round(recon, 2), 'b': round(bb['this_total'], 2),
|
| 735 |
+
'gap': round(recon - bb['this_total'], 2),
|
| 736 |
+
'ok': abs(recon - bb['this_total']) <= 1.0})
|
| 737 |
+
return checks
|
web/public/sample_customers.json
CHANGED
|
@@ -1,558 +1,558 @@
|
|
| 1 |
-
{
|
| 2 |
-
"fields": [
|
| 3 |
-
{
|
| 4 |
-
"key": "customer",
|
| 5 |
-
"label": "Customer",
|
| 6 |
-
"type": "text",
|
| 7 |
-
"source": "odoo",
|
| 8 |
-
"pinned": true,
|
| 9 |
-
"default": true,
|
| 10 |
-
"description": "The customer's name in Odoo. One row per customer who ordered in the last 24 months."
|
| 11 |
-
},
|
| 12 |
-
{
|
| 13 |
-
"key": "partner_id",
|
| 14 |
-
"label": "Odoo ID",
|
| 15 |
-
"type": "int",
|
| 16 |
-
"source": "odoo",
|
| 17 |
-
"derived": true,
|
| 18 |
-
"default": false,
|
| 19 |
-
"description": "The Odoo res.partner id β the key every Odoo document joins on. DERIVED: this row's pid IS the partner id, so a stored copy would be a second source."
|
| 20 |
-
},
|
| 21 |
-
{
|
| 22 |
-
"key": "odoo_status",
|
| 23 |
-
"label": "Odoo record",
|
| 24 |
-
"type": "status",
|
| 25 |
-
"source": "odoo",
|
| 26 |
-
"default": false,
|
| 27 |
-
"options": [
|
| 28 |
-
"Active",
|
| 29 |
-
"Archived"
|
| 30 |
-
],
|
| 31 |
-
"description": "Whether this customer still exists in Odoo. Archived means deleted there."
|
| 32 |
-
},
|
| 33 |
-
{
|
| 34 |
-
"key": "agent",
|
| 35 |
-
"label": "Agent",
|
| 36 |
-
"type": "text",
|
| 37 |
-
"source": "odoo",
|
| 38 |
-
"default": true,
|
| 39 |
-
"description": "The sales agent who owns this account."
|
| 40 |
-
},
|
| 41 |
-
{
|
| 42 |
-
"key": "dba",
|
| 43 |
-
"label": "DBA",
|
| 44 |
-
"type": "select",
|
| 45 |
-
"source": "odoo",
|
| 46 |
-
"default": false,
|
| 47 |
-
"options": [
|
| 48 |
-
"Fisch",
|
| 49 |
-
"Royal",
|
| 50 |
-
"Both"
|
| 51 |
-
],
|
| 52 |
-
"description": "The brand this customer buys from - Fisch, Royal, or both. Amazon-channel orders are not a DBA."
|
| 53 |
-
},
|
| 54 |
-
{
|
| 55 |
-
"key": "salesperson",
|
| 56 |
-
"label": "Salesperson",
|
| 57 |
-
"type": "text",
|
| 58 |
-
"source": "odoo",
|
| 59 |
-
"default": false,
|
| 60 |
-
"description": "Who keyed in most of this customer's orders β not the Agent, who owns the account."
|
| 61 |
-
},
|
| 62 |
-
{
|
| 63 |
-
"key": "street",
|
| 64 |
-
"label": "Street",
|
| 65 |
-
"type": "text",
|
| 66 |
-
"source": "odoo",
|
| 67 |
-
"default": false,
|
| 68 |
-
"description": "First address line, from res.partner directly - not the geocoder, so a customer the map cannot place still shows its address."
|
| 69 |
-
},
|
| 70 |
-
{
|
| 71 |
-
"key": "street2",
|
| 72 |
-
"label": "Street 2",
|
| 73 |
-
"type": "text",
|
| 74 |
-
"source": "odoo",
|
| 75 |
-
"default": false,
|
| 76 |
-
"description": "Second address line (suite, unit, floor) on the customer's Odoo address."
|
| 77 |
-
},
|
| 78 |
-
{
|
| 79 |
-
"key": "city",
|
| 80 |
-
"label": "City",
|
| 81 |
-
"type": "text",
|
| 82 |
-
"source": "odoo",
|
| 83 |
-
"default": true,
|
| 84 |
-
"description": "City on the customer's Odoo address."
|
| 85 |
-
},
|
| 86 |
-
{
|
| 87 |
-
"key": "state",
|
| 88 |
-
"label": "State",
|
| 89 |
-
"type": "text",
|
| 90 |
-
"source": "odoo",
|
| 91 |
-
"default": true,
|
| 92 |
-
"description": "State or province on the customer's Odoo address."
|
| 93 |
-
},
|
| 94 |
-
{
|
| 95 |
-
"key": "country",
|
| 96 |
-
"label": "Country",
|
| 97 |
-
"type": "text",
|
| 98 |
-
"source": "odoo",
|
| 99 |
-
"default": false,
|
| 100 |
-
"description": "Country on the customer's Odoo address."
|
| 101 |
-
},
|
| 102 |
-
{
|
| 103 |
-
"key": "zip",
|
| 104 |
-
"label": "ZIP",
|
| 105 |
-
"type": "text",
|
| 106 |
-
"source": "odoo",
|
| 107 |
-
"default": false,
|
| 108 |
-
"description": "Postal code on the customer's Odoo address."
|
| 109 |
-
},
|
| 110 |
-
{
|
| 111 |
-
"key": "customer_since",
|
| 112 |
-
"label": "Customer since",
|
| 113 |
-
"type": "date",
|
| 114 |
-
"source": "odoo",
|
| 115 |
-
"default": false,
|
| 116 |
-
"description": "When this customer was first set up in Odoo."
|
| 117 |
-
},
|
| 118 |
-
{
|
| 119 |
-
"key": "tags",
|
| 120 |
-
"label": "Tags",
|
| 121 |
-
"type": "text",
|
| 122 |
-
"source": "odoo",
|
| 123 |
-
"default": false,
|
| 124 |
-
"description": "Odoo labels on this customer, comma-separated."
|
| 125 |
-
},
|
| 126 |
-
{
|
| 127 |
-
"key": "pricelist",
|
| 128 |
-
"label": "Price list",
|
| 129 |
-
"type": "text",
|
| 130 |
-
"source": "odoo",
|
| 131 |
-
"default": false,
|
| 132 |
-
"description": "The price list this customer buys on."
|
| 133 |
-
},
|
| 134 |
-
{
|
| 135 |
-
"key": "payment_terms",
|
| 136 |
-
"label": "Payment terms",
|
| 137 |
-
"type": "text",
|
| 138 |
-
"source": "odoo",
|
| 139 |
-
"default": false,
|
| 140 |
-
"description": "Payment terms on this customer's account β Net 30, for example."
|
| 141 |
-
},
|
| 142 |
-
{
|
| 143 |
-
"key": "last_order",
|
| 144 |
-
"label": "Last order",
|
| 145 |
-
"type": "date",
|
| 146 |
-
"source": "odoo",
|
| 147 |
-
"default": true,
|
| 148 |
-
"description": "Date of the most recent confirmed order."
|
| 149 |
-
},
|
| 150 |
-
{
|
| 151 |
-
"key": "overdue_days",
|
| 152 |
-
"label": "Overdue days",
|
| 153 |
-
"type": "int",
|
| 154 |
-
"source": "odoo",
|
| 155 |
-
"default": true,
|
| 156 |
-
"description": "How many days late this customer is running against their own usual ordering rhythm."
|
| 157 |
-
},
|
| 158 |
-
{
|
| 159 |
-
"_note": "filterable:false β DERIVED ANALYTIC: est_missed is min(cycles missed, 3) x AOV, a score we compute rather than an object the business has, so a condition on it would read as a fact about the customer when it is a fact about our arithmetic. It still displays and still sorts. Until wave 6 this flag also covered the frozen-window presets (revenue_ytd / revenue_ly / orders_24m / aov / yoy_pct); those are now DELETED outright under the owner's no-buildable-presets rule β see _comment. est_missed itself STAYS: no creatable measure or formula reproduces the cadence model behind it.",
|
| 160 |
-
"key": "est_missed",
|
| 161 |
-
"label": "Est. missed $",
|
| 162 |
-
"type": "currency",
|
| 163 |
-
"source": "odoo",
|
| 164 |
-
"default": true,
|
| 165 |
-
"agg": "sum",
|
| 166 |
-
"filterable": false,
|
| 167 |
-
"description": "Estimated sales missed while quiet: missed orders (capped at 3) times average order value. An estimate, not money owed."
|
| 168 |
-
},
|
| 169 |
-
{
|
| 170 |
-
"_note": "wave 21 R1 β KEY UNCHANGED, LABEL RENAMED. The computation is a DISJOINT split (ar.py credit_exposure): this column is only the not-yet-due residual, its sibling is the past-grace residual, and the two sum to the total. Under the label 'AR open $' the majority-late book read as 'Overdue > Open', which is nonsense in AR vocabulary β 'open' universally means the total. The label now says what the number is; the key stays so saved views and filters keep working.",
|
| 171 |
-
"key": "ar_open",
|
| 172 |
-
"label": "AR current $",
|
| 173 |
-
"type": "currency",
|
| 174 |
-
"source": "odoo",
|
| 175 |
-
"default": false,
|
| 176 |
-
"description": "Invoiced money owed but not yet due (a 5-day grace applies before it counts as overdue)."
|
| 177 |
-
},
|
| 178 |
-
{
|
| 179 |
-
"key": "ar_overdue",
|
| 180 |
-
"label": "AR overdue $",
|
| 181 |
-
"type": "currency",
|
| 182 |
-
"source": "odoo",
|
| 183 |
-
"default": false,
|
| 184 |
-
"description": "Invoiced money past due β same basis as the Collections page."
|
| 185 |
-
},
|
| 186 |
-
{
|
| 187 |
-
"_note": "wave 21 R1 β the TOTAL, added beside the rename above. AR current $ + AR overdue $, i.e. what most people mean by 'open AR'. Composed from the same ar.credit_exposure rows the siblings use, so it is transitively reconciled by ar.validate()'s residual read_group tie β no second oracle.",
|
| 188 |
-
"key": "ar_outstanding",
|
| 189 |
-
"label": "AR outstanding $",
|
| 190 |
-
"type": "currency",
|
| 191 |
-
"source": "odoo",
|
| 192 |
-
"default": false,
|
| 193 |
-
"description": "Total invoiced money owed right now: AR current $ plus AR overdue $."
|
| 194 |
-
},
|
| 195 |
-
{
|
| 196 |
-
"key": "ar_exposure",
|
| 197 |
-
"label": "Credit exposure $",
|
| 198 |
-
"type": "currency",
|
| 199 |
-
"source": "odoo",
|
| 200 |
-
"default": false,
|
| 201 |
-
"description": "The most you could be out if they stopped paying today: open, overdue, draft and not-yet-invoiced."
|
| 202 |
-
},
|
| 203 |
-
{
|
| 204 |
-
"key": "ar_aged_1_30",
|
| 205 |
-
"label": "1-30 days $",
|
| 206 |
-
"type": "currency",
|
| 207 |
-
"source": "odoo",
|
| 208 |
-
"default": false,
|
| 209 |
-
"description": "Overdue between 1 and 30 days. The four aging buckets sum to AR overdue $."
|
| 210 |
-
},
|
| 211 |
-
{
|
| 212 |
-
"key": "ar_aged_31_60",
|
| 213 |
-
"label": "31-60 days $",
|
| 214 |
-
"type": "currency",
|
| 215 |
-
"source": "odoo",
|
| 216 |
-
"default": false,
|
| 217 |
-
"description": "Overdue between 31 and 60 days. The four aging buckets sum to AR overdue $."
|
| 218 |
-
},
|
| 219 |
-
{
|
| 220 |
-
"key": "ar_aged_61_90",
|
| 221 |
-
"label": "61-90 days $",
|
| 222 |
-
"type": "currency",
|
| 223 |
-
"source": "odoo",
|
| 224 |
-
"default": false,
|
| 225 |
-
"description": "Overdue between 61 and 90 days. The four aging buckets sum to AR overdue $."
|
| 226 |
-
},
|
| 227 |
-
{
|
| 228 |
-
"key": "ar_aged_90_plus",
|
| 229 |
-
"label": "90+ days $",
|
| 230 |
-
"type": "currency",
|
| 231 |
-
"source": "odoo",
|
| 232 |
-
"default": false,
|
| 233 |
-
"description": "Overdue by more than 90 days. The four aging buckets sum to AR overdue $."
|
| 234 |
-
},
|
| 235 |
-
{
|
| 236 |
-
"key": "days_to_pay",
|
| 237 |
-
"label": "Days to pay",
|
| 238 |
-
"type": "int",
|
| 239 |
-
"source": "odoo",
|
| 240 |
-
"default": false,
|
| 241 |
-
"description": "Average days to pay an invoice in full. Blank means no fully paid invoice yet."
|
| 242 |
-
},
|
| 243 |
-
{
|
| 244 |
-
"key": "top_category",
|
| 245 |
-
"label": "Top category",
|
| 246 |
-
"type": "text",
|
| 247 |
-
"source": "odoo",
|
| 248 |
-
"default": false,
|
| 249 |
-
"description": "The category this customer spent the most on in the last 12 months."
|
| 250 |
-
},
|
| 251 |
-
{
|
| 252 |
-
"key": "top_category_pct",
|
| 253 |
-
"label": "Top category %",
|
| 254 |
-
"type": "pct",
|
| 255 |
-
"source": "odoo",
|
| 256 |
-
"default": false,
|
| 257 |
-
"description": "Share of last-12-months spend that went to the top category."
|
| 258 |
-
},
|
| 259 |
-
{
|
| 260 |
-
"key": "sku_count",
|
| 261 |
-
"label": "SKUs bought",
|
| 262 |
-
"type": "int",
|
| 263 |
-
"source": "odoo",
|
| 264 |
-
"default": false,
|
| 265 |
-
"description": "Distinct products bought in the last 12 months."
|
| 266 |
-
},
|
| 267 |
-
{
|
| 268 |
-
"key": "top_sku",
|
| 269 |
-
"label": "Top SKU",
|
| 270 |
-
"type": "text",
|
| 271 |
-
"source": "odoo",
|
| 272 |
-
"default": false,
|
| 273 |
-
"description": "The product this customer spent the most on in the last 12 months."
|
| 274 |
-
},
|
| 275 |
-
{
|
| 276 |
-
"key": "days_since",
|
| 277 |
-
"label": "Days since order",
|
| 278 |
-
"type": "int",
|
| 279 |
-
"source": "odoo",
|
| 280 |
-
"default": false,
|
| 281 |
-
"description": "Days since the last confirmed order."
|
| 282 |
-
},
|
| 283 |
-
{
|
| 284 |
-
"key": "typical_gap_days",
|
| 285 |
-
"label": "Typical gap days",
|
| 286 |
-
"type": "int",
|
| 287 |
-
"source": "odoo",
|
| 288 |
-
"default": false,
|
| 289 |
-
"description": "Days this customer usually goes between orders, from their own history."
|
| 290 |
-
},
|
| 291 |
-
{
|
| 292 |
-
"key": "notes",
|
| 293 |
-
"label": "Notes",
|
| 294 |
-
"type": "text",
|
| 295 |
-
"source": "overlay",
|
| 296 |
-
"default": false,
|
| 297 |
-
"description": "Your notes on this customer. Saved in this app only, visible only to you."
|
| 298 |
-
}
|
| 299 |
-
],
|
| 300 |
-
"rows": [
|
| 301 |
-
{
|
| 302 |
-
"pid": 101,
|
| 303 |
-
"customer": "Poppy Flowers",
|
| 304 |
-
"status": "New",
|
| 305 |
-
"agent": "Naomi Linnell Rivera",
|
| 306 |
-
"city": "Charlottesville",
|
| 307 |
-
"state": "Virginia (US)",
|
| 308 |
-
"last_order": "2026-07-21",
|
| 309 |
-
"est_missed": 0,
|
| 310 |
-
"notes": "",
|
| 311 |
-
"country": "United States",
|
| 312 |
-
"zip": "02720",
|
| 313 |
-
"payment_terms": "30 Days",
|
| 314 |
-
"pricelist": "Fisch 1 (USD)",
|
| 315 |
-
"tags": "Royal",
|
| 316 |
-
"customer_since": "2023-01-10",
|
| 317 |
-
"salesperson": "Jessica",
|
| 318 |
-
"ar_open": 0,
|
| 319 |
-
"ar_overdue": 0,
|
| 320 |
-
"ar_exposure": 0,
|
| 321 |
-
"top_category": "Styrofoam",
|
| 322 |
-
"top_category_pct": 0.47,
|
| 323 |
-
"sku_count": 93,
|
| 324 |
-
"top_sku": "AQUAFOAM FLORAL FOAM BRICK | 48-Piece per Pack",
|
| 325 |
-
"days_to_pay": 34,
|
| 326 |
-
"_created": "2023-01-15 09:10:00",
|
| 327 |
-
"lat": 25.7617,
|
| 328 |
-
"lon": -80.1918,
|
| 329 |
-
"odoo_status": "Archived",
|
| 330 |
-
"dba": "Royal",
|
| 331 |
-
"ar_outstanding": 0
|
| 332 |
-
},
|
| 333 |
-
{
|
| 334 |
-
"pid": 102,
|
| 335 |
-
"customer": "Meadow & Vine Wholesale",
|
| 336 |
-
"status": "Growing",
|
| 337 |
-
"agent": "Carla Jimenez",
|
| 338 |
-
"city": "Portland",
|
| 339 |
-
"state": "Oregon (US)",
|
| 340 |
-
"last_order": "2026-07-19",
|
| 341 |
-
"est_missed": 0,
|
| 342 |
-
"notes": "Expanding to a second storefront.",
|
| 343 |
-
"country": "United States",
|
| 344 |
-
"zip": "77041",
|
| 345 |
-
"payment_terms": "Immediate Payment",
|
| 346 |
-
"pricelist": "Royal 1 (USD)",
|
| 347 |
-
"tags": "Royal, Key account",
|
| 348 |
-
"customer_since": "2024-02-11",
|
| 349 |
-
"salesperson": "Naomi",
|
| 350 |
-
"ar_open": 1240.5,
|
| 351 |
-
"ar_overdue": 0,
|
| 352 |
-
"ar_exposure": 1740.5,
|
| 353 |
-
"top_category": "Ribbon",
|
| 354 |
-
"top_category_pct": 0.95,
|
| 355 |
-
"sku_count": 4,
|
| 356 |
-
"top_sku": "2\" X 24\" X 36\" GREEN STYROFOAM BOARD",
|
| 357 |
-
"days_to_pay": null,
|
| 358 |
-
"_created": "2023-02-15 09:11:00",
|
| 359 |
-
"lat": 27.9506,
|
| 360 |
-
"lon": -82.4572,
|
| 361 |
-
"odoo_status": "Active",
|
| 362 |
-
"dba": "Fisch",
|
| 363 |
-
"ar_outstanding": 1240.5
|
| 364 |
-
},
|
| 365 |
-
{
|
| 366 |
-
"pid": 103,
|
| 367 |
-
"customer": "Bluestem Floral Supply",
|
| 368 |
-
"status": "Growing",
|
| 369 |
-
"agent": "Naomi Linnell Rivera",
|
| 370 |
-
"city": "Kansas City",
|
| 371 |
-
"state": "Missouri (US)",
|
| 372 |
-
"last_order": "2026-07-17",
|
| 373 |
-
"est_missed": 0,
|
| 374 |
-
"notes": "",
|
| 375 |
-
"country": "United States",
|
| 376 |
-
"zip": "11219",
|
| 377 |
-
"payment_terms": "60 Days",
|
| 378 |
-
"pricelist": "Royal 1 (USD)",
|
| 379 |
-
"tags": "Fisch",
|
| 380 |
-
"customer_since": "2025-03-12",
|
| 381 |
-
"salesperson": "Karen",
|
| 382 |
-
"ar_open": 0,
|
| 383 |
-
"ar_overdue": 5120.25,
|
| 384 |
-
"ar_exposure": 5120.25,
|
| 385 |
-
"top_category": "Foams & Finishes",
|
| 386 |
-
"top_category_pct": 0.31,
|
| 387 |
-
"sku_count": 27,
|
| 388 |
-
"top_sku": "SATIN RIBBON 2IN",
|
| 389 |
-
"days_to_pay": 61,
|
| 390 |
-
"_created": "2023-03-15 09:12:00",
|
| 391 |
-
"lat": 28.5384,
|
| 392 |
-
"lon": -81.3789,
|
| 393 |
-
"odoo_status": "Active",
|
| 394 |
-
"dba": "Both",
|
| 395 |
-
"ar_outstanding": 5120.25
|
| 396 |
-
},
|
| 397 |
-
{
|
| 398 |
-
"pid": 104,
|
| 399 |
-
"customer": "Camellia Row Florist",
|
| 400 |
-
"status": "Declining",
|
| 401 |
-
"agent": "Devon Marsh",
|
| 402 |
-
"city": "Savannah",
|
| 403 |
-
"state": "Georgia (US)",
|
| 404 |
-
"last_order": "2026-05-30",
|
| 405 |
-
"est_missed": 41200,
|
| 406 |
-
"notes": "Switched some volume to a local grower.",
|
| 407 |
-
"country": "United States",
|
| 408 |
-
"zip": "07649",
|
| 409 |
-
"payment_terms": "30 Days",
|
| 410 |
-
"pricelist": "Fisch 1 (USD)",
|
| 411 |
-
"tags": "(none)",
|
| 412 |
-
"customer_since": "2026-04-13",
|
| 413 |
-
"salesperson": "(none)",
|
| 414 |
-
"ar_open": 8300,
|
| 415 |
-
"ar_overdue": 940,
|
| 416 |
-
"ar_exposure": 11440,
|
| 417 |
-
"top_category": "All",
|
| 418 |
-
"top_category_pct": 1.0,
|
| 419 |
-
"sku_count": 1,
|
| 420 |
-
"top_sku": "(none)",
|
| 421 |
-
"days_to_pay": 12,
|
| 422 |
-
"_created": "2023-04-15 09:13:00",
|
| 423 |
-
"lat": 30.3322,
|
| 424 |
-
"lon": -81.6557,
|
| 425 |
-
"odoo_status": "Active",
|
| 426 |
-
"dba": "Royal",
|
| 427 |
-
"ar_outstanding": 9240
|
| 428 |
-
},
|
| 429 |
-
{
|
| 430 |
-
"pid": 105,
|
| 431 |
-
"customer": "Harborlight Wholesale Blooms",
|
| 432 |
-
"status": "Growing",
|
| 433 |
-
"agent": "Carla Jimenez",
|
| 434 |
-
"city": "Seattle",
|
| 435 |
-
"state": "Washington (US)",
|
| 436 |
-
"last_order": "2026-07-22",
|
| 437 |
-
"est_missed": 0,
|
| 438 |
-
"notes": "Top-10 account.",
|
| 439 |
-
"country": "United States",
|
| 440 |
-
"zip": "33125",
|
| 441 |
-
"payment_terms": "Immediate Payment",
|
| 442 |
-
"pricelist": "Royal 1 (USD)",
|
| 443 |
-
"tags": "Royal",
|
| 444 |
-
"customer_since": "2023-05-14",
|
| 445 |
-
"salesperson": "Jessica",
|
| 446 |
-
"ar_open": 0,
|
| 447 |
-
"ar_overdue": 0,
|
| 448 |
-
"ar_exposure": 0,
|
| 449 |
-
"top_category": "Styrofoam",
|
| 450 |
-
"top_category_pct": 0.47,
|
| 451 |
-
"sku_count": 93,
|
| 452 |
-
"top_sku": "AQUAFOAM FLORAL FOAM BRICK | 48-Piece per Pack",
|
| 453 |
-
"days_to_pay": 34,
|
| 454 |
-
"_created": "2023-05-15 09:14:00",
|
| 455 |
-
"lat": 26.1224,
|
| 456 |
-
"lon": -80.1373,
|
| 457 |
-
"odoo_status": "Active",
|
| 458 |
-
"dba": "",
|
| 459 |
-
"ar_outstanding": 0
|
| 460 |
-
},
|
| 461 |
-
{
|
| 462 |
-
"pid": 106,
|
| 463 |
-
"customer": "Dogwood & Fern Co.",
|
| 464 |
-
"status": "Dormant",
|
| 465 |
-
"agent": "Devon Marsh",
|
| 466 |
-
"city": "Asheville",
|
| 467 |
-
"state": "North Carolina (US)",
|
| 468 |
-
"last_order": "2026-02-11",
|
| 469 |
-
"est_missed": 52400,
|
| 470 |
-
"notes": "No spring order this year.",
|
| 471 |
-
"country": "United States",
|
| 472 |
-
"zip": "90210",
|
| 473 |
-
"payment_terms": "60 Days",
|
| 474 |
-
"pricelist": "Royal 1 (USD)",
|
| 475 |
-
"tags": "Royal, Key account",
|
| 476 |
-
"customer_since": "2024-06-15",
|
| 477 |
-
"salesperson": "Naomi",
|
| 478 |
-
"ar_open": 1240.5,
|
| 479 |
-
"ar_overdue": 0,
|
| 480 |
-
"ar_exposure": 1740.5,
|
| 481 |
-
"top_category": "Ribbon",
|
| 482 |
-
"top_category_pct": 0.95,
|
| 483 |
-
"sku_count": 4,
|
| 484 |
-
"top_sku": "2\" X 24\" X 36\" GREEN STYROFOAM BOARD",
|
| 485 |
-
"days_to_pay": null,
|
| 486 |
-
"_created": "2023-06-15 09:15:00",
|
| 487 |
-
"lat": 27.3364,
|
| 488 |
-
"lon": -82.5307,
|
| 489 |
-
"odoo_status": "Active",
|
| 490 |
-
"dba": "Fisch",
|
| 491 |
-
"ar_outstanding": 1240.5
|
| 492 |
-
},
|
| 493 |
-
{
|
| 494 |
-
"pid": 107,
|
| 495 |
-
"customer": "Verbena Market Florals",
|
| 496 |
-
"status": "Lost",
|
| 497 |
-
"agent": "Naomi Linnell Rivera",
|
| 498 |
-
"city": "Austin",
|
| 499 |
-
"state": "Texas (US)",
|
| 500 |
-
"last_order": "2025-11-04",
|
| 501 |
-
"est_missed": 78300,
|
| 502 |
-
"notes": "Went with a competitor on freight terms.",
|
| 503 |
-
"country": "United States",
|
| 504 |
-
"zip": "08701",
|
| 505 |
-
"payment_terms": "30 Days",
|
| 506 |
-
"pricelist": "Fisch 1 (USD)",
|
| 507 |
-
"tags": "Fisch",
|
| 508 |
-
"customer_since": "2025-07-16",
|
| 509 |
-
"salesperson": "Karen",
|
| 510 |
-
"ar_open": 0,
|
| 511 |
-
"ar_overdue": 5120.25,
|
| 512 |
-
"ar_exposure": 5120.25,
|
| 513 |
-
"top_category": "Foams & Finishes",
|
| 514 |
-
"top_category_pct": 0.31,
|
| 515 |
-
"sku_count": 27,
|
| 516 |
-
"top_sku": "SATIN RIBBON 2IN",
|
| 517 |
-
"days_to_pay": 61,
|
| 518 |
-
"_created": "2023-07-15 09:16:00",
|
| 519 |
-
"lat": null,
|
| 520 |
-
"lon": null,
|
| 521 |
-
"odoo_status": "Active",
|
| 522 |
-
"dba": "Both",
|
| 523 |
-
"ar_outstanding": 5120.25
|
| 524 |
-
},
|
| 525 |
-
{
|
| 526 |
-
"pid": 108,
|
| 527 |
-
"customer": "Larkspur Lane Supply",
|
| 528 |
-
"status": "New",
|
| 529 |
-
"agent": "Carla Jimenez",
|
| 530 |
-
"city": "Denver",
|
| 531 |
-
"state": "Colorado (US)",
|
| 532 |
-
"last_order": "2026-07-14",
|
| 533 |
-
"est_missed": 0,
|
| 534 |
-
"notes": "First order in April.",
|
| 535 |
-
"country": "United States",
|
| 536 |
-
"zip": "60614",
|
| 537 |
-
"payment_terms": "Immediate Payment",
|
| 538 |
-
"pricelist": "Royal 1 (USD)",
|
| 539 |
-
"tags": "(none)",
|
| 540 |
-
"customer_since": "2026-08-17",
|
| 541 |
-
"salesperson": "(none)",
|
| 542 |
-
"ar_open": 8300,
|
| 543 |
-
"ar_overdue": 940,
|
| 544 |
-
"ar_exposure": 11440,
|
| 545 |
-
"top_category": "All",
|
| 546 |
-
"top_category_pct": 1.0,
|
| 547 |
-
"sku_count": 1,
|
| 548 |
-
"top_sku": "(none)",
|
| 549 |
-
"days_to_pay": 12,
|
| 550 |
-
"_created": "2023-08-15 09:17:00",
|
| 551 |
-
"lat": 33.749,
|
| 552 |
-
"lon": -84.388,
|
| 553 |
-
"odoo_status": "Active",
|
| 554 |
-
"dba": "Royal",
|
| 555 |
-
"ar_outstanding": 9240
|
| 556 |
-
}
|
| 557 |
-
]
|
| 558 |
-
}
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"fields": [
|
| 3 |
+
{
|
| 4 |
+
"key": "customer",
|
| 5 |
+
"label": "Customer",
|
| 6 |
+
"type": "text",
|
| 7 |
+
"source": "odoo",
|
| 8 |
+
"pinned": true,
|
| 9 |
+
"default": true,
|
| 10 |
+
"description": "The customer's name in Odoo. One row per customer who ordered in the last 24 months."
|
| 11 |
+
},
|
| 12 |
+
{
|
| 13 |
+
"key": "partner_id",
|
| 14 |
+
"label": "Odoo ID",
|
| 15 |
+
"type": "int",
|
| 16 |
+
"source": "odoo",
|
| 17 |
+
"derived": true,
|
| 18 |
+
"default": false,
|
| 19 |
+
"description": "The Odoo res.partner id β the key every Odoo document joins on. DERIVED: this row's pid IS the partner id, so a stored copy would be a second source."
|
| 20 |
+
},
|
| 21 |
+
{
|
| 22 |
+
"key": "odoo_status",
|
| 23 |
+
"label": "Odoo record",
|
| 24 |
+
"type": "status",
|
| 25 |
+
"source": "odoo",
|
| 26 |
+
"default": false,
|
| 27 |
+
"options": [
|
| 28 |
+
"Active",
|
| 29 |
+
"Archived"
|
| 30 |
+
],
|
| 31 |
+
"description": "Whether this customer still exists in Odoo. Archived means deleted there."
|
| 32 |
+
},
|
| 33 |
+
{
|
| 34 |
+
"key": "agent",
|
| 35 |
+
"label": "Agent",
|
| 36 |
+
"type": "text",
|
| 37 |
+
"source": "odoo",
|
| 38 |
+
"default": true,
|
| 39 |
+
"description": "The sales agent who owns this account."
|
| 40 |
+
},
|
| 41 |
+
{
|
| 42 |
+
"key": "dba",
|
| 43 |
+
"label": "DBA",
|
| 44 |
+
"type": "select",
|
| 45 |
+
"source": "odoo",
|
| 46 |
+
"default": false,
|
| 47 |
+
"options": [
|
| 48 |
+
"Fisch",
|
| 49 |
+
"Royal",
|
| 50 |
+
"Both"
|
| 51 |
+
],
|
| 52 |
+
"description": "The brand this customer buys from - Fisch, Royal, or both. Amazon-channel orders are not a DBA."
|
| 53 |
+
},
|
| 54 |
+
{
|
| 55 |
+
"key": "salesperson",
|
| 56 |
+
"label": "Salesperson",
|
| 57 |
+
"type": "text",
|
| 58 |
+
"source": "odoo",
|
| 59 |
+
"default": false,
|
| 60 |
+
"description": "Who keyed in most of this customer's orders β not the Agent, who owns the account."
|
| 61 |
+
},
|
| 62 |
+
{
|
| 63 |
+
"key": "street",
|
| 64 |
+
"label": "Street",
|
| 65 |
+
"type": "text",
|
| 66 |
+
"source": "odoo",
|
| 67 |
+
"default": false,
|
| 68 |
+
"description": "First address line, from res.partner directly - not the geocoder, so a customer the map cannot place still shows its address."
|
| 69 |
+
},
|
| 70 |
+
{
|
| 71 |
+
"key": "street2",
|
| 72 |
+
"label": "Street 2",
|
| 73 |
+
"type": "text",
|
| 74 |
+
"source": "odoo",
|
| 75 |
+
"default": false,
|
| 76 |
+
"description": "Second address line (suite, unit, floor) on the customer's Odoo address."
|
| 77 |
+
},
|
| 78 |
+
{
|
| 79 |
+
"key": "city",
|
| 80 |
+
"label": "City",
|
| 81 |
+
"type": "text",
|
| 82 |
+
"source": "odoo",
|
| 83 |
+
"default": true,
|
| 84 |
+
"description": "City on the customer's Odoo address."
|
| 85 |
+
},
|
| 86 |
+
{
|
| 87 |
+
"key": "state",
|
| 88 |
+
"label": "State",
|
| 89 |
+
"type": "text",
|
| 90 |
+
"source": "odoo",
|
| 91 |
+
"default": true,
|
| 92 |
+
"description": "State or province on the customer's Odoo address."
|
| 93 |
+
},
|
| 94 |
+
{
|
| 95 |
+
"key": "country",
|
| 96 |
+
"label": "Country",
|
| 97 |
+
"type": "text",
|
| 98 |
+
"source": "odoo",
|
| 99 |
+
"default": false,
|
| 100 |
+
"description": "Country on the customer's Odoo address."
|
| 101 |
+
},
|
| 102 |
+
{
|
| 103 |
+
"key": "zip",
|
| 104 |
+
"label": "ZIP",
|
| 105 |
+
"type": "text",
|
| 106 |
+
"source": "odoo",
|
| 107 |
+
"default": false,
|
| 108 |
+
"description": "Postal code on the customer's Odoo address."
|
| 109 |
+
},
|
| 110 |
+
{
|
| 111 |
+
"key": "customer_since",
|
| 112 |
+
"label": "Customer since",
|
| 113 |
+
"type": "date",
|
| 114 |
+
"source": "odoo",
|
| 115 |
+
"default": false,
|
| 116 |
+
"description": "When this customer was first set up in Odoo."
|
| 117 |
+
},
|
| 118 |
+
{
|
| 119 |
+
"key": "tags",
|
| 120 |
+
"label": "Tags",
|
| 121 |
+
"type": "text",
|
| 122 |
+
"source": "odoo",
|
| 123 |
+
"default": false,
|
| 124 |
+
"description": "Odoo labels on this customer, comma-separated."
|
| 125 |
+
},
|
| 126 |
+
{
|
| 127 |
+
"key": "pricelist",
|
| 128 |
+
"label": "Price list",
|
| 129 |
+
"type": "text",
|
| 130 |
+
"source": "odoo",
|
| 131 |
+
"default": false,
|
| 132 |
+
"description": "The price list this customer buys on."
|
| 133 |
+
},
|
| 134 |
+
{
|
| 135 |
+
"key": "payment_terms",
|
| 136 |
+
"label": "Payment terms",
|
| 137 |
+
"type": "text",
|
| 138 |
+
"source": "odoo",
|
| 139 |
+
"default": false,
|
| 140 |
+
"description": "Payment terms on this customer's account β Net 30, for example."
|
| 141 |
+
},
|
| 142 |
+
{
|
| 143 |
+
"key": "last_order",
|
| 144 |
+
"label": "Last order",
|
| 145 |
+
"type": "date",
|
| 146 |
+
"source": "odoo",
|
| 147 |
+
"default": true,
|
| 148 |
+
"description": "Date of the most recent confirmed order."
|
| 149 |
+
},
|
| 150 |
+
{
|
| 151 |
+
"key": "overdue_days",
|
| 152 |
+
"label": "Overdue days",
|
| 153 |
+
"type": "int",
|
| 154 |
+
"source": "odoo",
|
| 155 |
+
"default": true,
|
| 156 |
+
"description": "How many days late this customer is running against their own usual ordering rhythm."
|
| 157 |
+
},
|
| 158 |
+
{
|
| 159 |
+
"_note": "filterable:false β DERIVED ANALYTIC: est_missed is min(cycles missed, 3) x AOV, a score we compute rather than an object the business has, so a condition on it would read as a fact about the customer when it is a fact about our arithmetic. It still displays and still sorts. Until wave 6 this flag also covered the frozen-window presets (revenue_ytd / revenue_ly / orders_24m / aov / yoy_pct); those are now DELETED outright under the owner's no-buildable-presets rule β see _comment. est_missed itself STAYS: no creatable measure or formula reproduces the cadence model behind it.",
|
| 160 |
+
"key": "est_missed",
|
| 161 |
+
"label": "Est. missed $",
|
| 162 |
+
"type": "currency",
|
| 163 |
+
"source": "odoo",
|
| 164 |
+
"default": true,
|
| 165 |
+
"agg": "sum",
|
| 166 |
+
"filterable": false,
|
| 167 |
+
"description": "Estimated sales missed while quiet: missed orders (capped at 3) times average order value. An estimate, not money owed."
|
| 168 |
+
},
|
| 169 |
+
{
|
| 170 |
+
"_note": "wave 21 R1 β KEY UNCHANGED, LABEL RENAMED. The computation is a DISJOINT split (ar.py credit_exposure): this column is only the not-yet-due residual, its sibling is the past-grace residual, and the two sum to the total. Under the label 'AR open $' the majority-late book read as 'Overdue > Open', which is nonsense in AR vocabulary β 'open' universally means the total. The label now says what the number is; the key stays so saved views and filters keep working.",
|
| 171 |
+
"key": "ar_open",
|
| 172 |
+
"label": "AR current $",
|
| 173 |
+
"type": "currency",
|
| 174 |
+
"source": "odoo",
|
| 175 |
+
"default": false,
|
| 176 |
+
"description": "Invoiced money owed but not yet due (a 5-day grace applies before it counts as overdue)."
|
| 177 |
+
},
|
| 178 |
+
{
|
| 179 |
+
"key": "ar_overdue",
|
| 180 |
+
"label": "AR overdue $",
|
| 181 |
+
"type": "currency",
|
| 182 |
+
"source": "odoo",
|
| 183 |
+
"default": false,
|
| 184 |
+
"description": "Invoiced money past due β same basis as the Collections page."
|
| 185 |
+
},
|
| 186 |
+
{
|
| 187 |
+
"_note": "wave 21 R1 β the TOTAL, added beside the rename above. AR current $ + AR overdue $, i.e. what most people mean by 'open AR'. Composed from the same ar.credit_exposure rows the siblings use, so it is transitively reconciled by ar.validate()'s residual read_group tie β no second oracle.",
|
| 188 |
+
"key": "ar_outstanding",
|
| 189 |
+
"label": "AR outstanding $",
|
| 190 |
+
"type": "currency",
|
| 191 |
+
"source": "odoo",
|
| 192 |
+
"default": false,
|
| 193 |
+
"description": "Total invoiced money owed right now: AR current $ plus AR overdue $."
|
| 194 |
+
},
|
| 195 |
+
{
|
| 196 |
+
"key": "ar_exposure",
|
| 197 |
+
"label": "Credit exposure $",
|
| 198 |
+
"type": "currency",
|
| 199 |
+
"source": "odoo",
|
| 200 |
+
"default": false,
|
| 201 |
+
"description": "The most you could be out if they stopped paying today: open, overdue, draft and not-yet-invoiced."
|
| 202 |
+
},
|
| 203 |
+
{
|
| 204 |
+
"key": "ar_aged_1_30",
|
| 205 |
+
"label": "1-30 days $",
|
| 206 |
+
"type": "currency",
|
| 207 |
+
"source": "odoo",
|
| 208 |
+
"default": false,
|
| 209 |
+
"description": "Overdue between 1 and 30 days. The four aging buckets sum to AR overdue $."
|
| 210 |
+
},
|
| 211 |
+
{
|
| 212 |
+
"key": "ar_aged_31_60",
|
| 213 |
+
"label": "31-60 days $",
|
| 214 |
+
"type": "currency",
|
| 215 |
+
"source": "odoo",
|
| 216 |
+
"default": false,
|
| 217 |
+
"description": "Overdue between 31 and 60 days. The four aging buckets sum to AR overdue $."
|
| 218 |
+
},
|
| 219 |
+
{
|
| 220 |
+
"key": "ar_aged_61_90",
|
| 221 |
+
"label": "61-90 days $",
|
| 222 |
+
"type": "currency",
|
| 223 |
+
"source": "odoo",
|
| 224 |
+
"default": false,
|
| 225 |
+
"description": "Overdue between 61 and 90 days. The four aging buckets sum to AR overdue $."
|
| 226 |
+
},
|
| 227 |
+
{
|
| 228 |
+
"key": "ar_aged_90_plus",
|
| 229 |
+
"label": "90+ days $",
|
| 230 |
+
"type": "currency",
|
| 231 |
+
"source": "odoo",
|
| 232 |
+
"default": false,
|
| 233 |
+
"description": "Overdue by more than 90 days. The four aging buckets sum to AR overdue $."
|
| 234 |
+
},
|
| 235 |
+
{
|
| 236 |
+
"key": "days_to_pay",
|
| 237 |
+
"label": "Days to pay",
|
| 238 |
+
"type": "int",
|
| 239 |
+
"source": "odoo",
|
| 240 |
+
"default": false,
|
| 241 |
+
"description": "Average days to pay an invoice in full. Blank means no fully paid invoice yet."
|
| 242 |
+
},
|
| 243 |
+
{
|
| 244 |
+
"key": "top_category",
|
| 245 |
+
"label": "Top category",
|
| 246 |
+
"type": "text",
|
| 247 |
+
"source": "odoo",
|
| 248 |
+
"default": false,
|
| 249 |
+
"description": "The category this customer spent the most on in the last 12 months."
|
| 250 |
+
},
|
| 251 |
+
{
|
| 252 |
+
"key": "top_category_pct",
|
| 253 |
+
"label": "Top category %",
|
| 254 |
+
"type": "pct",
|
| 255 |
+
"source": "odoo",
|
| 256 |
+
"default": false,
|
| 257 |
+
"description": "Share of last-12-months spend that went to the top category."
|
| 258 |
+
},
|
| 259 |
+
{
|
| 260 |
+
"key": "sku_count",
|
| 261 |
+
"label": "SKUs bought",
|
| 262 |
+
"type": "int",
|
| 263 |
+
"source": "odoo",
|
| 264 |
+
"default": false,
|
| 265 |
+
"description": "Distinct products bought in the last 12 months."
|
| 266 |
+
},
|
| 267 |
+
{
|
| 268 |
+
"key": "top_sku",
|
| 269 |
+
"label": "Top SKU",
|
| 270 |
+
"type": "text",
|
| 271 |
+
"source": "odoo",
|
| 272 |
+
"default": false,
|
| 273 |
+
"description": "The product this customer spent the most on in the last 12 months."
|
| 274 |
+
},
|
| 275 |
+
{
|
| 276 |
+
"key": "days_since",
|
| 277 |
+
"label": "Days since order",
|
| 278 |
+
"type": "int",
|
| 279 |
+
"source": "odoo",
|
| 280 |
+
"default": false,
|
| 281 |
+
"description": "Days since the last confirmed order."
|
| 282 |
+
},
|
| 283 |
+
{
|
| 284 |
+
"key": "typical_gap_days",
|
| 285 |
+
"label": "Typical gap days",
|
| 286 |
+
"type": "int",
|
| 287 |
+
"source": "odoo",
|
| 288 |
+
"default": false,
|
| 289 |
+
"description": "Days this customer usually goes between orders, from their own history."
|
| 290 |
+
},
|
| 291 |
+
{
|
| 292 |
+
"key": "notes",
|
| 293 |
+
"label": "Notes",
|
| 294 |
+
"type": "text",
|
| 295 |
+
"source": "overlay",
|
| 296 |
+
"default": false,
|
| 297 |
+
"description": "Your notes on this customer. Saved in this app only, visible only to you."
|
| 298 |
+
}
|
| 299 |
+
],
|
| 300 |
+
"rows": [
|
| 301 |
+
{
|
| 302 |
+
"pid": 101,
|
| 303 |
+
"customer": "Poppy Flowers",
|
| 304 |
+
"status": "New",
|
| 305 |
+
"agent": "Naomi Linnell Rivera",
|
| 306 |
+
"city": "Charlottesville",
|
| 307 |
+
"state": "Virginia (US)",
|
| 308 |
+
"last_order": "2026-07-21",
|
| 309 |
+
"est_missed": 0,
|
| 310 |
+
"notes": "",
|
| 311 |
+
"country": "United States",
|
| 312 |
+
"zip": "02720",
|
| 313 |
+
"payment_terms": "30 Days",
|
| 314 |
+
"pricelist": "Fisch 1 (USD)",
|
| 315 |
+
"tags": "Royal",
|
| 316 |
+
"customer_since": "2023-01-10",
|
| 317 |
+
"salesperson": "Jessica",
|
| 318 |
+
"ar_open": 0,
|
| 319 |
+
"ar_overdue": 0,
|
| 320 |
+
"ar_exposure": 0,
|
| 321 |
+
"top_category": "Styrofoam",
|
| 322 |
+
"top_category_pct": 0.47,
|
| 323 |
+
"sku_count": 93,
|
| 324 |
+
"top_sku": "AQUAFOAM FLORAL FOAM BRICK | 48-Piece per Pack",
|
| 325 |
+
"days_to_pay": 34,
|
| 326 |
+
"_created": "2023-01-15 09:10:00",
|
| 327 |
+
"lat": 25.7617,
|
| 328 |
+
"lon": -80.1918,
|
| 329 |
+
"odoo_status": "Archived",
|
| 330 |
+
"dba": "Royal",
|
| 331 |
+
"ar_outstanding": 0
|
| 332 |
+
},
|
| 333 |
+
{
|
| 334 |
+
"pid": 102,
|
| 335 |
+
"customer": "Meadow & Vine Wholesale",
|
| 336 |
+
"status": "Growing",
|
| 337 |
+
"agent": "Carla Jimenez",
|
| 338 |
+
"city": "Portland",
|
| 339 |
+
"state": "Oregon (US)",
|
| 340 |
+
"last_order": "2026-07-19",
|
| 341 |
+
"est_missed": 0,
|
| 342 |
+
"notes": "Expanding to a second storefront.",
|
| 343 |
+
"country": "United States",
|
| 344 |
+
"zip": "77041",
|
| 345 |
+
"payment_terms": "Immediate Payment",
|
| 346 |
+
"pricelist": "Royal 1 (USD)",
|
| 347 |
+
"tags": "Royal, Key account",
|
| 348 |
+
"customer_since": "2024-02-11",
|
| 349 |
+
"salesperson": "Naomi",
|
| 350 |
+
"ar_open": 1240.5,
|
| 351 |
+
"ar_overdue": 0,
|
| 352 |
+
"ar_exposure": 1740.5,
|
| 353 |
+
"top_category": "Ribbon",
|
| 354 |
+
"top_category_pct": 0.95,
|
| 355 |
+
"sku_count": 4,
|
| 356 |
+
"top_sku": "2\" X 24\" X 36\" GREEN STYROFOAM BOARD",
|
| 357 |
+
"days_to_pay": null,
|
| 358 |
+
"_created": "2023-02-15 09:11:00",
|
| 359 |
+
"lat": 27.9506,
|
| 360 |
+
"lon": -82.4572,
|
| 361 |
+
"odoo_status": "Active",
|
| 362 |
+
"dba": "Fisch",
|
| 363 |
+
"ar_outstanding": 1240.5
|
| 364 |
+
},
|
| 365 |
+
{
|
| 366 |
+
"pid": 103,
|
| 367 |
+
"customer": "Bluestem Floral Supply",
|
| 368 |
+
"status": "Growing",
|
| 369 |
+
"agent": "Naomi Linnell Rivera",
|
| 370 |
+
"city": "Kansas City",
|
| 371 |
+
"state": "Missouri (US)",
|
| 372 |
+
"last_order": "2026-07-17",
|
| 373 |
+
"est_missed": 0,
|
| 374 |
+
"notes": "",
|
| 375 |
+
"country": "United States",
|
| 376 |
+
"zip": "11219",
|
| 377 |
+
"payment_terms": "60 Days",
|
| 378 |
+
"pricelist": "Royal 1 (USD)",
|
| 379 |
+
"tags": "Fisch",
|
| 380 |
+
"customer_since": "2025-03-12",
|
| 381 |
+
"salesperson": "Karen",
|
| 382 |
+
"ar_open": 0,
|
| 383 |
+
"ar_overdue": 5120.25,
|
| 384 |
+
"ar_exposure": 5120.25,
|
| 385 |
+
"top_category": "Foams & Finishes",
|
| 386 |
+
"top_category_pct": 0.31,
|
| 387 |
+
"sku_count": 27,
|
| 388 |
+
"top_sku": "SATIN RIBBON 2IN",
|
| 389 |
+
"days_to_pay": 61,
|
| 390 |
+
"_created": "2023-03-15 09:12:00",
|
| 391 |
+
"lat": 28.5384,
|
| 392 |
+
"lon": -81.3789,
|
| 393 |
+
"odoo_status": "Active",
|
| 394 |
+
"dba": "Both",
|
| 395 |
+
"ar_outstanding": 5120.25
|
| 396 |
+
},
|
| 397 |
+
{
|
| 398 |
+
"pid": 104,
|
| 399 |
+
"customer": "Camellia Row Florist",
|
| 400 |
+
"status": "Declining",
|
| 401 |
+
"agent": "Devon Marsh",
|
| 402 |
+
"city": "Savannah",
|
| 403 |
+
"state": "Georgia (US)",
|
| 404 |
+
"last_order": "2026-05-30",
|
| 405 |
+
"est_missed": 41200,
|
| 406 |
+
"notes": "Switched some volume to a local grower.",
|
| 407 |
+
"country": "United States",
|
| 408 |
+
"zip": "07649",
|
| 409 |
+
"payment_terms": "30 Days",
|
| 410 |
+
"pricelist": "Fisch 1 (USD)",
|
| 411 |
+
"tags": "(none)",
|
| 412 |
+
"customer_since": "2026-04-13",
|
| 413 |
+
"salesperson": "(none)",
|
| 414 |
+
"ar_open": 8300,
|
| 415 |
+
"ar_overdue": 940,
|
| 416 |
+
"ar_exposure": 11440,
|
| 417 |
+
"top_category": "All",
|
| 418 |
+
"top_category_pct": 1.0,
|
| 419 |
+
"sku_count": 1,
|
| 420 |
+
"top_sku": "(none)",
|
| 421 |
+
"days_to_pay": 12,
|
| 422 |
+
"_created": "2023-04-15 09:13:00",
|
| 423 |
+
"lat": 30.3322,
|
| 424 |
+
"lon": -81.6557,
|
| 425 |
+
"odoo_status": "Active",
|
| 426 |
+
"dba": "Royal",
|
| 427 |
+
"ar_outstanding": 9240
|
| 428 |
+
},
|
| 429 |
+
{
|
| 430 |
+
"pid": 105,
|
| 431 |
+
"customer": "Harborlight Wholesale Blooms",
|
| 432 |
+
"status": "Growing",
|
| 433 |
+
"agent": "Carla Jimenez",
|
| 434 |
+
"city": "Seattle",
|
| 435 |
+
"state": "Washington (US)",
|
| 436 |
+
"last_order": "2026-07-22",
|
| 437 |
+
"est_missed": 0,
|
| 438 |
+
"notes": "Top-10 account.",
|
| 439 |
+
"country": "United States",
|
| 440 |
+
"zip": "33125",
|
| 441 |
+
"payment_terms": "Immediate Payment",
|
| 442 |
+
"pricelist": "Royal 1 (USD)",
|
| 443 |
+
"tags": "Royal",
|
| 444 |
+
"customer_since": "2023-05-14",
|
| 445 |
+
"salesperson": "Jessica",
|
| 446 |
+
"ar_open": 0,
|
| 447 |
+
"ar_overdue": 0,
|
| 448 |
+
"ar_exposure": 0,
|
| 449 |
+
"top_category": "Styrofoam",
|
| 450 |
+
"top_category_pct": 0.47,
|
| 451 |
+
"sku_count": 93,
|
| 452 |
+
"top_sku": "AQUAFOAM FLORAL FOAM BRICK | 48-Piece per Pack",
|
| 453 |
+
"days_to_pay": 34,
|
| 454 |
+
"_created": "2023-05-15 09:14:00",
|
| 455 |
+
"lat": 26.1224,
|
| 456 |
+
"lon": -80.1373,
|
| 457 |
+
"odoo_status": "Active",
|
| 458 |
+
"dba": "",
|
| 459 |
+
"ar_outstanding": 0
|
| 460 |
+
},
|
| 461 |
+
{
|
| 462 |
+
"pid": 106,
|
| 463 |
+
"customer": "Dogwood & Fern Co.",
|
| 464 |
+
"status": "Dormant",
|
| 465 |
+
"agent": "Devon Marsh",
|
| 466 |
+
"city": "Asheville",
|
| 467 |
+
"state": "North Carolina (US)",
|
| 468 |
+
"last_order": "2026-02-11",
|
| 469 |
+
"est_missed": 52400,
|
| 470 |
+
"notes": "No spring order this year.",
|
| 471 |
+
"country": "United States",
|
| 472 |
+
"zip": "90210",
|
| 473 |
+
"payment_terms": "60 Days",
|
| 474 |
+
"pricelist": "Royal 1 (USD)",
|
| 475 |
+
"tags": "Royal, Key account",
|
| 476 |
+
"customer_since": "2024-06-15",
|
| 477 |
+
"salesperson": "Naomi",
|
| 478 |
+
"ar_open": 1240.5,
|
| 479 |
+
"ar_overdue": 0,
|
| 480 |
+
"ar_exposure": 1740.5,
|
| 481 |
+
"top_category": "Ribbon",
|
| 482 |
+
"top_category_pct": 0.95,
|
| 483 |
+
"sku_count": 4,
|
| 484 |
+
"top_sku": "2\" X 24\" X 36\" GREEN STYROFOAM BOARD",
|
| 485 |
+
"days_to_pay": null,
|
| 486 |
+
"_created": "2023-06-15 09:15:00",
|
| 487 |
+
"lat": 27.3364,
|
| 488 |
+
"lon": -82.5307,
|
| 489 |
+
"odoo_status": "Active",
|
| 490 |
+
"dba": "Fisch",
|
| 491 |
+
"ar_outstanding": 1240.5
|
| 492 |
+
},
|
| 493 |
+
{
|
| 494 |
+
"pid": 107,
|
| 495 |
+
"customer": "Verbena Market Florals",
|
| 496 |
+
"status": "Lost",
|
| 497 |
+
"agent": "Naomi Linnell Rivera",
|
| 498 |
+
"city": "Austin",
|
| 499 |
+
"state": "Texas (US)",
|
| 500 |
+
"last_order": "2025-11-04",
|
| 501 |
+
"est_missed": 78300,
|
| 502 |
+
"notes": "Went with a competitor on freight terms.",
|
| 503 |
+
"country": "United States",
|
| 504 |
+
"zip": "08701",
|
| 505 |
+
"payment_terms": "30 Days",
|
| 506 |
+
"pricelist": "Fisch 1 (USD)",
|
| 507 |
+
"tags": "Fisch",
|
| 508 |
+
"customer_since": "2025-07-16",
|
| 509 |
+
"salesperson": "Karen",
|
| 510 |
+
"ar_open": 0,
|
| 511 |
+
"ar_overdue": 5120.25,
|
| 512 |
+
"ar_exposure": 5120.25,
|
| 513 |
+
"top_category": "Foams & Finishes",
|
| 514 |
+
"top_category_pct": 0.31,
|
| 515 |
+
"sku_count": 27,
|
| 516 |
+
"top_sku": "SATIN RIBBON 2IN",
|
| 517 |
+
"days_to_pay": 61,
|
| 518 |
+
"_created": "2023-07-15 09:16:00",
|
| 519 |
+
"lat": null,
|
| 520 |
+
"lon": null,
|
| 521 |
+
"odoo_status": "Active",
|
| 522 |
+
"dba": "Both",
|
| 523 |
+
"ar_outstanding": 5120.25
|
| 524 |
+
},
|
| 525 |
+
{
|
| 526 |
+
"pid": 108,
|
| 527 |
+
"customer": "Larkspur Lane Supply",
|
| 528 |
+
"status": "New",
|
| 529 |
+
"agent": "Carla Jimenez",
|
| 530 |
+
"city": "Denver",
|
| 531 |
+
"state": "Colorado (US)",
|
| 532 |
+
"last_order": "2026-07-14",
|
| 533 |
+
"est_missed": 0,
|
| 534 |
+
"notes": "First order in April.",
|
| 535 |
+
"country": "United States",
|
| 536 |
+
"zip": "60614",
|
| 537 |
+
"payment_terms": "Immediate Payment",
|
| 538 |
+
"pricelist": "Royal 1 (USD)",
|
| 539 |
+
"tags": "(none)",
|
| 540 |
+
"customer_since": "2026-08-17",
|
| 541 |
+
"salesperson": "(none)",
|
| 542 |
+
"ar_open": 8300,
|
| 543 |
+
"ar_overdue": 940,
|
| 544 |
+
"ar_exposure": 11440,
|
| 545 |
+
"top_category": "All",
|
| 546 |
+
"top_category_pct": 1.0,
|
| 547 |
+
"sku_count": 1,
|
| 548 |
+
"top_sku": "(none)",
|
| 549 |
+
"days_to_pay": 12,
|
| 550 |
+
"_created": "2023-08-15 09:17:00",
|
| 551 |
+
"lat": 33.749,
|
| 552 |
+
"lon": -84.388,
|
| 553 |
+
"odoo_status": "Active",
|
| 554 |
+
"dba": "Royal",
|
| 555 |
+
"ar_outstanding": 9240
|
| 556 |
+
}
|
| 557 |
+
]
|
| 558 |
+
}
|
web/src/alerts/alertsModel.ts
CHANGED
|
@@ -1,418 +1,418 @@
|
|
| 1 |
-
// ---------------------------------------------------------------------------
|
| 2 |
-
// alerts/alertsModel.ts β WAVE 20 item 25 (contract C-ALERT): the inbox's PURE
|
| 3 |
-
// half. React-free and fetch-free, so `verify_alerts.py` runs it under node.
|
| 4 |
-
//
|
| 5 |
-
// An alert says "tell me when a record ENTERS this view". The client half is
|
| 6 |
-
// small, and every part of it fails silently when it is wrong:
|
| 7 |
-
//
|
| 8 |
-
// Β· an unread count taken from `items.length` rather than from the server's
|
| 9 |
-
// own `unread` disagrees with the badge the moment a page is capped or a
|
| 10 |
-
// read lands in another tab β and a badge that says 3 when the list shows 9
|
| 11 |
-
// teaches the reader to ignore the badge;
|
| 12 |
-
// Β· a notification whose `viewId` no longer resolves must open NOTHING rather
|
| 13 |
-
// than a wrong view β alerts outlive the views they were made from;
|
| 14 |
-
// Β· `topic` is the surface's scope key ("customer", "product", "ut_β¦"), and
|
| 15 |
-
// the ROUTE is a registry key ("customer_data") β mapping one to the other
|
| 16 |
-
// by guesswork sends every click to a page that does not exist.
|
| 17 |
-
// ---------------------------------------------------------------------------
|
| 18 |
-
|
| 19 |
-
/**
|
| 20 |
-
* ββ WAVE 32 Β· T20 Β· CONTRACT C3 β WHERE AN INBOX ITEM OPENS.
|
| 21 |
-
*
|
| 22 |
-
* `module` is the destination surface (`"database"` or `"automation"`) and `id` is what to open
|
| 23 |
-
* in it. `tab` is the SUB-SELECTION inside that module: the literal `"runs"` for an automation's
|
| 24 |
-
* run log, or the VIEW ID to select on a database. One optional key, two destinations.
|
| 25 |
-
*
|
| 26 |
-
* β EVERY FIELD IS A PLAIN `string`, NEVER A UNION, and that is alertsModel's wave-9 law rather
|
| 27 |
-
* than laziness: the vocabulary is the SERVER's, and a client union over it turns "the server grew
|
| 28 |
-
* a module" into "the client silently drops the row". An unknown module is refused by the
|
| 29 |
-
* FRAME's dispatcher, out loud, which is a different thing from never arriving.
|
| 30 |
-
*/
|
| 31 |
-
export interface NotificationTarget {
|
| 32 |
-
module: string;
|
| 33 |
-
id: string;
|
| 34 |
-
tab?: string;
|
| 35 |
-
}
|
| 36 |
-
|
| 37 |
-
/** C3's `kind` vocabulary, mirroring `routes_alerts.NOTIF_KIND_*`. Held as constants so the
|
| 38 |
-
* Inbox's branch and the gate's fixtures cannot disagree about the word. */
|
| 39 |
-
export const NOTIF_KIND_ALERT = "alert";
|
| 40 |
-
export const NOTIF_KIND_AUTOMATION = "automation";
|
| 41 |
-
export const NOTIF_KIND_SHARE = "share";
|
| 42 |
-
|
| 43 |
-
/** C3's `target.module` vocabulary, and the automation sub-selection. */
|
| 44 |
-
export const TARGET_MODULE_DATABASE = "database";
|
| 45 |
-
export const TARGET_MODULE_AUTOMATION = "automation";
|
| 46 |
-
export const TARGET_TAB_RUNS = "runs";
|
| 47 |
-
|
| 48 |
-
/** One notification, as `GET /api/v1/notifications` sends it. */
|
| 49 |
-
export interface Notification {
|
| 50 |
-
id: string;
|
| 51 |
-
alertId: string;
|
| 52 |
-
viewId: string;
|
| 53 |
-
topic: string;
|
| 54 |
-
rowId: string;
|
| 55 |
-
/** What entered β the record's own label. */
|
| 56 |
-
label: string;
|
| 57 |
-
/** What the alert is called, so a row reads without opening anything. */
|
| 58 |
-
alertLabel: string;
|
| 59 |
-
/** UTC WITH OFFSET (D-18). Kept as the server's STRING: re-formatting it here
|
| 60 |
-
* would re-introduce the browser-clock drift the offset exists to remove. */
|
| 61 |
-
at: string;
|
| 62 |
-
read: boolean;
|
| 63 |
-
/**
|
| 64 |
-
* WAVE 23 (contract C6) β WHAT KIND of notification this is.
|
| 65 |
-
*
|
| 66 |
-
* Absent (and every notification written before this wave) means the original one: a record
|
| 67 |
-
* ENTERED a watched view, routed by `topic` + `viewId`. `"automation_review"` means a card
|
| 68 |
-
* arrived at a review stage and routes by `autoId` instead β a different destination reached
|
| 69 |
-
* from the same list.
|
| 70 |
-
*
|
| 71 |
-
* β A STRING, NEVER A UNION, and that is the wave-9 law rather than laziness: the vocabulary
|
| 72 |
-
* is the SERVER's, and a client union over it turns "the server grew a kind" into "the client
|
| 73 |
-
* silently drops the row". Unknown kinds fall through to the view route, which is exactly what
|
| 74 |
-
* they did before this field existed.
|
| 75 |
-
*/
|
| 76 |
-
kind?: string;
|
| 77 |
-
/** `automation_review` only: the automation whose review stage a card reached. Absent on
|
| 78 |
-
* every other kind β and an `automation_review` row that arrives WITHOUT one opens nothing
|
| 79 |
-
* rather than guessing, the same posture `viewId` gets. */
|
| 80 |
-
autoId?: string;
|
| 81 |
-
/** Advisory. A surface that does not scroll to a stage simply selects the automation. */
|
| 82 |
-
stageId?: string;
|
| 83 |
-
/** How many cards arrived in the batch. C6 queues ONE notification per run naming the count,
|
| 84 |
-
* never one per record β so this is the number the row's own text is built from. */
|
| 85 |
-
count?: number;
|
| 86 |
-
/**
|
| 87 |
-
* β WAVE 32 Β· C3 β THE HEADER LINE, so the Inbox can be laid out like mail.
|
| 88 |
-
*
|
| 89 |
-
* `subject` is what the item is ABOUT (the alert's name, the automation's name, the database
|
| 90 |
-
* that was shared); `label` stays what HAPPENED (the record that entered, the run summary).
|
| 91 |
-
* They were one field, which is why the pane could only ever render a sentence with no sender.
|
| 92 |
-
* Absent on a server that predates this wave β {@link subjectOf} falls back rather than
|
| 93 |
-
* rendering a blank header.
|
| 94 |
-
*/
|
| 95 |
-
subject?: string;
|
| 96 |
-
/**
|
| 97 |
-
* ββ WAVE 33 Β· W33-T28 β WHO IT IS FROM. Mail has a sender; this list did not.
|
| 98 |
-
*
|
| 99 |
-
* A `verifier` reading the finished wave-32 Inbox found the row's sender POSITION occupied by
|
| 100 |
-
* `kindLabel(n.kind)` β the literals "Alert" / "Automation" / "Shared with you", a CATEGORY
|
| 101 |
-
* standing where a who belongs β and no sender anywhere on the wire. Derived by the server
|
| 102 |
-
* (`routes_alerts.notification_view`), never here: only it knows that a share has a person
|
| 103 |
-
* behind it and an alert has a machine.
|
| 104 |
-
* β Optional, so a payload from a server that predates this wave keeps today's shape;
|
| 105 |
-
* {@link senderOf} falls back rather than rendering a blank From column.
|
| 106 |
-
*/
|
| 107 |
-
sender?: string;
|
| 108 |
-
/** β WAVE 32 Β· C3 β where clicking it goes. ABSENT when this product cannot resolve a
|
| 109 |
-
* destination (an alert on a table this account can no longer route to), and that absence is
|
| 110 |
-
* load-bearing: the Inbox renders such a row as plainly unclickable rather than as a click
|
| 111 |
-
* that silently does nothing. */
|
| 112 |
-
target?: NotificationTarget;
|
| 113 |
-
}
|
| 114 |
-
|
| 115 |
-
/** One alert, as `GET /api/v1/alerts` sends it. */
|
| 116 |
-
export interface Alert {
|
| 117 |
-
id: string;
|
| 118 |
-
viewId: string;
|
| 119 |
-
topic: string;
|
| 120 |
-
owner: string;
|
| 121 |
-
label: string;
|
| 122 |
-
createdAt: string;
|
| 123 |
-
/** How many records are in its remembered set right now. */
|
| 124 |
-
matched: number;
|
| 125 |
-
seeded: boolean;
|
| 126 |
-
lastRunAt: string;
|
| 127 |
-
lastError: string;
|
| 128 |
-
}
|
| 129 |
-
|
| 130 |
-
export interface Inbox {
|
| 131 |
-
unread: number;
|
| 132 |
-
items: Notification[];
|
| 133 |
-
/**
|
| 134 |
-
* ββ WAVE 33 Β· W33-T28 / D-208 β THE SERVER'S CLOCK, so a stamp can be mail-shaped.
|
| 135 |
-
*
|
| 136 |
-
* β It exists so {@link stampText} never reads the BROWSER's clock, which is D-208's exit
|
| 137 |
-
* condition word for word. `at` is UTC with its offset (D-18) precisely so every reader sees
|
| 138 |
-
* the same instant; asking "is this today?" of a local clock would put the drift back.
|
| 139 |
-
* β Optional: absent, `stampText` returns the old absolute string rather than guessing.
|
| 140 |
-
*/
|
| 141 |
-
now?: string;
|
| 142 |
-
}
|
| 143 |
-
|
| 144 |
-
export const EMPTY_INBOX: Inbox = { unread: 0, items: [] };
|
| 145 |
-
|
| 146 |
-
/**
|
| 147 |
-
* ββ WAVE 31 Β· T23 (owner item 1) β WHAT THE PANE IS ENTITLED TO SAY, AS A FUNCTION.
|
| 148 |
-
*
|
| 149 |
-
* Owner, verbatim: *"Alerts shows notification, but when clicked it says nothing, and it doesn't
|
| 150 |
-
* remove the notification number."* Both halves are ONE mechanism. `AlertsPane` held its own
|
| 151 |
-
* `inbox` seeded to {@link EMPTY_INBOX} and had no pending state, so between opening the panel
|
| 152 |
-
* and `GET /notifications` answering β measured at **3,280 ms live** β and for ever after a
|
| 153 |
-
* failed fetch:
|
| 154 |
-
* Β· `items` was `[]`, so the pane printed **"Nothing new."** β a claim, not a wait;
|
| 155 |
-
* Β· `unread` was `0`, so **"Mark all read" was `disabled`**, and the badge the frame had
|
| 156 |
-
* already loaded could never be cleared.
|
| 157 |
-
* A confident sentence about somebody's inbox, and the one control that would fix it, both
|
| 158 |
-
* switched off by the same uninitialised state.
|
| 159 |
-
*
|
| 160 |
-
* β IT IS A FUNCTION BECAUSE THE PANE IS TSX AND TSX IS NOT UNDER TEST HERE. `verify_alerts`
|
| 161 |
-
* compiles and RUNS this module under node; markup it cannot reach is markup no control can
|
| 162 |
-
* mutate. Deciding here means the truth table is asserted and each branch is load-bearing.
|
| 163 |
-
*/
|
| 164 |
-
export type PaneView = "pending" | "rows" | "empty" | "error";
|
| 165 |
-
|
| 166 |
-
/**
|
| 167 |
-
* `pending` while the first read is in flight Β· `error` when it failed and we have nothing to
|
| 168 |
-
* show Β· `rows` when there is something Β· `empty` ONLY when a successful read returned nothing.
|
| 169 |
-
*
|
| 170 |
-
* β ROWS WIN OVER AN ERROR, and that is deliberate rather than lax: a refresh that fails while
|
| 171 |
-
* the pane already holds notifications should not blank them β the reader loses real information
|
| 172 |
-
* to a transient. The error still reaches them as the pane's message line.
|
| 173 |
-
*/
|
| 174 |
-
export function paneView(
|
| 175 |
-
phase: "pending" | "ready" | "error",
|
| 176 |
-
rowCount: number,
|
| 177 |
-
seededUnread = 0
|
| 178 |
-
): PaneView {
|
| 179 |
-
if (rowCount > 0) return "rows";
|
| 180 |
-
if (phase === "pending") return "pending";
|
| 181 |
-
if (phase === "error") return "error";
|
| 182 |
-
// β A SUCCESSFUL READ WITH NO ROWS AND A NON-ZERO COUNT IS NOT "nothing new". The badge says
|
| 183 |
-
// there is something; the page we were given does not contain it. Saying "Nothing new" there
|
| 184 |
-
// is the same false confidence in a different costume.
|
| 185 |
-
return seededUnread > 0 ? "error" : "empty";
|
| 186 |
-
}
|
| 187 |
-
|
| 188 |
-
/**
|
| 189 |
-
* May "Mark all read" be pressed?
|
| 190 |
-
*
|
| 191 |
-
* β NOT `unread === 0`, WHICH IS THE SHIPPED BUG. That test asked the PANE's own state β zero
|
| 192 |
-
* until its fetch lands, zero for ever if the fetch fails β so the control was dead in exactly
|
| 193 |
-
* the situations the owner hit. The question is about the ACCOUNT, so it is asked of the count
|
| 194 |
-
* the frame already holds, and a failed read does not take the verb away: `POST /notifications/
|
| 195 |
-
* read` with `ids: null` clears the account's inbox whether or not we managed to list it.
|
| 196 |
-
*/
|
| 197 |
-
export function canMarkAll(unread: number, busy = false): boolean {
|
| 198 |
-
return !busy && Math.max(0, Math.floor(unread)) > 0;
|
| 199 |
-
}
|
| 200 |
-
|
| 201 |
-
const str = (v: unknown): string => (typeof v === "string" ? v : "");
|
| 202 |
-
const num = (v: unknown): number => (typeof v === "number" && isFinite(v) ? v : 0);
|
| 203 |
-
|
| 204 |
-
/**
|
| 205 |
-
* β WAVE 32 Β· C3 β one wire `target` β a {@link NotificationTarget}, or `null`.
|
| 206 |
-
*
|
| 207 |
-
* β BOTH `module` AND `id` ARE REQUIRED, and dropping either test is the failure this guard is
|
| 208 |
-
* for: a target with a module and no id dispatches an open request naming NOTHING β a click that
|
| 209 |
-
* appears to work and silently does not, which is this repo's most-repeated shape. `null` is
|
| 210 |
-
* rendered as an unclickable row, which a reader can SEE.
|
| 211 |
-
*/
|
| 212 |
-
export function parseTarget(raw: unknown): NotificationTarget | null {
|
| 213 |
-
if (!raw || typeof raw !== "object") return null;
|
| 214 |
-
const t = raw as Record<string, unknown>;
|
| 215 |
-
const module = str(t.module).trim();
|
| 216 |
-
const id = str(t.id).trim();
|
| 217 |
-
if (!module || !id) return null;
|
| 218 |
-
const tab = str(t.tab).trim();
|
| 219 |
-
return { module, id, ...(tab ? { tab } : {}) };
|
| 220 |
-
}
|
| 221 |
-
|
| 222 |
-
/**
|
| 223 |
-
* `GET /notifications` β the inbox, fail-closed.
|
| 224 |
-
*
|
| 225 |
-
* β `unread` COMES FROM THE SERVER, and is not recounted from `items`. The two
|
| 226 |
-
* can legitimately differ β the list is what this page holds, the count is what
|
| 227 |
-
* the account has β and recomputing it here would make the badge a function of
|
| 228 |
-
* whatever the last fetch happened to include.
|
| 229 |
-
*/
|
| 230 |
-
export function parseInbox(body: unknown): Inbox {
|
| 231 |
-
const b = (body && typeof body === "object" ? body : {}) as Record<string, unknown>;
|
| 232 |
-
const raw = Array.isArray(b.items) ? b.items : [];
|
| 233 |
-
const items: Notification[] = [];
|
| 234 |
-
for (const item of raw) {
|
| 235 |
-
if (!item || typeof item !== "object") continue;
|
| 236 |
-
const n = item as Record<string, unknown>;
|
| 237 |
-
const id = str(n.id);
|
| 238 |
-
// An id-less notification cannot be marked read, so it would sit unread for
|
| 239 |
-
// ever and hold the badge up. Dropped, not rendered.
|
| 240 |
-
if (!id) continue;
|
| 241 |
-
items.push({
|
| 242 |
-
id,
|
| 243 |
-
alertId: str(n.alertId),
|
| 244 |
-
viewId: str(n.viewId),
|
| 245 |
-
topic: str(n.topic),
|
| 246 |
-
rowId: String(n.rowId ?? ""),
|
| 247 |
-
label: str(n.label) || String(n.rowId ?? ""),
|
| 248 |
-
alertLabel: str(n.alertLabel),
|
| 249 |
-
at: str(n.at),
|
| 250 |
-
read: n.read === true,
|
| 251 |
-
// WAVE 23 C6 β ADDITIVE and spread-conditional, exactly like `NavPage`'s flags: a payload
|
| 252 |
-
// that predates this wave keeps today's shape rather than gaining four `undefined` keys,
|
| 253 |
-
// and an `automation_review` row missing its `autoId` is left WITHOUT one rather than
|
| 254 |
-
// with an empty string that would render as a real destination.
|
| 255 |
-
...(str(n.kind) ? { kind: str(n.kind) } : {}),
|
| 256 |
-
...(str(n.autoId) ? { autoId: str(n.autoId) } : {}),
|
| 257 |
-
...(str(n.stageId) ? { stageId: str(n.stageId) } : {}),
|
| 258 |
-
...(num(n.count) > 0 ? { count: num(n.count) } : {}),
|
| 259 |
-
// β WAVE 32 Β· C3 β additive and spread-conditional, exactly like the four above.
|
| 260 |
-
...(str(n.subject) ? { subject: str(n.subject) } : {}),
|
| 261 |
-
// β WAVE 33 Β· W33-T28 β additive and spread-conditional, like every flag above it.
|
| 262 |
-
...(str(n.sender) ? { sender: str(n.sender) } : {}),
|
| 263 |
-
...(parseTarget(n.target) ? { target: parseTarget(n.target)! } : {}),
|
| 264 |
-
});
|
| 265 |
-
}
|
| 266 |
-
// β W33-T28 / D-208 β the server's clock rides through, spread-conditionally like every other
|
| 267 |
-
// additive key here, so a server that predates this wave yields the same object it always did.
|
| 268 |
-
return {
|
| 269 |
-
unread: Math.max(0, num(b.unread)),
|
| 270 |
-
items,
|
| 271 |
-
...(str(b.now) ? { now: str(b.now) } : {}),
|
| 272 |
-
};
|
| 273 |
-
}
|
| 274 |
-
|
| 275 |
-
/** `GET /alerts` β the alert list, fail-closed. */
|
| 276 |
-
export function parseAlerts(body: unknown): Alert[] {
|
| 277 |
-
const b = (body && typeof body === "object" ? body : {}) as Record<string, unknown>;
|
| 278 |
-
const raw = Array.isArray(b.alerts) ? b.alerts : [];
|
| 279 |
-
const out: Alert[] = [];
|
| 280 |
-
for (const item of raw) {
|
| 281 |
-
if (!item || typeof item !== "object") continue;
|
| 282 |
-
const a = item as Record<string, unknown>;
|
| 283 |
-
const id = str(a.id);
|
| 284 |
-
if (!id) continue;
|
| 285 |
-
out.push({
|
| 286 |
-
id,
|
| 287 |
-
viewId: str(a.viewId),
|
| 288 |
-
topic: str(a.topic),
|
| 289 |
-
owner: str(a.owner),
|
| 290 |
-
label: str(a.label) || "Untitled alert",
|
| 291 |
-
createdAt: str(a.createdAt),
|
| 292 |
-
matched: num(a.matched),
|
| 293 |
-
seeded: a.seeded === true,
|
| 294 |
-
lastRunAt: str(a.lastRunAt),
|
| 295 |
-
lastError: str(a.lastError),
|
| 296 |
-
});
|
| 297 |
-
}
|
| 298 |
-
return out;
|
| 299 |
-
}
|
| 300 |
-
|
| 301 |
-
/**
|
| 302 |
-
* A topic (the grid's scope key) β the hash route that renders it.
|
| 303 |
-
*
|
| 304 |
-
* The two built-ins are the only pair that differ, and they differ because the
|
| 305 |
-
* REGISTRY names the surface while the GRID names the scope; a user table is its
|
| 306 |
-
* own key in both. `null` for anything else: a notification for a topic this
|
| 307 |
-
* client cannot route to must do nothing, not navigate somewhere plausible.
|
| 308 |
-
*/
|
| 309 |
-
export function routeForTopic(topic: string): string | null {
|
| 310 |
-
const t = str(topic).trim();
|
| 311 |
-
if (t === "customer") return "customer_data";
|
| 312 |
-
if (t === "product") return "product_data";
|
| 313 |
-
if (/^ut_[A-Za-z0-9_]+$/.test(t)) return t;
|
| 314 |
-
return null;
|
| 315 |
-
}
|
| 316 |
-
|
| 317 |
-
|
| 318 |
-
|
| 319 |
-
|
| 320 |
-
|
| 321 |
-
|
| 322 |
-
/**
|
| 323 |
-
* The stamp, made readable WITHOUT touching a clock.
|
| 324 |
-
*
|
| 325 |
-
* β NO `new Date()`, NO `toLocaleString()`, and that is the whole design. The
|
| 326 |
-
* server sends UTC WITH ITS OFFSET (D-18) precisely so every reader sees the
|
| 327 |
-
* same instant; parsing it into a browser Date and formatting it back would
|
| 328 |
-
* re-introduce the drift the offset exists to remove β a tenant a day ahead
|
| 329 |
-
* being told an event happened tomorrow ([[date-window-vocabulary]]). This is
|
| 330 |
-
* STRING SURGERY: keep the date and the minutes, drop the seconds and the `T`.
|
| 331 |
-
* Anything that does not look like an ISO stamp passes through untouched, so a
|
| 332 |
-
* format this function has never seen is shown as sent rather than mangled.
|
| 333 |
-
*/
|
| 334 |
-
const MONTHS = ["Jan", "Feb", "Mar", "Apr", "May", "Jun",
|
| 335 |
-
"Jul", "Aug", "Sep", "Oct", "Nov", "Dec"];
|
| 336 |
-
|
| 337 |
-
export function stampText(at: string, now?: string): string {
|
| 338 |
-
const m = /^(\d{4}-\d{2}-\d{2})T(\d{2}:\d{2})/.exec(str(at));
|
| 339 |
-
if (!m) return str(at);
|
| 340 |
-
const [, day, hhmm] = m;
|
| 341 |
-
// ββ W33-T28 / D-208 β MAIL-SHAPED, AND STILL STRING SURGERY.
|
| 342 |
-
//
|
| 343 |
-
// β `now` COMES FROM THE SERVER (`inbox.now`), NEVER FROM `new Date()`. That is D-208's exit
|
| 344 |
-
// condition word for word β "without reading the browser clock" β and the reason is the same
|
| 345 |
-
// one this function has always carried: `at` is sent as UTC WITH its offset (D-18) so every
|
| 346 |
-
// reader sees the same instant, and deciding "is this today?" against a browser clock would
|
| 347 |
-
// re-introduce exactly the drift the offset removes. Both operands are the server's.
|
| 348 |
-
//
|
| 349 |
-
// β AND WITHOUT `now` IT DEGRADES TO THE OLD FORMAT RATHER THAN GUESSING. A caller that has no
|
| 350 |
-
// server clock gets `2026-08-13 09:41` β the pre-wave string, unambiguous and never wrong β
|
| 351 |
-
// instead of a relative stamp computed from something we do not trust.
|
| 352 |
-
const today = /^(\d{4}-\d{2}-\d{2})/.exec(str(now));
|
| 353 |
-
if (!today) return `${day} ${hhmm}`;
|
| 354 |
-
if (today[1] === day) return hhmm;
|
| 355 |
-
const [y, mo, d] = day.split("-");
|
| 356 |
-
const month = MONTHS[Number(mo) - 1] || mo;
|
| 357 |
-
// The year only when it differs β a mail client does not print "2026" on a message from March.
|
| 358 |
-
return today[1].slice(0, 4) === y
|
| 359 |
-
? `${month} ${Number(d)}`
|
| 360 |
-
: `${month} ${Number(d)}, ${y}`;
|
| 361 |
-
}
|
| 362 |
-
|
| 363 |
-
/**
|
| 364 |
-
* The badge's text. Never the raw number past 99: a nav row is 236px wide and a
|
| 365 |
-
* four-digit badge pushes the label out of it.
|
| 366 |
-
*/
|
| 367 |
-
export function badgeText(unread: number): string {
|
| 368 |
-
const n = Math.max(0, Math.floor(unread));
|
| 369 |
-
if (n <= 0) return "";
|
| 370 |
-
return n > 99 ? "99+" : String(n);
|
| 371 |
-
}
|
| 372 |
-
|
| 373 |
-
/**
|
| 374 |
-
* Apply a read/unread change LOCALLY, mirroring what the server just stored, and
|
| 375 |
-
* return the new inbox with the count corrected.
|
| 376 |
-
*
|
| 377 |
-
* `ids === null` is "all of them" (the API's own convention for mark-all). The
|
| 378 |
-
* count is derived from the ITEMS here β deliberately, and it is the one place
|
| 379 |
-
* that is right to do so: the server's answer is in flight, and the alternative
|
| 380 |
-
* is a badge that keeps its old number until the refetch lands.
|
| 381 |
-
*/
|
| 382 |
-
export function applyRead(inbox: Inbox, ids: string[] | null, read: boolean): Inbox {
|
| 383 |
-
const wanted = ids === null ? null : new Set(ids);
|
| 384 |
-
const items = inbox.items.map((n) =>
|
| 385 |
-
wanted === null || wanted.has(n.id) ? { ...n, read } : n
|
| 386 |
-
);
|
| 387 |
-
const seenUnread = items.filter((n) => !n.read).length;
|
| 388 |
-
// A page can hold fewer notifications than the account has, so a partial read
|
| 389 |
-
// must SUBTRACT from the server's count rather than replace it with this
|
| 390 |
-
// page's tally β except when marking everything, where zero is the answer.
|
| 391 |
-
if (wanted === null) return { unread: read ? 0 : items.length, items };
|
| 392 |
-
const changed = inbox.items.filter(
|
| 393 |
-
(n) => wanted.has(n.id) && n.read !== read
|
| 394 |
-
).length;
|
| 395 |
-
const delta = read ? -changed : changed;
|
| 396 |
-
return { unread: Math.max(seenUnread, inbox.unread + delta), items };
|
| 397 |
-
}
|
| 398 |
-
|
| 399 |
-
/**
|
| 400 |
-
* The shellβrail channel for "make an alert out of this view" β the view rail
|
| 401 |
-
* raises it, the frame (which knows the current route, and therefore the topic)
|
| 402 |
-
* answers. Same reason as C-SHARE's event: the rail is host-neutral and cannot
|
| 403 |
-
* import the shell, and it does not know its own scope key.
|
| 404 |
-
*/
|
| 405 |
-
export const ALERT_CREATE_EVENT = "aios:alert-create";
|
| 406 |
-
|
| 407 |
-
export interface AlertCreateRequest {
|
| 408 |
-
viewId: string;
|
| 409 |
-
label: string;
|
| 410 |
-
}
|
| 411 |
-
|
| 412 |
-
export function parseAlertCreate(detail: unknown): AlertCreateRequest | null {
|
| 413 |
-
if (!detail || typeof detail !== "object") return null;
|
| 414 |
-
const d = detail as Record<string, unknown>;
|
| 415 |
-
const viewId = str(d.viewId).trim();
|
| 416 |
-
if (!viewId) return null;
|
| 417 |
-
return { viewId, label: str(d.label).trim() || viewId };
|
| 418 |
-
}
|
|
|
|
| 1 |
+
// ---------------------------------------------------------------------------
|
| 2 |
+
// alerts/alertsModel.ts β WAVE 20 item 25 (contract C-ALERT): the inbox's PURE
|
| 3 |
+
// half. React-free and fetch-free, so `verify_alerts.py` runs it under node.
|
| 4 |
+
//
|
| 5 |
+
// An alert says "tell me when a record ENTERS this view". The client half is
|
| 6 |
+
// small, and every part of it fails silently when it is wrong:
|
| 7 |
+
//
|
| 8 |
+
// Β· an unread count taken from `items.length` rather than from the server's
|
| 9 |
+
// own `unread` disagrees with the badge the moment a page is capped or a
|
| 10 |
+
// read lands in another tab β and a badge that says 3 when the list shows 9
|
| 11 |
+
// teaches the reader to ignore the badge;
|
| 12 |
+
// Β· a notification whose `viewId` no longer resolves must open NOTHING rather
|
| 13 |
+
// than a wrong view β alerts outlive the views they were made from;
|
| 14 |
+
// Β· `topic` is the surface's scope key ("customer", "product", "ut_β¦"), and
|
| 15 |
+
// the ROUTE is a registry key ("customer_data") β mapping one to the other
|
| 16 |
+
// by guesswork sends every click to a page that does not exist.
|
| 17 |
+
// ---------------------------------------------------------------------------
|
| 18 |
+
|
| 19 |
+
/**
|
| 20 |
+
* ββ WAVE 32 Β· T20 Β· CONTRACT C3 β WHERE AN INBOX ITEM OPENS.
|
| 21 |
+
*
|
| 22 |
+
* `module` is the destination surface (`"database"` or `"automation"`) and `id` is what to open
|
| 23 |
+
* in it. `tab` is the SUB-SELECTION inside that module: the literal `"runs"` for an automation's
|
| 24 |
+
* run log, or the VIEW ID to select on a database. One optional key, two destinations.
|
| 25 |
+
*
|
| 26 |
+
* β EVERY FIELD IS A PLAIN `string`, NEVER A UNION, and that is alertsModel's wave-9 law rather
|
| 27 |
+
* than laziness: the vocabulary is the SERVER's, and a client union over it turns "the server grew
|
| 28 |
+
* a module" into "the client silently drops the row". An unknown module is refused by the
|
| 29 |
+
* FRAME's dispatcher, out loud, which is a different thing from never arriving.
|
| 30 |
+
*/
|
| 31 |
+
export interface NotificationTarget {
|
| 32 |
+
module: string;
|
| 33 |
+
id: string;
|
| 34 |
+
tab?: string;
|
| 35 |
+
}
|
| 36 |
+
|
| 37 |
+
/** C3's `kind` vocabulary, mirroring `routes_alerts.NOTIF_KIND_*`. Held as constants so the
|
| 38 |
+
* Inbox's branch and the gate's fixtures cannot disagree about the word. */
|
| 39 |
+
export const NOTIF_KIND_ALERT = "alert";
|
| 40 |
+
export const NOTIF_KIND_AUTOMATION = "automation";
|
| 41 |
+
export const NOTIF_KIND_SHARE = "share";
|
| 42 |
+
|
| 43 |
+
/** C3's `target.module` vocabulary, and the automation sub-selection. */
|
| 44 |
+
export const TARGET_MODULE_DATABASE = "database";
|
| 45 |
+
export const TARGET_MODULE_AUTOMATION = "automation";
|
| 46 |
+
export const TARGET_TAB_RUNS = "runs";
|
| 47 |
+
|
| 48 |
+
/** One notification, as `GET /api/v1/notifications` sends it. */
|
| 49 |
+
export interface Notification {
|
| 50 |
+
id: string;
|
| 51 |
+
alertId: string;
|
| 52 |
+
viewId: string;
|
| 53 |
+
topic: string;
|
| 54 |
+
rowId: string;
|
| 55 |
+
/** What entered β the record's own label. */
|
| 56 |
+
label: string;
|
| 57 |
+
/** What the alert is called, so a row reads without opening anything. */
|
| 58 |
+
alertLabel: string;
|
| 59 |
+
/** UTC WITH OFFSET (D-18). Kept as the server's STRING: re-formatting it here
|
| 60 |
+
* would re-introduce the browser-clock drift the offset exists to remove. */
|
| 61 |
+
at: string;
|
| 62 |
+
read: boolean;
|
| 63 |
+
/**
|
| 64 |
+
* WAVE 23 (contract C6) β WHAT KIND of notification this is.
|
| 65 |
+
*
|
| 66 |
+
* Absent (and every notification written before this wave) means the original one: a record
|
| 67 |
+
* ENTERED a watched view, routed by `topic` + `viewId`. `"automation_review"` means a card
|
| 68 |
+
* arrived at a review stage and routes by `autoId` instead β a different destination reached
|
| 69 |
+
* from the same list.
|
| 70 |
+
*
|
| 71 |
+
* β A STRING, NEVER A UNION, and that is the wave-9 law rather than laziness: the vocabulary
|
| 72 |
+
* is the SERVER's, and a client union over it turns "the server grew a kind" into "the client
|
| 73 |
+
* silently drops the row". Unknown kinds fall through to the view route, which is exactly what
|
| 74 |
+
* they did before this field existed.
|
| 75 |
+
*/
|
| 76 |
+
kind?: string;
|
| 77 |
+
/** `automation_review` only: the automation whose review stage a card reached. Absent on
|
| 78 |
+
* every other kind β and an `automation_review` row that arrives WITHOUT one opens nothing
|
| 79 |
+
* rather than guessing, the same posture `viewId` gets. */
|
| 80 |
+
autoId?: string;
|
| 81 |
+
/** Advisory. A surface that does not scroll to a stage simply selects the automation. */
|
| 82 |
+
stageId?: string;
|
| 83 |
+
/** How many cards arrived in the batch. C6 queues ONE notification per run naming the count,
|
| 84 |
+
* never one per record β so this is the number the row's own text is built from. */
|
| 85 |
+
count?: number;
|
| 86 |
+
/**
|
| 87 |
+
* β WAVE 32 Β· C3 β THE HEADER LINE, so the Inbox can be laid out like mail.
|
| 88 |
+
*
|
| 89 |
+
* `subject` is what the item is ABOUT (the alert's name, the automation's name, the database
|
| 90 |
+
* that was shared); `label` stays what HAPPENED (the record that entered, the run summary).
|
| 91 |
+
* They were one field, which is why the pane could only ever render a sentence with no sender.
|
| 92 |
+
* Absent on a server that predates this wave β {@link subjectOf} falls back rather than
|
| 93 |
+
* rendering a blank header.
|
| 94 |
+
*/
|
| 95 |
+
subject?: string;
|
| 96 |
+
/**
|
| 97 |
+
* ββ WAVE 33 Β· W33-T28 β WHO IT IS FROM. Mail has a sender; this list did not.
|
| 98 |
+
*
|
| 99 |
+
* A `verifier` reading the finished wave-32 Inbox found the row's sender POSITION occupied by
|
| 100 |
+
* `kindLabel(n.kind)` β the literals "Alert" / "Automation" / "Shared with you", a CATEGORY
|
| 101 |
+
* standing where a who belongs β and no sender anywhere on the wire. Derived by the server
|
| 102 |
+
* (`routes_alerts.notification_view`), never here: only it knows that a share has a person
|
| 103 |
+
* behind it and an alert has a machine.
|
| 104 |
+
* β Optional, so a payload from a server that predates this wave keeps today's shape;
|
| 105 |
+
* {@link senderOf} falls back rather than rendering a blank From column.
|
| 106 |
+
*/
|
| 107 |
+
sender?: string;
|
| 108 |
+
/** β WAVE 32 Β· C3 β where clicking it goes. ABSENT when this product cannot resolve a
|
| 109 |
+
* destination (an alert on a table this account can no longer route to), and that absence is
|
| 110 |
+
* load-bearing: the Inbox renders such a row as plainly unclickable rather than as a click
|
| 111 |
+
* that silently does nothing. */
|
| 112 |
+
target?: NotificationTarget;
|
| 113 |
+
}
|
| 114 |
+
|
| 115 |
+
/** One alert, as `GET /api/v1/alerts` sends it. */
|
| 116 |
+
export interface Alert {
|
| 117 |
+
id: string;
|
| 118 |
+
viewId: string;
|
| 119 |
+
topic: string;
|
| 120 |
+
owner: string;
|
| 121 |
+
label: string;
|
| 122 |
+
createdAt: string;
|
| 123 |
+
/** How many records are in its remembered set right now. */
|
| 124 |
+
matched: number;
|
| 125 |
+
seeded: boolean;
|
| 126 |
+
lastRunAt: string;
|
| 127 |
+
lastError: string;
|
| 128 |
+
}
|
| 129 |
+
|
| 130 |
+
export interface Inbox {
|
| 131 |
+
unread: number;
|
| 132 |
+
items: Notification[];
|
| 133 |
+
/**
|
| 134 |
+
* ββ WAVE 33 Β· W33-T28 / D-208 β THE SERVER'S CLOCK, so a stamp can be mail-shaped.
|
| 135 |
+
*
|
| 136 |
+
* β It exists so {@link stampText} never reads the BROWSER's clock, which is D-208's exit
|
| 137 |
+
* condition word for word. `at` is UTC with its offset (D-18) precisely so every reader sees
|
| 138 |
+
* the same instant; asking "is this today?" of a local clock would put the drift back.
|
| 139 |
+
* β Optional: absent, `stampText` returns the old absolute string rather than guessing.
|
| 140 |
+
*/
|
| 141 |
+
now?: string;
|
| 142 |
+
}
|
| 143 |
+
|
| 144 |
+
export const EMPTY_INBOX: Inbox = { unread: 0, items: [] };
|
| 145 |
+
|
| 146 |
+
/**
|
| 147 |
+
* ββ WAVE 31 Β· T23 (owner item 1) β WHAT THE PANE IS ENTITLED TO SAY, AS A FUNCTION.
|
| 148 |
+
*
|
| 149 |
+
* Owner, verbatim: *"Alerts shows notification, but when clicked it says nothing, and it doesn't
|
| 150 |
+
* remove the notification number."* Both halves are ONE mechanism. `AlertsPane` held its own
|
| 151 |
+
* `inbox` seeded to {@link EMPTY_INBOX} and had no pending state, so between opening the panel
|
| 152 |
+
* and `GET /notifications` answering β measured at **3,280 ms live** β and for ever after a
|
| 153 |
+
* failed fetch:
|
| 154 |
+
* Β· `items` was `[]`, so the pane printed **"Nothing new."** β a claim, not a wait;
|
| 155 |
+
* Β· `unread` was `0`, so **"Mark all read" was `disabled`**, and the badge the frame had
|
| 156 |
+
* already loaded could never be cleared.
|
| 157 |
+
* A confident sentence about somebody's inbox, and the one control that would fix it, both
|
| 158 |
+
* switched off by the same uninitialised state.
|
| 159 |
+
*
|
| 160 |
+
* β IT IS A FUNCTION BECAUSE THE PANE IS TSX AND TSX IS NOT UNDER TEST HERE. `verify_alerts`
|
| 161 |
+
* compiles and RUNS this module under node; markup it cannot reach is markup no control can
|
| 162 |
+
* mutate. Deciding here means the truth table is asserted and each branch is load-bearing.
|
| 163 |
+
*/
|
| 164 |
+
export type PaneView = "pending" | "rows" | "empty" | "error";
|
| 165 |
+
|
| 166 |
+
/**
|
| 167 |
+
* `pending` while the first read is in flight Β· `error` when it failed and we have nothing to
|
| 168 |
+
* show Β· `rows` when there is something Β· `empty` ONLY when a successful read returned nothing.
|
| 169 |
+
*
|
| 170 |
+
* β ROWS WIN OVER AN ERROR, and that is deliberate rather than lax: a refresh that fails while
|
| 171 |
+
* the pane already holds notifications should not blank them β the reader loses real information
|
| 172 |
+
* to a transient. The error still reaches them as the pane's message line.
|
| 173 |
+
*/
|
| 174 |
+
export function paneView(
|
| 175 |
+
phase: "pending" | "ready" | "error",
|
| 176 |
+
rowCount: number,
|
| 177 |
+
seededUnread = 0
|
| 178 |
+
): PaneView {
|
| 179 |
+
if (rowCount > 0) return "rows";
|
| 180 |
+
if (phase === "pending") return "pending";
|
| 181 |
+
if (phase === "error") return "error";
|
| 182 |
+
// β A SUCCESSFUL READ WITH NO ROWS AND A NON-ZERO COUNT IS NOT "nothing new". The badge says
|
| 183 |
+
// there is something; the page we were given does not contain it. Saying "Nothing new" there
|
| 184 |
+
// is the same false confidence in a different costume.
|
| 185 |
+
return seededUnread > 0 ? "error" : "empty";
|
| 186 |
+
}
|
| 187 |
+
|
| 188 |
+
/**
|
| 189 |
+
* May "Mark all read" be pressed?
|
| 190 |
+
*
|
| 191 |
+
* β NOT `unread === 0`, WHICH IS THE SHIPPED BUG. That test asked the PANE's own state β zero
|
| 192 |
+
* until its fetch lands, zero for ever if the fetch fails β so the control was dead in exactly
|
| 193 |
+
* the situations the owner hit. The question is about the ACCOUNT, so it is asked of the count
|
| 194 |
+
* the frame already holds, and a failed read does not take the verb away: `POST /notifications/
|
| 195 |
+
* read` with `ids: null` clears the account's inbox whether or not we managed to list it.
|
| 196 |
+
*/
|
| 197 |
+
export function canMarkAll(unread: number, busy = false): boolean {
|
| 198 |
+
return !busy && Math.max(0, Math.floor(unread)) > 0;
|
| 199 |
+
}
|
| 200 |
+
|
| 201 |
+
const str = (v: unknown): string => (typeof v === "string" ? v : "");
|
| 202 |
+
const num = (v: unknown): number => (typeof v === "number" && isFinite(v) ? v : 0);
|
| 203 |
+
|
| 204 |
+
/**
|
| 205 |
+
* β WAVE 32 Β· C3 β one wire `target` β a {@link NotificationTarget}, or `null`.
|
| 206 |
+
*
|
| 207 |
+
* β BOTH `module` AND `id` ARE REQUIRED, and dropping either test is the failure this guard is
|
| 208 |
+
* for: a target with a module and no id dispatches an open request naming NOTHING β a click that
|
| 209 |
+
* appears to work and silently does not, which is this repo's most-repeated shape. `null` is
|
| 210 |
+
* rendered as an unclickable row, which a reader can SEE.
|
| 211 |
+
*/
|
| 212 |
+
export function parseTarget(raw: unknown): NotificationTarget | null {
|
| 213 |
+
if (!raw || typeof raw !== "object") return null;
|
| 214 |
+
const t = raw as Record<string, unknown>;
|
| 215 |
+
const module = str(t.module).trim();
|
| 216 |
+
const id = str(t.id).trim();
|
| 217 |
+
if (!module || !id) return null;
|
| 218 |
+
const tab = str(t.tab).trim();
|
| 219 |
+
return { module, id, ...(tab ? { tab } : {}) };
|
| 220 |
+
}
|
| 221 |
+
|
| 222 |
+
/**
|
| 223 |
+
* `GET /notifications` β the inbox, fail-closed.
|
| 224 |
+
*
|
| 225 |
+
* β `unread` COMES FROM THE SERVER, and is not recounted from `items`. The two
|
| 226 |
+
* can legitimately differ β the list is what this page holds, the count is what
|
| 227 |
+
* the account has β and recomputing it here would make the badge a function of
|
| 228 |
+
* whatever the last fetch happened to include.
|
| 229 |
+
*/
|
| 230 |
+
export function parseInbox(body: unknown): Inbox {
|
| 231 |
+
const b = (body && typeof body === "object" ? body : {}) as Record<string, unknown>;
|
| 232 |
+
const raw = Array.isArray(b.items) ? b.items : [];
|
| 233 |
+
const items: Notification[] = [];
|
| 234 |
+
for (const item of raw) {
|
| 235 |
+
if (!item || typeof item !== "object") continue;
|
| 236 |
+
const n = item as Record<string, unknown>;
|
| 237 |
+
const id = str(n.id);
|
| 238 |
+
// An id-less notification cannot be marked read, so it would sit unread for
|
| 239 |
+
// ever and hold the badge up. Dropped, not rendered.
|
| 240 |
+
if (!id) continue;
|
| 241 |
+
items.push({
|
| 242 |
+
id,
|
| 243 |
+
alertId: str(n.alertId),
|
| 244 |
+
viewId: str(n.viewId),
|
| 245 |
+
topic: str(n.topic),
|
| 246 |
+
rowId: String(n.rowId ?? ""),
|
| 247 |
+
label: str(n.label) || String(n.rowId ?? ""),
|
| 248 |
+
alertLabel: str(n.alertLabel),
|
| 249 |
+
at: str(n.at),
|
| 250 |
+
read: n.read === true,
|
| 251 |
+
// WAVE 23 C6 β ADDITIVE and spread-conditional, exactly like `NavPage`'s flags: a payload
|
| 252 |
+
// that predates this wave keeps today's shape rather than gaining four `undefined` keys,
|
| 253 |
+
// and an `automation_review` row missing its `autoId` is left WITHOUT one rather than
|
| 254 |
+
// with an empty string that would render as a real destination.
|
| 255 |
+
...(str(n.kind) ? { kind: str(n.kind) } : {}),
|
| 256 |
+
...(str(n.autoId) ? { autoId: str(n.autoId) } : {}),
|
| 257 |
+
...(str(n.stageId) ? { stageId: str(n.stageId) } : {}),
|
| 258 |
+
...(num(n.count) > 0 ? { count: num(n.count) } : {}),
|
| 259 |
+
// β WAVE 32 Β· C3 β additive and spread-conditional, exactly like the four above.
|
| 260 |
+
...(str(n.subject) ? { subject: str(n.subject) } : {}),
|
| 261 |
+
// β WAVE 33 Β· W33-T28 β additive and spread-conditional, like every flag above it.
|
| 262 |
+
...(str(n.sender) ? { sender: str(n.sender) } : {}),
|
| 263 |
+
...(parseTarget(n.target) ? { target: parseTarget(n.target)! } : {}),
|
| 264 |
+
});
|
| 265 |
+
}
|
| 266 |
+
// β W33-T28 / D-208 β the server's clock rides through, spread-conditionally like every other
|
| 267 |
+
// additive key here, so a server that predates this wave yields the same object it always did.
|
| 268 |
+
return {
|
| 269 |
+
unread: Math.max(0, num(b.unread)),
|
| 270 |
+
items,
|
| 271 |
+
...(str(b.now) ? { now: str(b.now) } : {}),
|
| 272 |
+
};
|
| 273 |
+
}
|
| 274 |
+
|
| 275 |
+
/** `GET /alerts` β the alert list, fail-closed. */
|
| 276 |
+
export function parseAlerts(body: unknown): Alert[] {
|
| 277 |
+
const b = (body && typeof body === "object" ? body : {}) as Record<string, unknown>;
|
| 278 |
+
const raw = Array.isArray(b.alerts) ? b.alerts : [];
|
| 279 |
+
const out: Alert[] = [];
|
| 280 |
+
for (const item of raw) {
|
| 281 |
+
if (!item || typeof item !== "object") continue;
|
| 282 |
+
const a = item as Record<string, unknown>;
|
| 283 |
+
const id = str(a.id);
|
| 284 |
+
if (!id) continue;
|
| 285 |
+
out.push({
|
| 286 |
+
id,
|
| 287 |
+
viewId: str(a.viewId),
|
| 288 |
+
topic: str(a.topic),
|
| 289 |
+
owner: str(a.owner),
|
| 290 |
+
label: str(a.label) || "Untitled alert",
|
| 291 |
+
createdAt: str(a.createdAt),
|
| 292 |
+
matched: num(a.matched),
|
| 293 |
+
seeded: a.seeded === true,
|
| 294 |
+
lastRunAt: str(a.lastRunAt),
|
| 295 |
+
lastError: str(a.lastError),
|
| 296 |
+
});
|
| 297 |
+
}
|
| 298 |
+
return out;
|
| 299 |
+
}
|
| 300 |
+
|
| 301 |
+
/**
|
| 302 |
+
* A topic (the grid's scope key) β the hash route that renders it.
|
| 303 |
+
*
|
| 304 |
+
* The two built-ins are the only pair that differ, and they differ because the
|
| 305 |
+
* REGISTRY names the surface while the GRID names the scope; a user table is its
|
| 306 |
+
* own key in both. `null` for anything else: a notification for a topic this
|
| 307 |
+
* client cannot route to must do nothing, not navigate somewhere plausible.
|
| 308 |
+
*/
|
| 309 |
+
export function routeForTopic(topic: string): string | null {
|
| 310 |
+
const t = str(topic).trim();
|
| 311 |
+
if (t === "customer") return "customer_data";
|
| 312 |
+
if (t === "product") return "product_data";
|
| 313 |
+
if (/^ut_[A-Za-z0-9_]+$/.test(t)) return t;
|
| 314 |
+
return null;
|
| 315 |
+
}
|
| 316 |
+
|
| 317 |
+
|
| 318 |
+
|
| 319 |
+
|
| 320 |
+
|
| 321 |
+
|
| 322 |
+
/**
|
| 323 |
+
* The stamp, made readable WITHOUT touching a clock.
|
| 324 |
+
*
|
| 325 |
+
* β NO `new Date()`, NO `toLocaleString()`, and that is the whole design. The
|
| 326 |
+
* server sends UTC WITH ITS OFFSET (D-18) precisely so every reader sees the
|
| 327 |
+
* same instant; parsing it into a browser Date and formatting it back would
|
| 328 |
+
* re-introduce the drift the offset exists to remove β a tenant a day ahead
|
| 329 |
+
* being told an event happened tomorrow ([[date-window-vocabulary]]). This is
|
| 330 |
+
* STRING SURGERY: keep the date and the minutes, drop the seconds and the `T`.
|
| 331 |
+
* Anything that does not look like an ISO stamp passes through untouched, so a
|
| 332 |
+
* format this function has never seen is shown as sent rather than mangled.
|
| 333 |
+
*/
|
| 334 |
+
const MONTHS = ["Jan", "Feb", "Mar", "Apr", "May", "Jun",
|
| 335 |
+
"Jul", "Aug", "Sep", "Oct", "Nov", "Dec"];
|
| 336 |
+
|
| 337 |
+
export function stampText(at: string, now?: string): string {
|
| 338 |
+
const m = /^(\d{4}-\d{2}-\d{2})T(\d{2}:\d{2})/.exec(str(at));
|
| 339 |
+
if (!m) return str(at);
|
| 340 |
+
const [, day, hhmm] = m;
|
| 341 |
+
// ββ W33-T28 / D-208 β MAIL-SHAPED, AND STILL STRING SURGERY.
|
| 342 |
+
//
|
| 343 |
+
// β `now` COMES FROM THE SERVER (`inbox.now`), NEVER FROM `new Date()`. That is D-208's exit
|
| 344 |
+
// condition word for word β "without reading the browser clock" β and the reason is the same
|
| 345 |
+
// one this function has always carried: `at` is sent as UTC WITH its offset (D-18) so every
|
| 346 |
+
// reader sees the same instant, and deciding "is this today?" against a browser clock would
|
| 347 |
+
// re-introduce exactly the drift the offset removes. Both operands are the server's.
|
| 348 |
+
//
|
| 349 |
+
// β AND WITHOUT `now` IT DEGRADES TO THE OLD FORMAT RATHER THAN GUESSING. A caller that has no
|
| 350 |
+
// server clock gets `2026-08-13 09:41` β the pre-wave string, unambiguous and never wrong β
|
| 351 |
+
// instead of a relative stamp computed from something we do not trust.
|
| 352 |
+
const today = /^(\d{4}-\d{2}-\d{2})/.exec(str(now));
|
| 353 |
+
if (!today) return `${day} ${hhmm}`;
|
| 354 |
+
if (today[1] === day) return hhmm;
|
| 355 |
+
const [y, mo, d] = day.split("-");
|
| 356 |
+
const month = MONTHS[Number(mo) - 1] || mo;
|
| 357 |
+
// The year only when it differs β a mail client does not print "2026" on a message from March.
|
| 358 |
+
return today[1].slice(0, 4) === y
|
| 359 |
+
? `${month} ${Number(d)}`
|
| 360 |
+
: `${month} ${Number(d)}, ${y}`;
|
| 361 |
+
}
|
| 362 |
+
|
| 363 |
+
/**
|
| 364 |
+
* The badge's text. Never the raw number past 99: a nav row is 236px wide and a
|
| 365 |
+
* four-digit badge pushes the label out of it.
|
| 366 |
+
*/
|
| 367 |
+
export function badgeText(unread: number): string {
|
| 368 |
+
const n = Math.max(0, Math.floor(unread));
|
| 369 |
+
if (n <= 0) return "";
|
| 370 |
+
return n > 99 ? "99+" : String(n);
|
| 371 |
+
}
|
| 372 |
+
|
| 373 |
+
/**
|
| 374 |
+
* Apply a read/unread change LOCALLY, mirroring what the server just stored, and
|
| 375 |
+
* return the new inbox with the count corrected.
|
| 376 |
+
*
|
| 377 |
+
* `ids === null` is "all of them" (the API's own convention for mark-all). The
|
| 378 |
+
* count is derived from the ITEMS here β deliberately, and it is the one place
|
| 379 |
+
* that is right to do so: the server's answer is in flight, and the alternative
|
| 380 |
+
* is a badge that keeps its old number until the refetch lands.
|
| 381 |
+
*/
|
| 382 |
+
export function applyRead(inbox: Inbox, ids: string[] | null, read: boolean): Inbox {
|
| 383 |
+
const wanted = ids === null ? null : new Set(ids);
|
| 384 |
+
const items = inbox.items.map((n) =>
|
| 385 |
+
wanted === null || wanted.has(n.id) ? { ...n, read } : n
|
| 386 |
+
);
|
| 387 |
+
const seenUnread = items.filter((n) => !n.read).length;
|
| 388 |
+
// A page can hold fewer notifications than the account has, so a partial read
|
| 389 |
+
// must SUBTRACT from the server's count rather than replace it with this
|
| 390 |
+
// page's tally β except when marking everything, where zero is the answer.
|
| 391 |
+
if (wanted === null) return { unread: read ? 0 : items.length, items };
|
| 392 |
+
const changed = inbox.items.filter(
|
| 393 |
+
(n) => wanted.has(n.id) && n.read !== read
|
| 394 |
+
).length;
|
| 395 |
+
const delta = read ? -changed : changed;
|
| 396 |
+
return { unread: Math.max(seenUnread, inbox.unread + delta), items };
|
| 397 |
+
}
|
| 398 |
+
|
| 399 |
+
/**
|
| 400 |
+
* The shellβrail channel for "make an alert out of this view" β the view rail
|
| 401 |
+
* raises it, the frame (which knows the current route, and therefore the topic)
|
| 402 |
+
* answers. Same reason as C-SHARE's event: the rail is host-neutral and cannot
|
| 403 |
+
* import the shell, and it does not know its own scope key.
|
| 404 |
+
*/
|
| 405 |
+
export const ALERT_CREATE_EVENT = "aios:alert-create";
|
| 406 |
+
|
| 407 |
+
export interface AlertCreateRequest {
|
| 408 |
+
viewId: string;
|
| 409 |
+
label: string;
|
| 410 |
+
}
|
| 411 |
+
|
| 412 |
+
export function parseAlertCreate(detail: unknown): AlertCreateRequest | null {
|
| 413 |
+
if (!detail || typeof detail !== "object") return null;
|
| 414 |
+
const d = detail as Record<string, unknown>;
|
| 415 |
+
const viewId = str(d.viewId).trim();
|
| 416 |
+
if (!viewId) return null;
|
| 417 |
+
return { viewId, label: str(d.label).trim() || viewId };
|
| 418 |
+
}
|
web/src/assistant/AssistantPage.tsx
CHANGED
|
@@ -105,7 +105,17 @@ const KIND_LABEL: Record<string, string> = {
|
|
| 105 |
* `modelStatus`: a model that IS offered and whose key this deployment does not hold.
|
| 106 |
*/
|
| 107 |
const MODEL_LABEL: Record<string, string> = {
|
| 108 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 109 |
};
|
| 110 |
const modelLabel = (key: string) => MODEL_LABEL[key] ?? key;
|
| 111 |
|
|
|
|
| 105 |
* `modelStatus`: a model that IS offered and whose key this deployment does not hold.
|
| 106 |
*/
|
| 107 |
const MODEL_LABEL: Record<string, string> = {
|
| 108 |
+
// β `anthropic` JOINS THE MAP (W36 Β· ASK E-11, ruling R4). Not a fifth option nobody wanted.
|
| 109 |
+
// E measured all three side rungs refusing live today (cerebras 402, groq 404, openrouter 402),
|
| 110 |
+
// so Anthropic is the only rung that answers at all, and C5 makes it the tool-calling path by
|
| 111 |
+
// ruling rather than by accident. The fall-through below still covers a provider added
|
| 112 |
+
// server-side; this row exists so the ONE rung that works is not the one rendered as a raw key.
|
| 113 |
+
// β NO `word` FOLLOWED BY A COLON IN THIS COMMENT, and that is not fussiness. `verify_query`
|
| 114 |
+
// reads this map with a regex over the block, comments included, so a phrase like "asked for"
|
| 115 |
+
// with a colon after it registered as a SIXTH key named `for` and reddened the check that
|
| 116 |
+
// exists to keep this map equal to the server's list [[prose-that-becomes-its-own-marker]].
|
| 117 |
+
auto: "Auto", anthropic: "Anthropic", cerebras: "Cerebras", groq: "Groq",
|
| 118 |
+
openrouter: "OpenRouter",
|
| 119 |
};
|
| 120 |
const modelLabel = (key: string) => MODEL_LABEL[key] ?? key;
|
| 121 |
|
web/src/automation/AutomationBuilder.tsx
CHANGED
|
@@ -46,13 +46,19 @@ import type {
|
|
| 46 |
FlowVocab,
|
| 47 |
GraphNode,
|
| 48 |
OAuthStatus,
|
|
|
|
| 49 |
TriggerOption,
|
| 50 |
UserTable,
|
| 51 |
} from "./automationApi";
|
| 52 |
// β WAVE 26 Β· ITEM 22 / D-70 β the Run guard, imported rather than re-derived. This file and
|
| 53 |
// `AutomationDetail` each paint a Run button onto the same paid action; ONE function decides.
|
| 54 |
// ITEM 9 / R11 β `createTable` + `AutomationError` for the in-place "+ New database".
|
| 55 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 56 |
import CondBuilder, { condComplete } from "./CondBuilder";
|
| 57 |
import { groupActions, groupBranches, groupByPanel, numberActions, reorderList }
|
| 58 |
from "./steps";
|
|
@@ -141,7 +147,27 @@ interface Props {
|
|
| 141 |
* β REQUIRED. An optional flag defaulting to `true` would make an unmounted toggle look like
|
| 142 |
* a working panel, which is the shape this repo keeps paying for.
|
| 143 |
*/
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 144 |
showProperties: boolean;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 145 |
/** The Properties/Run-history switch, OWNED by `AutomationDetail` and drawn in both panel
|
| 146 |
* heads so it is in the same place whichever one is showing. */
|
| 147 |
panelTabs: ReactNode;
|
|
@@ -420,7 +446,9 @@ export default function AutomationBuilder({
|
|
| 420 |
onRunNow,
|
| 421 |
renderNodeBody,
|
| 422 |
scheduleFace,
|
|
|
|
| 423 |
showProperties,
|
|
|
|
| 424 |
panelTabs,
|
| 425 |
onStepPicked,
|
| 426 |
schedules,
|
|
@@ -1091,6 +1119,30 @@ export default function AutomationBuilder({
|
|
| 1091 |
vocabulary β it would have gone on saying the old words after the catalog
|
| 1092 |
changed, which is the one failure this file's header names twice. */}
|
| 1093 |
{row?.label || a.kind}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1094 |
</span>
|
| 1095 |
{isGroup ? (
|
| 1096 |
<span className="autox-card-sub">
|
|
@@ -1224,7 +1276,10 @@ export default function AutomationBuilder({
|
|
| 1224 |
|
| 1225 |
return (
|
| 1226 |
<>
|
| 1227 |
-
|
|
|
|
|
|
|
|
|
|
| 1228 |
{/* ββ TRIGGER βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ */}
|
| 1229 |
<div className="autox-step">
|
| 1230 |
<div className="autox-side">
|
|
@@ -1504,6 +1559,11 @@ export default function AutomationBuilder({
|
|
| 1504 |
) : (
|
| 1505 |
<ActionProps
|
| 1506 |
action={findAction(actions, sel.id)}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1507 |
/* β THE SAME RULE `actionCard` DERIVES AND THE SERVER ENFORCES (`ig_action_pinned`),
|
| 1508 |
passed down rather than re-derived a third time. The panel needs it because the
|
| 1509 |
pinned step is the one action in the product whose value map the ENGINE NEVER
|
|
@@ -1938,7 +1998,8 @@ function TriggerProps({
|
|
| 1938 |
<p className="auto-note">This server did not offer a view list for that database.</p>
|
| 1939 |
) : !table.views.length ? (
|
| 1940 |
<p className="auto-note">
|
| 1941 |
-
That database has no saved views yet
|
|
|
|
| 1942 |
</p>
|
| 1943 |
) : (
|
| 1944 |
<select
|
|
@@ -2379,8 +2440,8 @@ export function WebActionConfig({
|
|
| 2379 |
<div className="auto-field" key="field">
|
| 2380 |
<label>{mark("field")} Column to write into</label>
|
| 2381 |
<p className="auto-note">
|
| 2382 |
-
This flow has no record to write to
|
| 2383 |
-
first.
|
| 2384 |
</p>
|
| 2385 |
</div>
|
| 2386 |
);
|
|
@@ -2462,6 +2523,379 @@ export function WebActionConfig({
|
|
| 2462 |
);
|
| 2463 |
}
|
| 2464 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2465 |
/**
|
| 2466 |
* β WAVE 28 β EXPORTED so the include-row render suite can mount the real panel.
|
| 2467 |
*
|
|
@@ -2474,6 +2908,7 @@ export function WebActionConfig({
|
|
| 2474 |
export function ActionProps({
|
| 2475 |
action,
|
| 2476 |
pinned,
|
|
|
|
| 2477 |
catalog,
|
| 2478 |
tables,
|
| 2479 |
onTablesChanged,
|
|
@@ -2501,6 +2936,13 @@ export function ActionProps({
|
|
| 2501 |
* refuses in five other places.
|
| 2502 |
*/
|
| 2503 |
pinned: boolean;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2504 |
catalog: ActionCatalogRow[];
|
| 2505 |
tables: UserTable[];
|
| 2506 |
/** ITEM 9 / R11 β re-read the list after this panel's picker creates one. REQUIRED; see Props. */
|
|
@@ -2556,6 +2998,17 @@ export function ActionProps({
|
|
| 2556 |
* picker that reads as "this database has no views".
|
| 2557 |
*/
|
| 2558 |
const walkViews = (tables.find((t) => t.key === walkTable) || null)?.views;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2559 |
|
| 2560 |
return (
|
| 2561 |
<>
|
|
@@ -2564,7 +3017,60 @@ export function ActionProps({
|
|
| 2564 |
|
| 2565 |
<h3>Configuration</h3>
|
| 2566 |
|
| 2567 |
-
{
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2568 |
<>
|
| 2569 |
{/*
|
| 2570 |
β ONE CONDITION EDITOR PER BRANCH (C-FORK). The group's own `config.cond` is GONE β
|
|
@@ -2606,8 +3112,8 @@ export function ActionProps({
|
|
| 2606 |
</div>
|
| 2607 |
))}
|
| 2608 |
<p className="auto-hint">
|
| 2609 |
-
The first branch whose conditions match is the one that runs
|
| 2610 |
-
down two of them.
|
| 2611 |
</p>
|
| 2612 |
</>
|
| 2613 |
) : null}
|
|
@@ -2665,9 +3171,9 @@ export function ActionProps({
|
|
| 2665 |
the same lie the value map was. */}
|
| 2666 |
{pinned ? (
|
| 2667 |
<p className="auto-hint">
|
| 2668 |
-
The search writes one row per profile it finds
|
| 2669 |
-
following, average engagement, bio, link in
|
| 2670 |
-
|
| 2671 |
adding another. There is nothing to map: the columns come from the search.
|
| 2672 |
</p>
|
| 2673 |
) : null}
|
|
@@ -3254,7 +3760,7 @@ export function ActionProps({
|
|
| 3254 |
start, what to do in words, and where to put the answer. Deliberately NOT routed through
|
| 3255 |
`WebActionConfig`: that component walks `WEB_SEEDS`, and sharing it would mean adding
|
| 3256 |
`ai_agent` to a table the browser job's runner does not implement. */}
|
| 3257 |
-
{action.kind === "ai_agent" ? (
|
| 3258 |
<>
|
| 3259 |
<div className="auto-field">
|
| 3260 |
<label htmlFor="autox-ai-url">
|
|
@@ -3296,7 +3802,8 @@ export function ActionProps({
|
|
| 3296 |
<div className="auto-field">
|
| 3297 |
<label>Column to write the answer into</label>
|
| 3298 |
<p className="auto-note">
|
| 3299 |
-
This flow has no record to write to
|
|
|
|
| 3300 |
</p>
|
| 3301 |
</div>
|
| 3302 |
) : (
|
|
@@ -3359,7 +3866,7 @@ export function ActionProps({
|
|
| 3359 |
</>
|
| 3360 |
) : null}
|
| 3361 |
|
| 3362 |
-
{isWeb(action.kind) ? (
|
| 3363 |
<WebActionConfig
|
| 3364 |
kind={action.kind}
|
| 3365 |
cfg={cfg as Record<string, unknown>}
|
|
@@ -3371,6 +3878,13 @@ export function ActionProps({
|
|
| 3371 |
/>
|
| 3372 |
) : null}
|
| 3373 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3374 |
{/*
|
| 3375 |
β WAVE 26 Β· ITEM 5 β OWNER RULINGS R9 AND R10, and they are two different fixes to one
|
| 3376 |
symptom (*"the condition on action act_1 names no field"*).
|
|
@@ -3396,11 +3910,38 @@ export function ActionProps({
|
|
| 3396 |
walking a database that genuinely has no columns yet keeps its (empty) picker, because
|
| 3397 |
for that flow the picker is the right control and adding a column is the fix.
|
| 3398 |
*/}
|
| 3399 |
-
{
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3400 |
<>
|
| 3401 |
<h3>Run this only when</h3>
|
| 3402 |
<p className="auto-note">
|
| 3403 |
-
This flow has no record to test
|
| 3404 |
</p>
|
| 3405 |
</>
|
| 3406 |
) : (
|
|
|
|
| 46 |
FlowVocab,
|
| 47 |
GraphNode,
|
| 48 |
OAuthStatus,
|
| 49 |
+
StatementWorld,
|
| 50 |
TriggerOption,
|
| 51 |
UserTable,
|
| 52 |
} from "./automationApi";
|
| 53 |
// β WAVE 26 Β· ITEM 22 / D-70 β the Run guard, imported rather than re-derived. This file and
|
| 54 |
// `AutomationDetail` each paint a Run button onto the same paid action; ONE function decides.
|
| 55 |
// ITEM 9 / R11 β `createTable` + `AutomationError` for the in-place "+ New database".
|
| 56 |
+
// β W36-T44 β the statements panel's three doors. They go to `routes_statements`, not
|
| 57 |
+
// `/automations`, for the reason `statementBatch`'s own note gives: that is where the send
|
| 58 |
+
// guardrails already are and R10's whole point is that they do not move.
|
| 59 |
+
import {
|
| 60 |
+
AutomationError, createTable, previewStatement, runBlock, statementWorld, testSendStatement,
|
| 61 |
+
} from "./automationApi";
|
| 62 |
import CondBuilder, { condComplete } from "./CondBuilder";
|
| 63 |
import { groupActions, groupBranches, groupByPanel, numberActions, reorderList }
|
| 64 |
from "./steps";
|
|
|
|
| 147 |
* β REQUIRED. An optional flag defaulting to `true` would make an unmounted toggle look like
|
| 148 |
* a working panel, which is the shape this repo keeps paying for.
|
| 149 |
*/
|
| 150 |
+
/**
|
| 151 |
+
* β W36-T69 / R9 β `agentActions` off the wire, keyed by action id. Optional, and absent reads
|
| 152 |
+
* as "nothing is agent-owned" (the wire type carries the reasoning).
|
| 153 |
+
*/
|
| 154 |
+
agentMarks?: Record<string, { agent: string; agentName?: string; created?: string; by?: string }>;
|
| 155 |
showProperties: boolean;
|
| 156 |
+
/**
|
| 157 |
+
* ββ W36-T32 (owner item 3) β HIDE THE FLOW, KEEP THE PANEL. `true` while the Chat view holds
|
| 158 |
+
* the work area.
|
| 159 |
+
*
|
| 160 |
+
* β WHY THE BUILDER STAYS MOUNTED AT ALL, WHICH REVISES A WAVE-34 TRADE RATHER THAN IGNORING
|
| 161 |
+
* IT. `AutomationDetail`'s own note recorded that keeping this component mounted behind
|
| 162 |
+
* `hidden` would "preserve the selection too, at the price of two live editors of one
|
| 163 |
+
* automation", and chose to unmount. The owner has since asked for the opposite in as many
|
| 164 |
+
* words: *"even when the Chat is shown first, we still need to be able to see the
|
| 165 |
+
* Properties/the new module/Run history tab on the right side."* The Properties panel is a
|
| 166 |
+
* child of THIS component, so an unmounted builder means no Properties tab during Chat.
|
| 167 |
+
* β The wave-34 concern is answered rather than overruled: what it feared was two live EDITORS
|
| 168 |
+
* of the flow, and the flow is exactly what this prop hides. The panel is the only live half.
|
| 169 |
+
*/
|
| 170 |
+
flowHidden?: boolean;
|
| 171 |
/** The Properties/Run-history switch, OWNED by `AutomationDetail` and drawn in both panel
|
| 172 |
* heads so it is in the same place whichever one is showing. */
|
| 173 |
panelTabs: ReactNode;
|
|
|
|
| 446 |
onRunNow,
|
| 447 |
renderNodeBody,
|
| 448 |
scheduleFace,
|
| 449 |
+
agentMarks,
|
| 450 |
showProperties,
|
| 451 |
+
flowHidden,
|
| 452 |
panelTabs,
|
| 453 |
onStepPicked,
|
| 454 |
schedules,
|
|
|
|
| 1119 |
vocabulary β it would have gone on saying the old words after the catalog
|
| 1120 |
changed, which is the one failure this file's header names twice. */}
|
| 1121 |
{row?.label || a.kind}
|
| 1122 |
+
{/*
|
| 1123 |
+
ββ W36-T69 / R9 β THE LOCK, ON THE CARD, so the rule is visible BEFORE the panel
|
| 1124 |
+
is opened. The done-when asks for a lock that NAMES its agent; the glyph carries
|
| 1125 |
+
the name in its `title` and its `aria-label` rather than as a third line of card
|
| 1126 |
+
text, because every card in this column has exactly two lines and a third on one
|
| 1127 |
+
of them is the ragged edge the owner reads as unfinished.
|
| 1128 |
+
β `aria-hidden` on the SVG and the name on the wrapper: a screen reader should
|
| 1129 |
+
hear "set up by Odoo sync", never "image".
|
| 1130 |
+
*/}
|
| 1131 |
+
{agentMarks?.[a.id] ? (
|
| 1132 |
+
<span
|
| 1133 |
+
className="autox-card-lock"
|
| 1134 |
+
title={`Set up by ${agentMarks[a.id].agentName || agentMarks[a.id].agent}. Its `
|
| 1135 |
+
+ "configuration is edited by that agent."}
|
| 1136 |
+
aria-label={`Set up by ${agentMarks[a.id].agentName || agentMarks[a.id].agent}`}
|
| 1137 |
+
>
|
| 1138 |
+
<svg width="12" height="12" viewBox="0 0 16 16" fill="none" aria-hidden="true"
|
| 1139 |
+
stroke="currentColor" strokeWidth="1.4" strokeLinecap="round"
|
| 1140 |
+
strokeLinejoin="round">
|
| 1141 |
+
<rect x="3.4" y="7" width="9.2" height="6.4" rx="1.4" />
|
| 1142 |
+
<path d="M5.6 7V5.2a2.4 2.4 0 0 1 4.8 0V7" />
|
| 1143 |
+
</svg>
|
| 1144 |
+
</span>
|
| 1145 |
+
) : null}
|
| 1146 |
</span>
|
| 1147 |
{isGroup ? (
|
| 1148 |
<span className="autox-card-sub">
|
|
|
|
| 1276 |
|
| 1277 |
return (
|
| 1278 |
<>
|
| 1279 |
+
{/* β `hidden`, not unmounted (W36-T32). `.autox-flow` sets no `display`, so the UA's own
|
| 1280 |
+
`[hidden] { display: none }` applies and no `!important` override is needed here β unlike
|
| 1281 |
+
`.auto-panel[hidden]`, whose class rule sets `display: flex` and had to be beaten. */}
|
| 1282 |
+
<div className="autox-flow" hidden={flowHidden}>
|
| 1283 |
{/* ββ TRIGGER βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ */}
|
| 1284 |
<div className="autox-step">
|
| 1285 |
<div className="autox-side">
|
|
|
|
| 1559 |
) : (
|
| 1560 |
<ActionProps
|
| 1561 |
action={findAction(actions, sel.id)}
|
| 1562 |
+
/* β THE MARK FOR THIS ACTION, LOOKED UP ONCE HERE. The panel takes a mark rather than
|
| 1563 |
+
the whole map so it cannot accidentally read another step's ownership, which is the
|
| 1564 |
+
kind of off-by-one that renders a lock on the wrong card and is invisible to a
|
| 1565 |
+
source grep. */
|
| 1566 |
+
agentMark={agentMarks?.[sel.id] || null}
|
| 1567 |
/* β THE SAME RULE `actionCard` DERIVES AND THE SERVER ENFORCES (`ig_action_pinned`),
|
| 1568 |
passed down rather than re-derived a third time. The panel needs it because the
|
| 1569 |
pinned step is the one action in the product whose value map the ENGINE NEVER
|
|
|
|
| 1998 |
<p className="auto-note">This server did not offer a view list for that database.</p>
|
| 1999 |
) : !table.views.length ? (
|
| 2000 |
<p className="auto-note">
|
| 2001 |
+
That database has no saved views yet. Make one on its grid and it
|
| 2002 |
+
appears here.
|
| 2003 |
</p>
|
| 2004 |
) : (
|
| 2005 |
<select
|
|
|
|
| 2440 |
<div className="auto-field" key="field">
|
| 2441 |
<label>{mark("field")} Column to write into</label>
|
| 2442 |
<p className="auto-note">
|
| 2443 |
+
This flow has no record to write to. Give it a database on the
|
| 2444 |
+
trigger first.
|
| 2445 |
</p>
|
| 2446 |
</div>
|
| 2447 |
);
|
|
|
|
| 2523 |
);
|
| 2524 |
}
|
| 2525 |
|
| 2526 |
+
/**
|
| 2527 |
+
* ββ WAVE 36 Β· W36-T44 (D-316 + D-299) β THE STATEMENTS STEP'S CONFIGURATION, WHICH DID NOT EXIST.
|
| 2528 |
+
*
|
| 2529 |
+
* β THE DEFECT WAS NOT A WRONG CONTROL. IT WAS NO CONTROL, ON THE PUBLIC DOMAIN. The server half
|
| 2530 |
+
* has been complete since wave 35: `automation_engine` seeds `{tier, subject, intro, footer}`,
|
| 2531 |
+
* `_clean_action_config` validates all four, and `assemble_statements` reads all four. This file's
|
| 2532 |
+
* `kind ===` chain had no arm for the kind, so a Royal admin opening *Monthly statements* met a
|
| 2533 |
+
* heading, a hint and NOT ONE INPUT. QA measured it exactly: a page-wide `input, textarea` count
|
| 2534 |
+
* of **1**, the agent's own Name field. And the run's parked-batch banner said *"or narrow the
|
| 2535 |
+
* tier filter"*, naming a control that existed nowhere.
|
| 2536 |
+
* β THIS IS `ai_enrich` (D-277) A SECOND TIME. A step kind is a CONTRACT BETWEEN TWO TREES and
|
| 2537 |
+
* each end's gate was green because each end was internally consistent
|
| 2538 |
+
* [[artifact-with-no-importer]]. W36-T39 is the gate that makes the class unrepeatable; this is
|
| 2539 |
+
* the instance [[reachable-is-not-the-same-as-built]].
|
| 2540 |
+
*
|
| 2541 |
+
* β THE FIX IS THE RENDERER, NEVER A SEED. Blank tier means EVERY tier and blank template means
|
| 2542 |
+
* the sender's own default, and both are deliberate: the seed's own comment says the blank "is the
|
| 2543 |
+
* value the config panel shows as all of them rather than an empty box". Writing non-empty values
|
| 2544 |
+
* in would hide the missing panel and freeze today's wording into every stored agent.
|
| 2545 |
+
*
|
| 2546 |
+
* β A COMPONENT AND NOT AN ARM, for a reason the file enforces structurally: `ActionProps` opens
|
| 2547 |
+
* with `if (!action) return`, an early return BEFORE any hook, so nothing inside it may hold
|
| 2548 |
+
* state. This panel has to load the sender's own vocabulary, so it needs its own frame. That is
|
| 2549 |
+
* `WebActionConfig`'s shape too, and exported for `WebActionConfig`'s reason: a panel's defects
|
| 2550 |
+
* (a control bound to no key, a stored value that never reaches its box) are invisible to a grep.
|
| 2551 |
+
*/
|
| 2552 |
+
export function StatementActionConfig({
|
| 2553 |
+
cfg,
|
| 2554 |
+
busy,
|
| 2555 |
+
setCfg,
|
| 2556 |
+
}: {
|
| 2557 |
+
cfg: Record<string, unknown>;
|
| 2558 |
+
busy: boolean;
|
| 2559 |
+
setCfg: (patch: Record<string, unknown>) => void;
|
| 2560 |
+
}) {
|
| 2561 |
+
/**
|
| 2562 |
+
* β THE WORKLIST IS A THIRD STATE AND IS KEPT AS ONE. `null` is "not answered yet", an error
|
| 2563 |
+
* string is "asked and refused", and a value is a value. Collapsing the first two would paint
|
| 2564 |
+
* "this tenant has no tiers" over a request still in flight [[empty-answer-vs-unfinished-answer]].
|
| 2565 |
+
*/
|
| 2566 |
+
const [world, setWorld] = useState<StatementWorld | null>(null);
|
| 2567 |
+
const [worldErr, setWorldErr] = useState("");
|
| 2568 |
+
const [customer, setCustomer] = useState("");
|
| 2569 |
+
const [shot, setShot] = useState<{ html: string; to: string; subject: string } | null>(null);
|
| 2570 |
+
const [testTo, setTestTo] = useState("");
|
| 2571 |
+
const [outcome, setOutcome] = useState("");
|
| 2572 |
+
const [working, setWorking] = useState<"" | "preview" | "test">("");
|
| 2573 |
+
|
| 2574 |
+
/*
|
| 2575 |
+
β ONE READ, ON MOUNT, ABORTED ON UNMOUNT. `GET /admin/statements` reaches Odoo through
|
| 2576 |
+
`load_collection_list` (cached ~30 min server side), so it is the one slow thing on this panel
|
| 2577 |
+
and everything above it paints without waiting. Selecting a different step unmounts this, and
|
| 2578 |
+
an in-flight answer landing in a dead component is how a panel paints the previous step's
|
| 2579 |
+
customers over the current one's.
|
| 2580 |
+
β The header of this file forbids a fetch in the BUILDER's own render (C14 leg 1, the ghost
|
| 2581 |
+
fix). That rule is about the trigger/actions/graph painting synchronously from the list
|
| 2582 |
+
payload, and they still do. This is a leaf panel for one selected step, the same shape
|
| 2583 |
+
`NewDatabase`'s create call already has.
|
| 2584 |
+
*/
|
| 2585 |
+
useEffect(() => {
|
| 2586 |
+
const ac = new AbortController();
|
| 2587 |
+
statementWorld(ac.signal)
|
| 2588 |
+
.then((w) => {
|
| 2589 |
+
if (!ac.signal.aborted) setWorld(w);
|
| 2590 |
+
})
|
| 2591 |
+
.catch((e: unknown) => {
|
| 2592 |
+
if (ac.signal.aborted) return;
|
| 2593 |
+
setWorldErr(e instanceof Error ? e.message : "the collection list could not be read");
|
| 2594 |
+
});
|
| 2595 |
+
return () => ac.abort();
|
| 2596 |
+
}, []);
|
| 2597 |
+
|
| 2598 |
+
const text = (v: unknown) => (v === undefined || v === null ? "" : String(v));
|
| 2599 |
+
const tier = text(cfg.tier);
|
| 2600 |
+
const templates = {
|
| 2601 |
+
subject: text(cfg.subject),
|
| 2602 |
+
intro: text(cfg.intro),
|
| 2603 |
+
footer: text(cfg.footer),
|
| 2604 |
+
};
|
| 2605 |
+
const tiers = world?.tiers || [];
|
| 2606 |
+
const rows = world?.rows || [];
|
| 2607 |
+
|
| 2608 |
+
/* Free text commits on BLUR, which is this file's idiom and its stated reason: `setCfg` merges
|
| 2609 |
+
into the action and re-renders the whole builder, so a controlled box would round-trip the
|
| 2610 |
+
tree on every keystroke of a footer. */
|
| 2611 |
+
const template = (
|
| 2612 |
+
key: "subject" | "intro" | "footer",
|
| 2613 |
+
label: string,
|
| 2614 |
+
rowsCount: number,
|
| 2615 |
+
hint: string
|
| 2616 |
+
) => (
|
| 2617 |
+
<div className="auto-field" key={key}>
|
| 2618 |
+
<label htmlFor={`autox-stmt-${key}`}>{label}</label>
|
| 2619 |
+
{rowsCount > 1 ? (
|
| 2620 |
+
<textarea
|
| 2621 |
+
id={`autox-stmt-${key}`}
|
| 2622 |
+
className="auto-input"
|
| 2623 |
+
rows={rowsCount}
|
| 2624 |
+
defaultValue={templates[key]}
|
| 2625 |
+
/* β THE DEFAULT IS THE PLACEHOLDER, NOT THE VALUE. Blank means "use the sender's own",
|
| 2626 |
+
and the send door resolves it at send time, so painting it as a placeholder shows what
|
| 2627 |
+
blank actually does without storing a copy that stops tracking the sender. With no
|
| 2628 |
+
worklist yet there is nothing honest to show, so it shows nothing. */
|
| 2629 |
+
placeholder={world?.templates?.[key] || ""}
|
| 2630 |
+
disabled={busy}
|
| 2631 |
+
onBlur={(e) => setCfg({ [key]: e.target.value })}
|
| 2632 |
+
/>
|
| 2633 |
+
) : (
|
| 2634 |
+
<input
|
| 2635 |
+
id={`autox-stmt-${key}`}
|
| 2636 |
+
className="auto-input"
|
| 2637 |
+
defaultValue={templates[key]}
|
| 2638 |
+
placeholder={world?.templates?.[key] || ""}
|
| 2639 |
+
disabled={busy}
|
| 2640 |
+
onBlur={(e) => setCfg({ [key]: e.target.value })}
|
| 2641 |
+
/>
|
| 2642 |
+
)}
|
| 2643 |
+
<p className="auto-hint">{hint}</p>
|
| 2644 |
+
</div>
|
| 2645 |
+
);
|
| 2646 |
+
|
| 2647 |
+
const canPreview = !!customer && !busy && working === "";
|
| 2648 |
+
const canTest = canPreview && !!testTo.trim();
|
| 2649 |
+
|
| 2650 |
+
return (
|
| 2651 |
+
<>
|
| 2652 |
+
<div className="auto-field">
|
| 2653 |
+
<label htmlFor="autox-stmt-tier">Customers to include</label>
|
| 2654 |
+
<select
|
| 2655 |
+
id="autox-stmt-tier"
|
| 2656 |
+
className="auto-input"
|
| 2657 |
+
value={tier}
|
| 2658 |
+
disabled={busy}
|
| 2659 |
+
onChange={(e) => setCfg({ tier: e.target.value })}
|
| 2660 |
+
>
|
| 2661 |
+
{/*
|
| 2662 |
+
ββ THE BLANK OPTION IS THE STORED DEFAULT AND IT MUST EXIST. `tier: ""` is what the
|
| 2663 |
+
seed writes and it means EVERY tier. A `<select>` whose `value` matches no `<option>`
|
| 2664 |
+
renders the FIRST one, so without this row a freshly seeded agent would paint
|
| 2665 |
+
"A-Urgent", and the next edit would write that over the blank: blank-means-all would
|
| 2666 |
+
silently become a tier filter, and a batch of 200 would quietly become a batch of 40.
|
| 2667 |
+
Same guard the `find_records` database picker carries, and D-10 is why it carries it.
|
| 2668 |
+
*/}
|
| 2669 |
+
<option value="">Every tier</option>
|
| 2670 |
+
{/*
|
| 2671 |
+
β AND THE STORED VALUE IS ALWAYS AN OPTION, with its wording decided by WHICH THIRD
|
| 2672 |
+
STATE we are in. Before the worklist answers, an unmatched stored tier is simply
|
| 2673 |
+
painted: calling it unknown while the list is still empty would be a lie about a
|
| 2674 |
+
request in flight. After it answers, an unmatched value is genuinely not a tier the
|
| 2675 |
+
sender knows, and saying so is the honest reading.
|
| 2676 |
+
*/}
|
| 2677 |
+
{tier && !tiers.includes(tier) ? (
|
| 2678 |
+
<option value={tier}>{world ? `${tier} (not a tier the sender offers)` : tier}</option>
|
| 2679 |
+
) : null}
|
| 2680 |
+
{tiers.map((t) => (
|
| 2681 |
+
<option key={t} value={t}>
|
| 2682 |
+
{t}
|
| 2683 |
+
</option>
|
| 2684 |
+
))}
|
| 2685 |
+
</select>
|
| 2686 |
+
<p className="auto-hint">
|
| 2687 |
+
{world
|
| 2688 |
+
? "Leave this on Every tier to statement the whole collection list. One batch holds "
|
| 2689 |
+
+ "200 customers."
|
| 2690 |
+
: worldErr
|
| 2691 |
+
? "The collection list could not be read, so the tier names are not available yet. "
|
| 2692 |
+
+ "What is already saved here is kept."
|
| 2693 |
+
: "Reading the collection list to get the tier names."}
|
| 2694 |
+
</p>
|
| 2695 |
+
{worldErr ? <p className="auto-note">{worldErr}</p> : null}
|
| 2696 |
+
</div>
|
| 2697 |
+
|
| 2698 |
+
{template("subject", "Subject", 1,
|
| 2699 |
+
"Leave blank to use the sender's own subject, shown greyed above. {customer}, {company} "
|
| 2700 |
+
+ "and {month} are filled in for each customer.")}
|
| 2701 |
+
{template("intro", "Opening", 4,
|
| 2702 |
+
"The paragraph above the invoice table. Blank uses the sender's own. HTML is allowed, and "
|
| 2703 |
+
+ "the same three placeholders work here.")}
|
| 2704 |
+
{template("footer", "Closing", 4,
|
| 2705 |
+
"The paragraph below the total. Blank uses the sender's own.")}
|
| 2706 |
+
|
| 2707 |
+
{/*
|
| 2708 |
+
ββ D-299 SHIPS IN THIS SAME PANEL, AND THAT IS THE POINT OF PUTTING IT HERE.
|
| 2709 |
+
`W35-T38` deleted `settings/StatementsPane.tsx` and left `routes_statements.preview` and
|
| 2710 |
+
the `overrideTo` branch of `.send` working, gated, and reachable from nowhere. A template
|
| 2711 |
+
editor with no preview and no test send is how somebody mails 200 real debtors to find out
|
| 2712 |
+
what the template looks like, so the editor above and these two controls are ONE piece of
|
| 2713 |
+
work rather than two tickets [[artifact-with-no-importer]].
|
| 2714 |
+
*/}
|
| 2715 |
+
<h3>Check it before it goes</h3>
|
| 2716 |
+
<div className="auto-field">
|
| 2717 |
+
<label htmlFor="autox-stmt-customer">Try it on this customer</label>
|
| 2718 |
+
<select
|
| 2719 |
+
id="autox-stmt-customer"
|
| 2720 |
+
className="auto-input"
|
| 2721 |
+
value={customer}
|
| 2722 |
+
disabled={busy || !world}
|
| 2723 |
+
onChange={(e) => {
|
| 2724 |
+
setCustomer(e.target.value);
|
| 2725 |
+
setShot(null);
|
| 2726 |
+
setOutcome("");
|
| 2727 |
+
}}
|
| 2728 |
+
>
|
| 2729 |
+
<option value="">
|
| 2730 |
+
{world
|
| 2731 |
+
? "Select a customerβ¦"
|
| 2732 |
+
: worldErr
|
| 2733 |
+
? "The collection list could not be read"
|
| 2734 |
+
: "Reading the collection listβ¦"}
|
| 2735 |
+
</option>
|
| 2736 |
+
{rows.map((r) => (
|
| 2737 |
+
<option key={r.Customer} value={r.Customer}>
|
| 2738 |
+
{r.Customer}
|
| 2739 |
+
{r.Tier ? ` (${r.Tier})` : ""}
|
| 2740 |
+
{r.Email ? "" : ", no email on record"}
|
| 2741 |
+
</option>
|
| 2742 |
+
))}
|
| 2743 |
+
</select>
|
| 2744 |
+
<p className="auto-hint">
|
| 2745 |
+
Whoever you pick here is only used for the preview and the test send below. It does not
|
| 2746 |
+
narrow what the agent sends.
|
| 2747 |
+
</p>
|
| 2748 |
+
</div>
|
| 2749 |
+
|
| 2750 |
+
<div className="autox-stmt-row">
|
| 2751 |
+
<button
|
| 2752 |
+
type="button"
|
| 2753 |
+
className="auto-btn"
|
| 2754 |
+
disabled={!canPreview}
|
| 2755 |
+
onClick={async () => {
|
| 2756 |
+
setWorking("preview");
|
| 2757 |
+
setOutcome("");
|
| 2758 |
+
try {
|
| 2759 |
+
setShot(await previewStatement(customer, templates));
|
| 2760 |
+
} catch (e: unknown) {
|
| 2761 |
+
setShot(null);
|
| 2762 |
+
setOutcome(e instanceof Error ? e.message : "the preview could not be rendered");
|
| 2763 |
+
} finally {
|
| 2764 |
+
setWorking("");
|
| 2765 |
+
}
|
| 2766 |
+
}}
|
| 2767 |
+
>
|
| 2768 |
+
{working === "preview" ? "Renderingβ¦" : "Preview"}
|
| 2769 |
+
</button>
|
| 2770 |
+
</div>
|
| 2771 |
+
|
| 2772 |
+
{shot ? (
|
| 2773 |
+
<div className="autox-stmt-shot">
|
| 2774 |
+
<p className="auto-hint">
|
| 2775 |
+
To {shot.to || "no address on this customer's record"}. Subject: {shot.subject}
|
| 2776 |
+
</p>
|
| 2777 |
+
{/*
|
| 2778 |
+
β SANDBOXED, AND `sandbox=""` IS THE WHOLE POINT. This body is rendered by the SAME
|
| 2779 |
+
`render_statement_html` the send door uses, so what a person approves is what goes out.
|
| 2780 |
+
But it interpolates customer names and document numbers straight out of Odoo, and an
|
| 2781 |
+
empty sandbox attribute is the most restrictive value there is: no scripts, no forms,
|
| 2782 |
+
no same-origin. Painting it with `dangerouslySetInnerHTML` instead would run whatever a
|
| 2783 |
+
partner name happens to contain, inside our own origin.
|
| 2784 |
+
*/}
|
| 2785 |
+
<iframe
|
| 2786 |
+
className="autox-stmt-frame"
|
| 2787 |
+
title="Statement preview"
|
| 2788 |
+
sandbox=""
|
| 2789 |
+
srcDoc={shot.html}
|
| 2790 |
+
/>
|
| 2791 |
+
</div>
|
| 2792 |
+
) : null}
|
| 2793 |
+
|
| 2794 |
+
<div className="auto-field">
|
| 2795 |
+
<label htmlFor="autox-stmt-testto">Send one test to this address</label>
|
| 2796 |
+
<input
|
| 2797 |
+
id="autox-stmt-testto"
|
| 2798 |
+
className="auto-input"
|
| 2799 |
+
value={testTo}
|
| 2800 |
+
disabled={busy || !world}
|
| 2801 |
+
onChange={(e) => setTestTo(e.target.value)}
|
| 2802 |
+
/>
|
| 2803 |
+
<p className="auto-hint">
|
| 2804 |
+
{world?.safeMode
|
| 2805 |
+
? "Safe mode is on, so mail can only reach "
|
| 2806 |
+
+ (world.safeRecipients.length
|
| 2807 |
+
? world.safeRecipients.join(", ")
|
| 2808 |
+
: "an allow-list nobody is on yet")
|
| 2809 |
+
+ ". Anything else is refused."
|
| 2810 |
+
: "One statement, to this address only. The customer above is never mailed by this "
|
| 2811 |
+
+ "button."}
|
| 2812 |
+
</p>
|
| 2813 |
+
</div>
|
| 2814 |
+
|
| 2815 |
+
<div className="autox-stmt-row">
|
| 2816 |
+
<button
|
| 2817 |
+
type="button"
|
| 2818 |
+
className="auto-btn"
|
| 2819 |
+
disabled={!canTest}
|
| 2820 |
+
onClick={async () => {
|
| 2821 |
+
setWorking("test");
|
| 2822 |
+
setOutcome("");
|
| 2823 |
+
try {
|
| 2824 |
+
const r = await testSendStatement(customer, testTo, templates);
|
| 2825 |
+
/* β THE OUTCOME IS READ OFF THE RESPONSE, never assumed from a 200. The send door
|
| 2826 |
+
answers `{sent, failed, skipped}` per customer precisely so that "we could not"
|
| 2827 |
+
and "there was nowhere to send" stay distinct, and SAFE_MODE's refusal arrives as
|
| 2828 |
+
a `failed` row with its own sentence rather than as an HTTP error. A caller that
|
| 2829 |
+
printed "sent" on a 200 would report a delivery the guardrail blocked. */
|
| 2830 |
+
const failed = r.failed?.[0];
|
| 2831 |
+
const skipped = r.skipped?.[0];
|
| 2832 |
+
setOutcome(
|
| 2833 |
+
failed
|
| 2834 |
+
? `Not sent. ${failed.error}`
|
| 2835 |
+
: skipped
|
| 2836 |
+
? `Not sent. ${skipped.reason}`
|
| 2837 |
+
: r.sent?.length
|
| 2838 |
+
? `Sent one test to ${r.sent[0].to}.`
|
| 2839 |
+
: "The send door reported nothing sent and gave no reason."
|
| 2840 |
+
);
|
| 2841 |
+
} catch (e: unknown) {
|
| 2842 |
+
setOutcome(e instanceof Error ? e.message : "the test send was refused");
|
| 2843 |
+
} finally {
|
| 2844 |
+
setWorking("");
|
| 2845 |
+
}
|
| 2846 |
+
}}
|
| 2847 |
+
>
|
| 2848 |
+
{working === "test" ? "Sendingβ¦" : "Send one test"}
|
| 2849 |
+
</button>
|
| 2850 |
+
</div>
|
| 2851 |
+
{outcome ? <p className="auto-note">{outcome}</p> : null}
|
| 2852 |
+
</>
|
| 2853 |
+
);
|
| 2854 |
+
}
|
| 2855 |
+
|
| 2856 |
+
/**
|
| 2857 |
+
* ββ WAVE 36 Β· W36-T69 (owner item 8, ruling R9) + E-13 β A CONFIG THAT CAN BE READ AND NOT TYPED.
|
| 2858 |
+
*
|
| 2859 |
+
* β ONE MECHANISM, THREE CALLERS, and that is the point rather than a tidiness. Three different
|
| 2860 |
+
* steps need the same thing β a configuration a person may READ but not EDIT:
|
| 2861 |
+
* Β· an agent-authored Action (R9: *"its configuration can only be touched by the agent"*),
|
| 2862 |
+
* Β· `ai_enrich`, whose settings belong to a COLUMN (D-277's exit condition: "a NAMED first step
|
| 2863 |
+
* with the Field and the Database under Config"),
|
| 2864 |
+
* Β· `odoo_sync`, configured on the connector rather than here.
|
| 2865 |
+
* A second read-only renderer per case is how three surfaces come to disagree about what
|
| 2866 |
+
* "read-only" looks like, and E booked the last two as PENDING precisely because they had no
|
| 2867 |
+
* mechanism yet. They have one now.
|
| 2868 |
+
*
|
| 2869 |
+
* β AND IT IS A DEFINITION LIST, NOT DISABLED INPUTS. A greyed-out `<input>` still reads as a
|
| 2870 |
+
* control somebody could enable, invites a click that does nothing, and is announced as a form
|
| 2871 |
+
* field to a screen reader. What this renders is what it is: the values, as text, labelled.
|
| 2872 |
+
*/
|
| 2873 |
+
function ReadOnlyConfig({ rows, note }: {
|
| 2874 |
+
rows: { label: string; value: string }[];
|
| 2875 |
+
note: string;
|
| 2876 |
+
}) {
|
| 2877 |
+
const shown = rows.filter((r) => r.value);
|
| 2878 |
+
return (
|
| 2879 |
+
<>
|
| 2880 |
+
{shown.length ? (
|
| 2881 |
+
<dl className="autox-ro">
|
| 2882 |
+
{shown.map((r) => (
|
| 2883 |
+
<div key={r.label}>
|
| 2884 |
+
<dt>{r.label}</dt>
|
| 2885 |
+
<dd>{r.value}</dd>
|
| 2886 |
+
</div>
|
| 2887 |
+
))}
|
| 2888 |
+
</dl>
|
| 2889 |
+
) : (
|
| 2890 |
+
/* β NEVER AN EMPTY PANEL. A step whose config has not been filled in yet still has to say
|
| 2891 |
+
what it IS, or the reader meets the captioned-step-over-nothing that D-277 reported. */
|
| 2892 |
+
<p className="auto-note">This step has nothing configured yet.</p>
|
| 2893 |
+
)}
|
| 2894 |
+
<p className="auto-hint">{note}</p>
|
| 2895 |
+
</>
|
| 2896 |
+
);
|
| 2897 |
+
}
|
| 2898 |
+
|
| 2899 |
/**
|
| 2900 |
* β WAVE 28 β EXPORTED so the include-row render suite can mount the real panel.
|
| 2901 |
*
|
|
|
|
| 2908 |
export function ActionProps({
|
| 2909 |
action,
|
| 2910 |
pinned,
|
| 2911 |
+
agentMark,
|
| 2912 |
catalog,
|
| 2913 |
tables,
|
| 2914 |
onTablesChanged,
|
|
|
|
| 2936 |
* refuses in five other places.
|
| 2937 |
*/
|
| 2938 |
pinned: boolean;
|
| 2939 |
+
/**
|
| 2940 |
+
* β W36-T69 / R9 β the `agentActions` entry for THIS action, or `null`.
|
| 2941 |
+
* β OPTIONAL, and absent reads as "not agent-owned" (see the wire type's own note): an older
|
| 2942 |
+
* server sends no map, and a client that defaulted the other way would lock every step in the
|
| 2943 |
+
* product behind a rule nobody had applied.
|
| 2944 |
+
*/
|
| 2945 |
+
agentMark?: { agent: string; agentName?: string; created?: string; by?: string } | null;
|
| 2946 |
catalog: ActionCatalogRow[];
|
| 2947 |
tables: UserTable[];
|
| 2948 |
/** ITEM 9 / R11 β re-read the list after this panel's picker creates one. REQUIRED; see Props. */
|
|
|
|
| 2998 |
* picker that reads as "this database has no views".
|
| 2999 |
*/
|
| 3000 |
const walkViews = (tables.find((t) => t.key === walkTable) || null)?.views;
|
| 3001 |
+
/**
|
| 3002 |
+
* ββ W36-T69 β IS THIS STEP'S CONFIGURATION READ-ONLY? ONE PREDICATE, CONSULTED BY EVERY EDITOR.
|
| 3003 |
+
*
|
| 3004 |
+
* β THE FIRST DRAFT GUARDED ONLY THE `group`/`create_record`/`find_records` TERNARY CHAIN, and
|
| 3005 |
+
* the render suite caught it in one line: `isWeb(...)`, `ai_agent` and the condition editor are
|
| 3006 |
+
* SEPARATE JSX expressions, so an agent-owned `web_read` still painted seven inputs under a
|
| 3007 |
+
* notice saying it could not be edited. A rule spelled once per arm is a rule with a list to
|
| 3008 |
+
* maintain, and the arm added next wave will not be on it.
|
| 3009 |
+
*/
|
| 3010 |
+
const readOnly = !!agentMark
|
| 3011 |
+
|| action.kind === "ai_enrich" || action.kind === "odoo_sync";
|
| 3012 |
|
| 3013 |
return (
|
| 3014 |
<>
|
|
|
|
| 3017 |
|
| 3018 |
<h3>Configuration</h3>
|
| 3019 |
|
| 3020 |
+
{/*
|
| 3021 |
+
ββ W36-T69 β THE AGENT-OWNED BRANCH COMES FIRST AND RETURNS, so no per-kind editor below
|
| 3022 |
+
can paint a control over a config the server will refuse. Putting it after them would leave
|
| 3023 |
+
the wall depending on which arm happened to match, which is a wall with a list to maintain.
|
| 3024 |
+
β R9 IS A WRITE RULE. The user reads everything and may still DELETE the step; only the
|
| 3025 |
+
typing is gone. Today the server 409s a PATCH and no client knows, so a person configures,
|
| 3026 |
+
saves, and meets a refusal they were never warned about.
|
| 3027 |
+
*/}
|
| 3028 |
+
{agentMark ? (
|
| 3029 |
+
<>
|
| 3030 |
+
<p className="auto-note">
|
| 3031 |
+
{agentMark.agentName || agentMark.agent} set this step up, so its configuration is
|
| 3032 |
+
edited by that agent rather than by hand. You can still read it here, and you can
|
| 3033 |
+
delete the step.
|
| 3034 |
+
</p>
|
| 3035 |
+
<ReadOnlyConfig
|
| 3036 |
+
rows={Object.entries(cfg as Record<string, unknown>)
|
| 3037 |
+
.filter(([, v]) => typeof v !== "object" || v === null)
|
| 3038 |
+
.map(([k, v]) => ({ label: k, value: v === null ? "" : String(v) }))}
|
| 3039 |
+
note="Ask the agent to change any of this."
|
| 3040 |
+
/>
|
| 3041 |
+
</>
|
| 3042 |
+
) : action.kind === "ai_enrich" ? (
|
| 3043 |
+
/* β D-277's exit condition, verbatim: "a NAMED first step with the Field and the Database
|
| 3044 |
+
under Config". The NAME arrives from E's catalog row; these are the two lines under it.
|
| 3045 |
+
An enrichment is configured on the COLUMN, which is why it is read-only here rather than
|
| 3046 |
+
missing: `patch_automation` already refuses a `field:` id. */
|
| 3047 |
+
<ReadOnlyConfig
|
| 3048 |
+
rows={[
|
| 3049 |
+
{ label: "Column", value: String((cfg as { field?: string }).field || "") },
|
| 3050 |
+
{ label: "Database",
|
| 3051 |
+
value: String((cfg as { tableLabel?: string; table?: string }).tableLabel
|
| 3052 |
+
|| (cfg as { table?: string }).table || "") },
|
| 3053 |
+
]}
|
| 3054 |
+
/* β A TEMPLATE, NOT A WRAPPED ATTRIBUTE STRING. JSX collapses whitespace in TEXT
|
| 3055 |
+
children but preserves it verbatim inside an attribute, so the wrapped literal that
|
| 3056 |
+
was here rendered a newline and fourteen spaces into the middle of the sentence. It
|
| 3057 |
+
looked right in the source and wrong on the screen, which is the only kind of typo
|
| 3058 |
+
a type-checker cannot help with. */
|
| 3059 |
+
note={"An enrichment belongs to its column. Change its prompt, model or schedule "
|
| 3060 |
+
+ "on the column itself."}
|
| 3061 |
+
/>
|
| 3062 |
+
) : action.kind === "odoo_sync" ? (
|
| 3063 |
+
<ReadOnlyConfig
|
| 3064 |
+
rows={[
|
| 3065 |
+
{ label: "Connector",
|
| 3066 |
+
value: String((cfg as { connector?: string }).connector || "") },
|
| 3067 |
+
{ label: "Runs", value: String((cfg as { every?: string }).every || "") },
|
| 3068 |
+
]}
|
| 3069 |
+
note="This schedule is configured on the connector, under Keychains."
|
| 3070 |
+
/>
|
| 3071 |
+
) : null}
|
| 3072 |
+
|
| 3073 |
+
{readOnly ? null : action.kind === "group" ? (
|
| 3074 |
<>
|
| 3075 |
{/*
|
| 3076 |
β ONE CONDITION EDITOR PER BRANCH (C-FORK). The group's own `config.cond` is GONE β
|
|
|
|
| 3112 |
</div>
|
| 3113 |
))}
|
| 3114 |
<p className="auto-hint">
|
| 3115 |
+
The first branch whose conditions match is the one that runs. The record
|
| 3116 |
+
does not go down two of them.
|
| 3117 |
</p>
|
| 3118 |
</>
|
| 3119 |
) : null}
|
|
|
|
| 3171 |
the same lie the value map was. */}
|
| 3172 |
{pinned ? (
|
| 3173 |
<p className="auto-hint">
|
| 3174 |
+
The search writes one row per profile it finds, into that database: handle,
|
| 3175 |
+
profile link, name, followers, following, average engagement, bio, link in
|
| 3176 |
+
bio, verified and category. It matches on the handle, so a profile found again updates its row instead of
|
| 3177 |
adding another. There is nothing to map: the columns come from the search.
|
| 3178 |
</p>
|
| 3179 |
) : null}
|
|
|
|
| 3760 |
start, what to do in words, and where to put the answer. Deliberately NOT routed through
|
| 3761 |
`WebActionConfig`: that component walks `WEB_SEEDS`, and sharing it would mean adding
|
| 3762 |
`ai_agent` to a table the browser job's runner does not implement. */}
|
| 3763 |
+
{action.kind === "ai_agent" && !readOnly ? (
|
| 3764 |
<>
|
| 3765 |
<div className="auto-field">
|
| 3766 |
<label htmlFor="autox-ai-url">
|
|
|
|
| 3802 |
<div className="auto-field">
|
| 3803 |
<label>Column to write the answer into</label>
|
| 3804 |
<p className="auto-note">
|
| 3805 |
+
This flow has no record to write to. Give it a database on the trigger
|
| 3806 |
+
first.
|
| 3807 |
</p>
|
| 3808 |
</div>
|
| 3809 |
) : (
|
|
|
|
| 3866 |
</>
|
| 3867 |
) : null}
|
| 3868 |
|
| 3869 |
+
{isWeb(action.kind) && !readOnly ? (
|
| 3870 |
<WebActionConfig
|
| 3871 |
kind={action.kind}
|
| 3872 |
cfg={cfg as Record<string, unknown>}
|
|
|
|
| 3878 |
/>
|
| 3879 |
) : null}
|
| 3880 |
|
| 3881 |
+
{/* ββ W36-T44 (D-316) β the arm whose absence was the whole defect. See
|
| 3882 |
+
`StatementActionConfig`: the server has seeded, validated and consumed these four keys
|
| 3883 |
+
since wave 35, and this line is what lets anybody type them. */}
|
| 3884 |
+
{action.kind === "send_statement" && !readOnly ? (
|
| 3885 |
+
<StatementActionConfig cfg={cfg as Record<string, unknown>} busy={busy} setCfg={setCfg} />
|
| 3886 |
+
) : null}
|
| 3887 |
+
|
| 3888 |
{/*
|
| 3889 |
β WAVE 26 Β· ITEM 5 β OWNER RULINGS R9 AND R10, and they are two different fixes to one
|
| 3890 |
symptom (*"the condition on action act_1 names no field"*).
|
|
|
|
| 3910 |
walking a database that genuinely has no columns yet keeps its (empty) picker, because
|
| 3911 |
for that flow the picker is the right control and adding a column is the fix.
|
| 3912 |
*/}
|
| 3913 |
+
{/*
|
| 3914 |
+
ββ W36-T44 β AND A `send_statement` HAS NO CONDITIONS EITHER, for R9's reason arriving at
|
| 3915 |
+
a different kind. R9 removed the editor from `create_record` because that step MAKES the
|
| 3916 |
+
record there is nothing to test yet. This step is not record-scoped at all: a statements
|
| 3917 |
+
flow is BATCH-scoped, it binds no database (its customers come from the Odoo collection
|
| 3918 |
+
list, not a `ut_*` grid), and `assemble_statements` reads the step's `config` and NOTHING
|
| 3919 |
+
else. `is_statement_flow` requires exactly ONE enabled action, so there is no walk to
|
| 3920 |
+
condition and no second step to condition it against.
|
| 3921 |
+
β AND THE STORED KEY SURVIVES, WHICH IS WHY THE EDITOR MUST NOT. `clean_actions` drops
|
| 3922 |
+
`when` for `create_record` alone; on this kind it is kept, stored, reloaded, and read by
|
| 3923 |
+
nobody. That is a control whose value appears to be ignored, which is exactly the state R9
|
| 3924 |
+
called out as indefensible and exactly what the `discover_instagram` values editor was
|
| 3925 |
+
[[a-flag-can-ship-without-its-writer]]. The sentence points at the control that DOES narrow
|
| 3926 |
+
the batch, which is the tier filter above, and which is the control the run's own
|
| 3927 |
+
parked-batch banner has been naming all along.
|
| 3928 |
+
*/}
|
| 3929 |
+
{/* β A CONDITION IS CONFIGURATION TOO. `clean_actions` keeps `when` on an agent-authored
|
| 3930 |
+
step exactly as it keeps its config, so leaving this editor live would be a control the
|
| 3931 |
+
server refuses on the same PATCH β the whole defect one field over. */}
|
| 3932 |
+
{readOnly ? null : action.kind === "send_statement" ? (
|
| 3933 |
+
<>
|
| 3934 |
+
<h3>Run this only when</h3>
|
| 3935 |
+
<p className="auto-note">
|
| 3936 |
+
A statements step runs over the whole collection list at once, not once per record, so
|
| 3937 |
+
there is no record here to test. Narrow it with Customers to include, above.
|
| 3938 |
+
</p>
|
| 3939 |
+
</>
|
| 3940 |
+
) : action.kind === "create_record" ? null : !walkTable ? (
|
| 3941 |
<>
|
| 3942 |
<h3>Run this only when</h3>
|
| 3943 |
<p className="auto-note">
|
| 3944 |
+
This flow has no record to test. Set conditions on the trigger instead.
|
| 3945 |
</p>
|
| 3946 |
</>
|
| 3947 |
) : (
|