diff --git "a/api/automation_engine.py" "b/api/automation_engine.py" --- "a/api/automation_engine.py" +++ "b/api/automation_engine.py" @@ -1,961 +1,961 @@ -"""automation_engine.py — wave-18 item 5 (contract C4-AUTO): the automation RUNTIME. - -THE SHAPE, and why it is this shape. An automation is a DEFINITION that lives in the tenant -store and a RUN that lives in this process. The definition is durable, small and rarely written; -the run is hot, chatty and worthless five minutes later. Conflating them is the failure this file -is built to avoid, and it has two halves: - - * **Run state is NEVER persisted while it is running.** A `state: 'running'` written to the - store outlives the process that wrote it, so a container restart mid-run leaves an automation - that can never run again — the 409 sees a `running` nothing will ever clear. Hot state lives - in `_RUNNING` (a process dict, cleared by definition on restart) and the store is written - exactly ONCE per run, at the end. - * **One coalesced store update per run per bucket.** `core/store.py` coalesces on a 20 s - floor per key against a 256-commits/hour repo budget, so a per-ROW write is the wrong shape - by two orders of magnitude — the S&P demo alone is ~500 rows. Every runner below computes - its whole result first and commits it in a single `update` callback. - -WHAT IS VENDORED HERE AND WHY. `.claude/skills/browse/scrape.py` is not on `sys.path` and is not -deployed, so its extraction core is copied into this file (the wave doc's instruction), ported -httpx → requests (the only HTTP dependency this API already carries). The SSRF rail comes with -it, HARDENED rather than merely preserved — see `guard`/`fetch`. That hardening is the point: -the skill version takes a URL from a developer on a CLI; this takes one from a request body. -""" -from __future__ import annotations - -import copy -import datetime as _dt -import hashlib -import hmac -import ipaddress -import json -import os -import re -import socket -import threading -import time -from urllib.parse import urljoin, urlparse - -import requests - -# ⭐ `providers` IS DELIBERATELY NOT IMPORTED HERE ANY MORE (wave 27 item 23). The capability -# router had exactly one consumer in this file — the views top-up inside the Bright Data rung — -# and that rung now lives in `connectors_ig`. So the vendor-routing seam is reached only by the -# connector that routes, which is the shape the split was for: this file no longer knows that -# there is more than one vendor, or that vendors cost money. Re-adding this import is therefore -# a signal, not a convenience — it means something in the automation runtime started making a -# vendor decision, and that belongs one layer down. - -try: # the parser the extraction half needs - from bs4 import BeautifulSoup -except Exception: # pragma: no cover — deploy lag; see mailbox ② - BeautifulSoup = None - -# --------------------------------------------------------------------------------------------- -# THE BUCKET (C4-AUTO) -# --------------------------------------------------------------------------------------------- - -#: The per-tenant store key. Colocated with its reader, the `routes_nav._NAV_PREFS_KEY` pattern. -STORE_KEY = "automations" -#: The user-table bucket. ⚠ Read/written HERE through `TenantRuntime`, never through -#: `core.user_tables` — that module writes the UNPREFIXED module-global key, which is correct for -#: tenant #0 (empty prefix) and silently cross-tenant for every R2 tenant after it. Booked in the -#: session-D mailbox as a finding against A's file rather than fixed from here. -UT_STORE_KEY = "user_tables" -UT_PREFIX = "ut_" - -MAX_AUTOMATIONS = 40 -MAX_RUNS = 20 # trimmed history per automation -MAX_NAME = 80 -#: ⛔ IT DOES **NOT** MIRROR `core.user_tables.MAX_ROWS`, AND THE COMMENT THAT SAID SO WAS A 12x -#: STALE CLAIM — `MAX_ROWS` is 60,000. That sentence is most of what made D-143 hard to see: it -#: read as "the substrate's bound, kept in step", so nobody asked whether a flow was silently -#: walking 5,000 of 32,826 connected rows. It was. -#: **This is the FALLBACK ONLY.** The live answer is per TABLE and comes from -#: `core.user_tables.row_limit` via `_flow_record_cap` — `None` for a connected source (R6: no -#: cap), `MAX_ROWS` for the editable substrate, `0` for a read-through grid. This constant is -#: reached only when that import fails, and it is deliberately the CONSERVATIVE direction. -#: ⚠ Do not "fix" it by raising it to 60,000: a fallback that runs when the evaluator is missing -#: should not also be the widest one. And do not delete it — D-143's own row says the editable -#: substrate still needs a bound. -MAX_UT_ROWS = 5000 -MAX_UT_TABLES = 40 - -#: ⛔⛔ D-112's SENTENCE, AS A CONSTANT, BECAUSE TWO PLACES DEPEND ON IT AGREEING. -#: `enrich_selection` produces it when a step's `fromView` names a view that no longer resolves; -#: `run_flow` tests for it to turn that run `partial` instead of `ok`. Measured live on a real -#: tenant: an enrich action pointed at a deleted view walked ZERO records and reported **`ok`** — -#: indistinguishable on every surface from a run that worked, and a strong candidate for why the -#: owner's enrichment kept appearing to do nothing. -#: ⚠ THE WORDING IS FREE TO CHANGE; the AGREEMENT is not. Both sites read this name, so a better -#: sentence stays a one-line edit instead of a silent regression [[constant-two-features-share]]. -ENRICH_VIEW_UNREADABLE = "the enrich step names a view it cannot read" - -#: ⚠ THE APPEND TABLES NEED A DIFFERENT CAP, and the reason is arithmetic rather than taste. -#: `MAX_UT_ROWS` was sized for a scraped LIST — a page of ~500 companies that is re-read, so the -#: row count is bounded by the page. The `ut_ig_*` tables are the opposite shape: every pull -#: INSERTS a timestamped row (see `SNAPSHOT_FIELDS` and the compound keys below), so at R1's -#: 1k-profiles-daily the snapshot table crosses 5000 rows on **day five** — and the old behaviour -#: was a SILENT `skipped++`, i.e. the table would quietly stop growing and the run would still -#: report success. A time series that stops after five days without saying so is worse than one -#: that was never built. -#: 200k is the owner-directed ceiling (R1's "~200k"). ⚠ IT BUYS VERY DIFFERENT HORIZONS FOR THE -#: TWO APPEND TABLES, and the difference is worth knowing before anyone plans around it: -#: ut_ig_snapshots 1 row per profile per pull -> ~200 days at 1k profiles/day -#: ut_ig_post_snapshots maxPosts rows per pull -> ~8 days at 1k profiles x 24 posts -#: So the post series is the one that fills, and it fills FAST. That is exactly why the breach had -#: to become LOUD (a distinct `capped` count -> a `partial` run naming the table): at this rate the -#: silent version would have flatlined a chart inside a fortnight with a green dot over it. -#: -#: ⚠ MEASURED COST AT THE CEILING (2026-08-04, this box): a full `ut_ig_post_snapshots` serialises -#: to **35.8 MB** of JSON (~1.4 s). `UT_STORE_KEY` is ONE bucket for ALL of a tenant's user tables, -#: so that cost is paid by every unrelated automation write in the tenant too. Booked for the B-3 -#: Postgres tripwires (R2 clause b) rather than papered over — the fix is a row store, not a -#: smaller number. -#: (W19-C; the GENERIC loud-breach for every other table stays booked as DEBT D-11.) -MAX_UT_IG_ROWS = 200_000 -IG_TABLE_PREFIX = "ut_ig_" - -#: ⚠ THE RAISED CEILING BELONGS TO THE **APPEND** TABLES, NOT TO A NAME PREFIX (wave 20). It was -#: keyed off `ut_ig_`, which was exactly right while every `ut_ig_*` table was an append table — -#: and stopped being right the moment discovery added `ut_ig_candidates`, which is UPSERTED BY -#: HANDLE and is bounded by how many accounts exist rather than by how often we look. It would -#: have inherited a 200,000-row ceiling, and with it the measured 35.8 MB single-bucket -#: serialisation cost, purely by an accident of naming. Naming the append tables is the version -#: that stays true when the next `ut_ig_*` table is not one. -IG_SNAPSHOTS_TABLE = "ut_ig_snapshots" -IG_POSTS_TABLE = "ut_ig_posts" -IG_POST_SNAPSHOTS_TABLE = "ut_ig_post_snapshots" -IG_COMMENTS_TABLE = "ut_ig_comments" - -# --------------------------------------------------------------------------------------------- -# ⭐⭐ WAVE 29 (item 7, DEBT D-9, owner rulings R1 + R2) — TIKTOK, AS A PARALLEL FAMILY -# --------------------------------------------------------------------------------------------- -# R1: FULL PARITY — profile AND posts AND comments, not a profile tier. -# R2: a PARALLEL `ut_tt_*` family. `ut_ig_*` is untouched: zero migration, zero risk to the live -# Instagram rows, and the two schemas may DIVERGE where the vendors do. -# -# ⛔ WHY A SECOND FAMILY RATHER THAN A `platform` COLUMN ON THE FIRST, stated here because it is -# the question every reader will ask. `PRESET_PROFILE_FIELDS` carries `platform` and its identity -# is `(platform, handle)` — so a TikTok PROFILE row could already have lived in `ut_ig_profile`. -# The other four tables carry NO discriminator at all: `ut_ig_posts`/`ut_ig_comments` key on -# `shortcode` and the snapshot tables on `influencer_key`, so an Instagram post and a TikTok video -# that happened to share a code would MERGE SILENTLY. Adding `platform` to four live append tables -# holding hundreds of thousands of rows is a migration on production data to buy a shared grid -# nobody asked for. R2 chose the version with no migration. -# -# ⚠ THE ACCEPTED COST, so it is not rediscovered as a defect: the engine, rollups and grids learn -# two families, and a cross-platform view needs a union. In exchange the two schemas can be HONEST -# about their vendors — which is why `ut_tt_posts` has no `plays` column and `ut_tt_profile` says -# `tt_id` rather than inheriting a column labelled "Instagram id". -TT_TABLE_PREFIX = "ut_tt_" -TT_PROFILE_TABLE = "ut_tt_profile" -TT_SNAPSHOTS_TABLE = "ut_tt_snapshots" -TT_POSTS_TABLE = "ut_tt_posts" -TT_POST_SNAPSHOTS_TABLE = "ut_tt_post_snapshots" -TT_COMMENTS_TABLE = "ut_tt_comments" - -AUTOMATION_RECORD_MODE = "automation" -#: ⚠ THE RAISED CEILING FOLLOWS THE APPEND SHAPE, NOT THE PLATFORM. `ut_tt_snapshots` and -#: `ut_tt_post_snapshots` are append tables for exactly the reason their IG twins are — one row per -#: profile per pull, `maxPosts` rows per pull — so they inherit the ceiling by JOINING THIS SET, -#: which is the mechanism the note above says survives the next table that is not an append table. -APPEND_TABLES = frozenset({IG_SNAPSHOTS_TABLE, IG_POST_SNAPSHOTS_TABLE, - TT_SNAPSHOTS_TABLE, TT_POST_SNAPSHOTS_TABLE}) - -#: ⭐ WAVE 24 (owner ruling R6) — `plain` IS WHAT AN AUTOMATION IS NOW, and it is the DEFAULT. -#: The create wizard is deleted, so nobody picks a kind any more: a new automation is a trigger -#: plus actions and NO machine step. The other three are MACHINE kinds — a scrape, an Instagram -#: column, an Instagram search — and they survive on the automations that already use them -#: (`discover_instagram` stays reachable through the `ig_profile_match` trigger; see C-TRIG). -#: -#: ⚠ ADDING A KIND HERE IS THE SMALL HALF, and the reason is worth reading before adding a fifth: -#: TWO dispatches in this file used to end in a bare `else` that belonged to a SPECIFIC kind -#: rather than to a default — `graph()`'s was `scrape_db`'s and `compose_sentence`'s was -#: `discover_instagram`'s. A kind added here alone would have inherited another kind's whole -#: description: three machine nodes it does not have, and a one-line summary about searching -#: Instagram for up to 0 profiles. Both are explicit arms now, and neither has an `else`. -#: ⭐ WAVE 29 (D-9 / R1) — `discover_tiktok` joins, reachable ONLY through the -#: `tiktok_profile_match` trigger (the same law that makes `discover_instagram` reachable). It -#: lands here IN THE SAME CHANGE as its `RUNNERS` entry and its flip law: a kind in this tuple -#: with no runner is a control that must refuse. -KINDS = ("plain", "scrape_db", "field_instagram", "discover_instagram", "discover_tiktok") -#: ⭐⭐ WAVE 30 · T04 — THE DISCOVERY KINDS UNDER ONE NAME, and this constant is a bug fix rather -#: than tidying. `discover_tiktok` shipped in wave 29 by being added to `KINDS`, `RUNNERS`, -#: `clean_config` and `TRIGGER_*` — four sites that were found — while FIVE more tested the string -#: `"discover_instagram"` directly and were not. The visible symptom was the owner's: picking the -#: TikTok trigger 400'd with *"say how many profiles to fetch"* on the FIRST save, because the seed -#: below was one of the five. The other four are silent — a canvas with no `find` panel, a flow -#: table resolving to the wrong default, and a summary sentence announcing the automation would -#: "do nothing yet". -#: -#: ⛔ SO THE RULE IS: A DISCOVERY BRANCH TESTS MEMBERSHIP OF THIS TUPLE, NEVER A KIND STRING. -#: Three parallel string comparisons is precisely how the third platform gets missed twice more, -#: and the misses are individually invisible — each one degrades a different surface, none of them -#: raises, and every gate stays green (this whole family shipped green in wave 29). -#: ⚠ The kind ↔ platform facts that genuinely DIFFER — the dataset id, the default target table, -#: the field map — stay resolved per kind where they are used. This tuple answers *"is this a -#: corpus search?"* and nothing else; widening it into a platform registry would just move the -#: problem somewhere with a longer name. -DISCOVERY_KINDS = ("discover_instagram", "discover_tiktok") -#: ⭐ WAVE 30 · T06 — THE TRIGGER HALF, and it is a SEPARATE map because the two are NOT -#: interchangeable, which cost a red to learn. Law 1 makes a discovery TRIGGER choose its kind, so -#: `trigger ⇒ kind` always holds — but the converse does NOT: a definition can carry -#: `kind: "discover_instagram"` with no trigger at all (a direct API create does exactly that, and -#: `verify_automation`'s `_discover_defn` fixture is one). Testing the KIND where the rule is about -#: the trigger therefore fires on definitions the picker could never produce — measured: it spawned -#: a preset database under a dry-run fixture that asserts none exists. -#: ⛔ SO: "did somebody PICK a corpus search in the picker?" reads THIS. "Is this stored definition a -#: corpus search?" reads `DISCOVERY_KINDS`. Two questions, two maps, and the gate asserts this one -#: agrees with law 1 rather than trusting that it does. -DISCOVERY_TRIGGER_KIND = {"ig_profile_match": "discover_instagram", - "tiktok_profile_match": "discover_tiktok"} -#: ⭐ WAVE 30 · T08 — the ENRICH action kinds, one per network. Same argument as `DISCOVERY_KINDS` -#: one paragraph up: these two share a validator, a selection, a cooldown and a run summary, and the -#: only things that differ are which connector answers and which tables the rows land in. -#: ⛔ ONE VALIDATOR, NOT TWO. `clean_actions`' enrich branch is ~40 lines of clamps whose comments -#: record why each one is shaped the way it is (`submitted=False` because the client re-posts the -#: whole action list; `limit` clamped twice because the run sees STORED configs). A second copy for -#: TikTok would start identical and drift, and the way it fails is that one network silently accepts -#: a limit the other refuses. -ENRICH_KINDS = ("enrich_instagram", "enrich_tiktok") -#: What a definition with no kind becomes (C-TRIG law 2) — the shape `POST /automations` stores -#: when the body names none, which after R6 is every create the client makes. -DEFAULT_KIND = "plain" -#: R6: no NEW automation may be either of these. They still RUN, still PATCH and still validate — -#: the ruling retires the door, not the two automations behind it (see `create`). -#: ⚠ `discover_instagram` is NOT here: it stays creatable, through the `ig_profile_match` trigger. -RETIRED_KINDS = ("scrape_db", "field_instagram") -#: ⛔ DEBT D-65 — WHAT TO DO INSTEAD, said at every door that refuses one of these. The kinds were -#: not deleted, they were REPLACED by actions any flow can take, and a refusal that does not name -#: the replacement sends somebody looking for a bug in a decision made on purpose. One sentence -#: per kind, in one place, because `create` and `clean_definition` both say it. -RETIRED_KIND_REPLACEMENT = { - "field_instagram": "Add the 'Enrich Instagram profile' action to any automation instead", - "scrape_db": "Add a scrape step to any automation instead", -} -#: ⛔ D-65 — THE RETIRED KINDS THAT ACTUALLY HAVE SOMEWHERE ELSE TO GO, and the distinction is the -#: whole reason this is a second, narrower tuple rather than a reuse of `RETIRED_KINDS`. -#: `field_instagram` was REPLACED: W25/R4 shipped `enrich_instagram` as a `ready:true` action any -#: flow can take, so refusing the kind costs a person nothing but a different click. -#: ⚠ `scrape_db` IS DELIBERATELY ABSENT. Its replacement — the five `web_*` actions — is declared -#: `ready:false` and is DEBT D-51, so refusing a PATCH to it would delete the only way to build a -#: scrape automation and call it tidying up. W24/R6 retired its CREATE door and kept the patch -#: door open on purpose, and there is a gate check that says so in those words. A kind may only be -#: walled off at a door once something else answers the same need. -#: (`REPLACED_KINDS` was here and is deleted with the refusal it gated — see `clean_definition`.) -#: ⭐ WAVE 34 · R12 — `plain` READS "Agent". The owner renamed the module to Agents, so the KIND -#: that means "an ordinary one of these" is an Agent, singular (the module is the plural). -#: ⛔ THE KEY `"plain"` IS UNTOUCHED and so is every other key here: D-65's rule, restated three -#: times in this file, is that a kind is permanent — the LABELS are display and may move freely, -#: the keys are stored on every automation ever created and may not. -KIND_LABELS = {"plain": "Agent", "scrape_db": "Web page to database", - "field_instagram": "Instagram profile column", - "discover_instagram": "Find Instagram profiles", - "discover_tiktok": "Find TikTok profiles"} -#: ⭐⭐ WAVE 34 · CONTRACTS C3 + C4 — WHY A SYSTEM AGENT CANNOT BE DELETED, one sentence per slug. -#: -#: ⛔ ONE PLACE, because two doors ask this question (the DELETE route refuses with it, and the -#: Canvas explains why its controls are read-only) and a refusal that says something different -#: from the surface it refuses on is worse than either sentence alone. D-65's own remedy in a new -#: family: a refusal that does not name the alternative sends somebody looking for a bug in a -#: decision made on purpose. -SYSTEM_AGENT_REASON = { - "field_agent": "this agent is an AI enrichment column. Delete the column on its database and " - "the agent goes with it", - "odoo_sync": "this is the Odoo connection's own sync schedule. Disconnect Odoo in Connectors " - "to stop it, or change how often it runs on this page", -} - - -def system_agent_refusal(slug): - """The sentence for a `system:` slug, or a general one for a slug nobody has written yet. - - ⚠ IT NEVER RETURNS EMPTY. A refusal with no sentence is a 409 a person cannot act on, and the - fallback is the one branch that will be reached by a slug added later and forgotten here. - """ - return SYSTEM_AGENT_REASON.get(str(slug or ""), "this agent is part of the workspace setup " - "and cannot be deleted here") - - -EXTRACTS = ("table", "jsonld") -STATES = ("idle", "running", "ok", "error", "partial") -#: Which capture rung a `field_instagram` automation is allowed to reach for (R1's hybrid). -#: `anonymous` = the $0 ladder only. `brightdata` = the paid rung FIRST, then the ladder as a -#: fallback (unless the fallback is switched off — see `graph`'s `fallback` toggle). -TIERS = ("anonymous", "brightdata") -#: ⚠ STORED CONFIGS SAY `hiker`, AND THEY MEAN "THE PAID RUNG" (wave-20 D-21). The vendor swap -#: must not silently answer that request with the free ladder: `clean_config` refuses an unknown -#: tier by falling back to `anonymous`, so without this alias every existing Instagram automation -#: would quietly stop reaching for exact counts and nothing would say so. Mapping FORWARD keeps -#: the user's expressed intent (they turned the paid step ON) at a cost of ~$0.0015 a profile. -TIER_ALIASES = {"hiker": "brightdata"} - - -def clean_tier(raw): - """A stored/posted tier → a tier this engine runs, or '' when it is neither.""" - t = str(raw or "").strip().lower() - t = TIER_ALIASES.get(t, t) - return t if t in TIERS else "" - - -def row_cap(table_key): - """The row ceiling for ONE table. Per-table rather than global — see `MAX_UT_IG_ROWS`. - - An APPEND table (one row per subject per pull, forever) gets the raised ceiling; everything - else — including the `ut_ig_` table that is an UPSERT — keeps the list-table one. - """ - return MAX_UT_IG_ROWS if str(table_key or "") in APPEND_TABLES else MAX_UT_ROWS - -#: Schedule presets the editor offers. Kept here (not in the client) so the vocabulary the UI -#: shows and the vocabulary the parser accepts cannot drift. -CRON_PRESETS = [ - {"cron": "*/15 * * * *", "label": "Every 15 minutes"}, - {"cron": "0 * * * *", "label": "Hourly"}, - {"cron": "0 6 * * *", "label": "Daily at 06:00"}, - {"cron": "0 6 * * 1", "label": "Weekly (Monday 06:00)"}, - {"cron": "0 6 1 * *", "label": "Monthly (1st, 06:00)"}, -] - -UA = "Mozilla/5.0 (compatible; AIOS-automation/1.0; +https://aios.local/automation)" - - -def _now(): - return _dt.datetime.now() - - -def _stamp(dt=None): - return (dt or _now()).strftime("%Y-%m-%d %H:%M") - - -def _iso(dt=None): - """An ISO stamp **WITH its UTC offset** — `2026-08-05T14:03:11+07:00` (DEBT D-18). - - ⚠ THE OFFSET IS NOT COSMETIC. These stamps are the time axis of the `ut_ig_*` append tables - and they are rendered by a browser, which can only subtract from an instant it can LOCATE. A - naive `2026-08-05T14:03:11` is read as the *reader's* local time, so a container running UTC - minted cells a Jakarta browser would place seven hours in the future — and "2 minutes ago" is - not expressible at all. With the offset the same string is an instant, and relative rendering - becomes possible without migrating a single stored row. - - ⚠ `_parse_iso` reads it BACK as naive local ON PURPOSE. Every cron / `is_due` comparison in - this module is against a naive `_now()`, and mixing aware and naive datetimes raises - `TypeError` — so the offset rides on the WIRE and never enters the arithmetic. - """ - dt = dt or _now() - if dt.tzinfo is None: - dt = dt.astimezone() # a naive stamp from this process IS local time - return dt.isoformat(timespec="seconds") - - -#: ⭐ WAVE 26 · R3 — the DAY out of any stamp we have ever written, for the `date`-typed columns. -#: -#: ⛔ IT MUST READ EVERY SHAPE THE STORE HOLDS, which is the same trap `_parse_iso` documents one -#: function down: `first_found` cells exist as post-D-18 offset stamps -#: (`2026-08-05T14:03:11+07:00`), as pre-D-18 naive ones (`2026-08-05T14:03:11`), as `_stamp()`'s -#: space-separated minute form (`2026-08-05 14:03`) and, after this wave, as bare days. A -#: converter that understood only the newest shape would blank the oldest rows — and a migration -#: that empties cells is indistinguishable from one that moved them. -#: ⚠ Returns "" for anything it cannot read rather than guessing a day. The migration treats "" -#: as LEAVE ALONE, never as a value to write, so an unparseable cell keeps its original text and -#: shows up as itself instead of disappearing. -def _day(s): - """`2026-08-05T14:03:11+07:00` → `2026-08-05`. "" when there is no day in there.""" - raw = str(s or "").strip() - if not raw: - return "" - head = raw.replace("T", " ").split(" ")[0] - try: - _dt.date.fromisoformat(head) - except ValueError: - dt = _parse_iso(raw) - return dt.strftime("%Y-%m-%d") if dt else "" - return head - - -#: ⭐ WAVE 26 · AMENDMENT C1-a — the vendor's 0–1 engagement fraction → this product's 0–100 `pct`. -#: -#: MEASURED on corpus rows: `0.0074`, `0.0656`, `0.0014`, `0.0148`, `0.0274`, `0.0091`. Our `pct` -#: renderer prints the stored number and appends `%`, so storing the raw fraction would show every -#: creator in the book as `0.0%` — a measurement replaced by a wrong measurement, which is worse -#: than the blank cell it came from ([[analyst-chart-library]]: this repo already carries a 0–1 -#: dialect and a 0–100 dialect, and they meet here). -#: ⚠ BLANK STAYS BLANK. `""` means the vendor did not send one — both scrape probe rows were null -#: — and `0.0` would claim we measured zero engagement. -def _pct100(v): - """A 0–1 engagement fraction → a 0–100 percentage. "" when there is nothing to convert.""" - if v is None or (isinstance(v, str) and not v.strip()): - return "" - try: - return _s(round(float(v) * 100.0, 4)) - except (TypeError, ValueError): - return "" - - -def _parse_iso(s): - """A stamp → a NAIVE LOCAL datetime, with or without an offset. - - Both shapes exist in the store simultaneously and always will: every run committed before - D-18 wrote a naive stamp, and nothing rewrites history. A parser that understood only the new - shape would silently return None for them — and `is_due` reads None as "never ran", which - would re-fire every schedule once. Reading both is what makes the change additive. - """ - raw = str(s or "").strip().replace("Z", "+00:00") - dt = None - try: - dt = _dt.datetime.fromisoformat(raw) - except Exception: # noqa: BLE001 - try: - dt = _dt.datetime.strptime(raw[:19], "%Y-%m-%dT%H:%M:%S") - except Exception: # noqa: BLE001 - return None - return dt.astimezone().replace(tzinfo=None) if dt.tzinfo is not None else dt - - -# --------------------------------------------------------------------------------------------- -# THE SSRF RAIL — vendored from scrape.py `guard`, then hardened for a SERVER-SIDE fetcher -# --------------------------------------------------------------------------------------------- -# The skill version guards the URL a developer typed. This one guards a URL that arrived in a -# request body, which changes the threat model in two ways the original does not cover: -# -# 1. REDIRECTS. `httpx.Client(follow_redirects=True)` / `requests.get(allow_redirects=True)` -# never re-enter the guard, so `http://evil.example/x` → 302 → `http://169.254.169.254/…` -# sails straight past a guard that only ever saw the first URL. The negative control ("a -# localhost URL is refused") would still pass while the rail was wide open. `fetch` below -# therefore takes the hops MANUALLY and re-guards every one. -# 2. DNS. A perfectly public hostname may resolve to a private address. `guard` resolves the -# host and checks EVERY answer, not just the literal. -# -# ⚠ STATED, NOT HIDDEN: this is check-then-connect, so a DNS-rebinding attacker who flips the -# record between the guard and the socket is not stopped by it. Closing that needs a pinned -# connection to the validated IP (a custom adapter). Out of scope for v1 and recorded here rather -# than implied away — the rail refuses the realistic cases and says what it does not cover. - -class Refused(ValueError): - """The rail refused a URL. A distinct type so a refusal is never logged as a fetch error.""" - - -def _ip_public(ip): - return not (ip.is_private or ip.is_loopback or ip.is_link_local - or ip.is_reserved or ip.is_unspecified or ip.is_multicast) - - -def guard(url, allowed=()): - """PUBLIC-WEB-ONLY rail: http(s) only, no loopback/private/link-local host, DNS answers - checked too, plus the optional `--allowed` domain rail scrape.py carries.""" - p = urlparse(url or "") - if p.scheme not in ("http", "https"): - raise Refused(f"only http(s) allowed, not {p.scheme!r}") - host = (p.hostname or "").strip() - if not host: - raise Refused("no host in the URL") - low = host.lower() - if (low in ("localhost", "localhost.localdomain") - or low.endswith(".local") or low.endswith(".internal") - or low.endswith(".localhost")): - raise Refused(f"refused non-public host {host!r} (public web only)") - try: # a bare-IP host is decided on the literal - ip = ipaddress.ip_address(low) - except ValueError: - ip = None - if ip is not None and not _ip_public(ip): - raise Refused(f"refused non-public IP {host} (public web only)") - if allowed and not any(low == d.lower() or low.endswith("." + d.lower()) for d in allowed): - raise Refused(f"host {host!r} not in the allowed domains {list(allowed)}") - if ip is None: - try: - infos = socket.getaddrinfo(host, None) - except Exception as e: - raise Refused(f"could not resolve {host!r}: {type(e).__name__}") - for info in infos: - addr = info[4][0] - try: - resolved = ipaddress.ip_address(addr.split("%")[0]) - except ValueError: - continue - if not _ip_public(resolved): - raise Refused( - f"refused {host!r}: it resolves to the non-public address {resolved}") - return True - - -def fetch(url, timeout=20.0, max_kb=2048, allowed=(), max_hops=5, headers=None): - """GET `url`, taking redirects BY HAND so every hop passes `guard`. - - Returns `(status, final_url, body_bytes)`. A hop chain longer than `max_hops` is a refusal, - not a silent truncation — an endless redirect is indistinguishable from an attempt to walk - the fetcher somewhere it was told not to go. - """ - current = url - hdrs = {"User-Agent": UA, "Accept-Language": "en-US,en;q=0.9"} - hdrs.update(headers or {}) - for _hop in range(max_hops + 1): - guard(current, allowed) - r = requests.get(current, timeout=timeout, headers=hdrs, - allow_redirects=False, stream=True) - if r.status_code in (301, 302, 303, 307, 308): - loc = r.headers.get("Location") - r.close() - if not loc: - raise Refused(f"redirect {r.status_code} with no Location header") - current = urljoin(current, loc) - continue - body = r.raw.read(max_kb * 1024, decode_content=True) or b"" - status, final = r.status_code, str(r.url) - r.close() - return status, final, body - raise Refused(f"more than {max_hops} redirects. Refusing to follow further") - - -def fetch_json(url, body, timeout=60.0, max_kb=8192, headers=None): - """POST a JSON body through the SAME guard. Returns `(status, body_bytes)`. - - ⛔ NO REDIRECT FOLLOWING, AND THAT IS THE WHOLE DIFFERENCE FROM `fetch`. `fetch` walks hops by - hand because a redirected GET is ordinary. A redirected POST is not: re-sending a - credential-bearing body to a Location the *server* chose is precisely the hop the SSRF rail - exists to refuse, and `requests`' own `allow_redirects=True` would do it without ever - re-entering `guard`. So a 3xx here is a REFUSAL with the reason on it, never a second request. - (The vendor calls below are the only POSTs this module makes, and they carry the API key.) - """ - guard(url) - hdrs = {"User-Agent": UA, "Content-Type": "application/json", "Accept": "application/json"} - hdrs.update(headers or {}) - r = requests.post(url, timeout=timeout, headers=hdrs, allow_redirects=False, - data=json.dumps(body if body is not None else {}), stream=True) - try: - if r.status_code in (301, 302, 303, 307, 308): - raise Refused(f"the POST answered {r.status_code}. A redirected POST is refused, " - f"never re-sent to a location the server picked") - return r.status_code, (r.raw.read(max_kb * 1024, decode_content=True) or b"") - finally: - r.close() - - -# --------------------------------------------------------------------------------------------- -# EXTRACTION — vendored from scrape.py, unchanged in behaviour -# --------------------------------------------------------------------------------------------- - -def _soup(body): - if BeautifulSoup is None: - raise RuntimeError( - "beautifulsoup4 is not installed in this environment. The automation engine's " - "extraction half needs it (see the wave-18 mailbox: add beautifulsoup4 + lxml to " - "aios-web/requirements.txt).") - try: - return BeautifulSoup(body, "lxml") - except Exception: - return BeautifulSoup(body, "html.parser") - - -def tables(soup): - """Every HTML table as rows. Dict rows when the first row looks like a header.""" - out = [] - for t in soup.find_all("table"): - rows = [] - for tr in t.find_all("tr"): - cells = [c.get_text(" ", strip=True) for c in tr.find_all(["th", "td"])] - if cells: - rows.append(cells) - if not rows: - continue - head, body = rows[0], rows[1:] - if body and len(head) == len(body[0]) and all(head): - out.append([dict(zip(head, r)) for r in body if len(r) == len(head)]) - else: - out.append(rows) - return out - - -def jsonld(soup): - out = [] - for s in soup.find_all("script", attrs={"type": "application/ld+json"}): - try: - out.append(json.loads(s.string or s.get_text())) - except Exception: - pass - return out - - -def meta(soup): - m = {} - if soup.title and soup.title.string: - m["title"] = soup.title.string.strip() - for tag in soup.find_all("meta"): - k = tag.get("name") or tag.get("property") - v = tag.get("content") - if k and v and (k in ("description", "keywords", "author") - or k.startswith(("og:", "twitter:"))): - m[k] = v.strip() - return m - - -def preview(url, extract="table", table_index=0, allowed=()): - """The field-map preview the editor calls BEFORE anything is created: what columns does this - page actually offer, and what do the first rows look like? Never writes.""" - status, final, body = fetch(url, allowed=allowed) - if not (200 <= status < 300): - return {"ok": False, "status": status, "url": final, - "note": f"the page answered {status}. It may be bot-gated or gone", - "columns": [], "sample": [], "rowCount": 0} - soup = _soup(body) - rows = [] - if extract == "jsonld": - blocks = jsonld(soup) - flat = [] - for b in blocks: - if isinstance(b, list): - flat.extend([x for x in b if isinstance(x, dict)]) - elif isinstance(b, dict): - items = b.get("itemListElement") - flat.extend([x for x in items if isinstance(x, dict)] if isinstance(items, list) - else [b]) - rows = [{k: _scalar(v) for k, v in d.items()} for d in flat] - else: - found = tables(soup) - idx = max(0, min(int(table_index or 0), len(found) - 1)) if found else 0 - picked = found[idx] if found else [] - rows = [r for r in picked if isinstance(r, dict)] - cols, seen = [], set() - for r in rows[:50]: - for k in r: - if k not in seen: - seen.add(k) - cols.append(k) - return {"ok": True, "status": status, "url": final, "columns": cols, - "sample": rows[:8], "rowCount": len(rows), - "tableCount": len(tables(soup)) if extract != "jsonld" else 0, - "title": (meta(soup) or {}).get("title", "")} - - -def _scalar(v): - if isinstance(v, (str, int, float)) and not isinstance(v, bool): - return str(v) - if isinstance(v, bool): - return "1" if v else "" - if isinstance(v, dict): - return str(v.get("name") or v.get("@id") or "") - if isinstance(v, list): - return ", ".join(_scalar(x) for x in v[:8]) - return "" - - -# --------------------------------------------------------------------------------------------- -# CRON — a 5-field parser and a "what was the last fire time" walk -# --------------------------------------------------------------------------------------------- -# Deliberately NOT `croniter` (a dependency for ~60 lines) and deliberately NOT a forward -# scheduler. The question a tick asks is backwards-looking — *"was there a scheduled minute -# between the last run and now?"* — and answering it that way is what makes a missed tick -# self-healing: a container that was asleep for two hours runs once on wake, not eleven times and -# not never. - -_FIELD_RANGES = [(0, 59), (0, 23), (1, 31), (1, 12), (0, 6)] - - -def _parse_field(spec, lo, hi): - out = set() - for part in str(spec).split(","): - part = part.strip() - if not part: - raise ValueError("empty cron field part") - step = 1 - if "/" in part: - part, _, s = part.partition("/") - step = int(s) - if step < 1: - raise ValueError("cron step must be >= 1") - if part in ("*", "?"): - a, b = lo, hi - elif "-" in part.lstrip("-"): - a_s, _, b_s = part.partition("-") - a, b = int(a_s), int(b_s) - else: - a = b = int(part) - if a < lo or b > hi or a > b: - raise ValueError(f"cron value {part!r} out of range {lo}..{hi}") - out.update(range(a, b + 1, step)) - return out - - -def parse_cron(expr): - """`'m h dom mon dow'` → `(minutes, hours, doms, months, dows, dom_restricted, dow_restricted)`. - - Raises ValueError on anything malformed — a schedule that cannot be parsed must be refused at - WRITE time, because a cron string nobody can evaluate is an automation that silently never - runs and reports no error anywhere. - """ - parts = str(expr or "").split() - if len(parts) != 5: - raise ValueError("a cron schedule has exactly 5 fields: minute hour day month weekday") - sets = [_parse_field(p, lo, hi) for p, (lo, hi) in zip(parts, _FIELD_RANGES)] - dom_r = parts[2].strip() not in ("*", "?") - dow_r = parts[4].strip() not in ("*", "?") - return (sets[0], sets[1], sets[2], sets[3], sets[4] | ({0} if 7 in sets[4] else set()), - dom_r, dow_r) - - -def _day_matches(day, doms, months, dows, dom_r, dow_r): - if day.month not in months: - return False - # POSIX rule: with BOTH day-of-month and weekday restricted the match is the UNION, not the - # intersection. `0 6 1 * 1` means "the 1st, and every Monday" — getting this backwards makes - # a schedule that looks right fire almost never. - dow = (day.weekday() + 1) % 7 # python Mon=0 -> cron Sun=0 - if dom_r and dow_r: - return day.day in doms or dow in dows - if dom_r: - return day.day in doms - if dow_r: - return dow in dows - return True - - -def prev_fire(expr, now=None, lookback_days=400): - """The most recent scheduled minute at or before `now`, or None inside the lookback. - - Walks by DAY (≤400 iterations) rather than by minute (≥500k) — the day fields decide first, - and only a matching day needs its hours/minutes searched. - """ - minutes, hours, doms, months, dows, dom_r, dow_r = parse_cron(expr) - now = (now or _now()).replace(second=0, microsecond=0) - for back in range(lookback_days + 1): - day = (now - _dt.timedelta(days=back)).date() - if not _day_matches(day, doms, months, dows, dom_r, dow_r): - continue - same_day = back == 0 - for h in sorted(hours, reverse=True): - if same_day and h > now.hour: - continue - for m in sorted(minutes, reverse=True): - if same_day and h == now.hour and m > now.minute: - continue - return _dt.datetime(day.year, day.month, day.day, h, m) - return None - - -def is_due(defn, now=None): - """Should the scheduler run this automation right now? - - Due when a scheduled minute exists strictly AFTER the reference point (the last run, else the - moment the schedule was enabled, else creation) and at or before now. Anchoring on - `enabledAt` rather than firing on the first tick is what stops "enable a daily 06:00 job at - 14:00" from running immediately and looking like a bug. - """ - if not isinstance(defn, dict): - return False - sched = defn.get("schedule") or {} - if not sched.get("enabled"): - return False - try: - fire = prev_fire(sched.get("cron"), now=now) - except ValueError: - return False - if fire is None: - return False - since = (_parse_iso((defn.get("status") or {}).get("lastRunAt")) - or _parse_iso(sched.get("enabledAt")) - or _parse_iso(defn.get("created"))) - return since is None or fire > since - - -def next_fire(expr, now=None, lookahead_days=400): - """The next scheduled minute strictly after `now` — display only (the rail is `is_due`).""" - try: - minutes, hours, doms, months, dows, dom_r, dow_r = parse_cron(expr) - except ValueError: - return None - now = (now or _now()).replace(second=0, microsecond=0) - for ahead in range(lookahead_days + 1): - day = (now + _dt.timedelta(days=ahead)).date() - if not _day_matches(day, doms, months, dows, dom_r, dow_r): - continue - same_day = ahead == 0 - for h in sorted(hours): - if same_day and h < now.hour: - continue - for m in sorted(minutes): - if same_day and h == now.hour and m <= now.minute: - continue - return _dt.datetime(day.year, day.month, day.day, h, m) - return None - - -# --------------------------------------------------------------------------------------------- -# THE UPSERT — pure, so the arithmetic is testable without a store or a network -# --------------------------------------------------------------------------------------------- - -#: ⛔⛔ D-116 — THE SIX CELLS A CORPUS RE-FIND MUST NOT OVERWRITE ONCE AN ENRICHMENT MEASURED THEM. -#: A discovery re-find emits these on EVERY match, not only on insert, so the moment a scheduled -#: search re-matched an account somebody had PAID to enrich, six exact measurements were replaced by -#: pre-collected corpus values of unknown vintage — no error, no visible change other than the -#: number. Measured 2026-08-10 on a live tenant: 0 of 105 profiles had been re-found yet, so this -#: was a defect waiting for its first scheduled re-run rather than one anybody had seen. -#: ⚠ SCALARS ONLY, and that is the ruling. The OBSERVATION is appended either way — the corpus -#: genuinely saw that account at that follower count, and the snapshot series is where a corpus read -#: belongs. Suppressing the observation would be the same defect arriving from the other side. -CORPUS_SOFT_KEYS = ("followers", "following", "avg_engagement", "verified", "category", "bio") - - -def corpus_protect(before, _src): - """Which keys this incoming CORPUS row may not overwrite on `before`. D-116's precedence rule. - - ⛔ THE TEST IS `enriched_at`, i.e. "did an exact read ever write this row", NOT "is the cell - non-empty". The exit condition rules the second one out by name, and rightly: *"NOT by making - `upsert_rows` skip non-empty cells — that would break every re-scrape in the product."* A - re-scrape SHOULD move a number the corpus owns; what it may not do is move one an exact read - owns. - """ - return CORPUS_SOFT_KEYS if str((before or {}).get("enriched_at") or "").strip() else () - - -def upsert_rows(existing, incoming, key_field, cap=None, protect=None): - """Merge scraped rows into a user table's rows BY KEY. Returns `(rows, counts)`. - - `protect` is an optional `f(before, src) -> keys` naming, PER ROW, the keys this incoming row - may not overwrite. Default `None` = the old behaviour exactly, so the other nine call sites are - untouched — the precedence rule belongs to the CALLER that knows its data's provenance, not to - the merge, and a rule baked in here would apply to nine paths that never asked for one. - - THE RULE THAT MATTERS: **an orphan is COUNTED, NEVER DELETED.** A row that has stopped - appearing on the source page has not necessarily stopped existing — the page changed its - filter, the fetch was partial, the site paginated. Deleting on absence turns any upstream - hiccup into silent data loss, and the row may be carrying user-typed overlay values in - columns the scrape never touches. So the run reports `orphans: N` and leaves them alone. - - Only MAPPED keys are written: a re-run never clobbers a column a user added by hand. - - ⚠ CALL THIS ONCE PER TABLE PER RUN, NOT ONCE PER ROW. It rebuilds the whole row dict on - entry, so it is O(existing) per call — fine once, quadratic in a loop. The IG runner used to - call it per POST, which was survivable only because the cap was 5000; against `MAX_UT_IG_ROWS` - that same loop is hundreds of millions of dict copies and the automation simply never - finishes. Raising a cap and batching the writer are ONE change, not two. (W19-C.) - - ⚠ `capped` IS ITS OWN COUNT, deliberately not folded into `skipped`. They are different - facts: `skipped` means "this row had no key, so it could not be upserted" — a property of the - DATA, and usually benign. `capped` means "this table is full and the run is now losing rows" — - a property of the SYSTEM, and never benign. One number for both meant a table hitting its - ceiling was indistinguishable from a page with a few blank cells, which is how a time series - stops silently. The runners turn any `capped` into a `partial` run that NAMES the table. - """ - cap = MAX_UT_ROWS if cap is None else int(cap) - rows = {str(k): dict(v or {}) for k, v in (existing or {}).items()} - counts = {"inserted": 0, "updated": 0, "unchanged": 0, - "skipped": 0, "duplicates": 0, "orphans": 0, "capped": 0} - by_key, dupe_ids = {}, set() - for rid, row in rows.items(): - kv = str(row.get(key_field, "") or "").strip() - if not kv: - continue - if kv in by_key: - dupe_ids.add(rid) # a pre-existing duplicate: first id wins, second left - continue - by_key[kv] = rid - next_id = max((int(r) for r in rows if str(r).isdigit()), default=0) + 1 - seen_keys, incoming_dupes = set(), 0 - for src in incoming or []: - kv = str((src or {}).get(key_field, "") or "").strip() - if not kv: - counts["skipped"] += 1 # no key -> cannot be upserted; never guessed - continue - if kv in seen_keys: - incoming_dupes += 1 # the SOURCE listed it twice; first wins - continue - seen_keys.add(kv) - rid = by_key.get(kv) - if rid is None: - if len(rows) >= cap: - counts["capped"] += 1 # LOUD: the runner turns this into a partial run - continue - rid = str(next_id) - next_id += 1 - rows[rid] = dict(src) - by_key[kv] = rid - counts["inserted"] += 1 - continue - before = rows[rid] - # ⛔ D-116's PRECEDENCE RULE, applied per ROW because provenance is a property of the row. - # `held` counts the cells a lower-provenance source was refused, so the run can SAY it - # rather than quietly doing the right thing — a protection nobody is told about is - # indistinguishable from a source that happened to agree. - keep = set(protect(before, src) or ()) if protect else set() - use = {k: v for k, v in src.items() if k not in keep} if keep else src - if keep: - counts["held"] = counts.get("held", 0) + sum( - 1 for k in keep if k in src and str(before.get(k, "")) != str(src.get(k))) - changed = {k: v for k, v in use.items() if str(before.get(k, "")) != str(v)} - if changed: - before.update(use) - counts["updated"] += 1 - else: - counts["unchanged"] += 1 - counts["duplicates"] = incoming_dupes + len(dupe_ids) - counts["orphans"] = sum( - 1 for kv, rid in by_key.items() if kv not in seen_keys and rid not in dupe_ids) - return rows, counts - - -def dedupe_canonical_rows(existing, key_field, newest_by=""): - """Collapse duplicate logical rows while preserving the lowest stable row id. - - Canonical entity tables use this before every upsert. Snapshot tables deliberately do not: - repeated shortcodes there are new timestamped observations, not duplicate posts. Values are - taken newest-first and then filled from older rows, so a sparse fresh projection does not - erase a field an earlier row knew. - """ - rows = {str(k): dict(v or {}) for k, v in (existing or {}).items()} - groups = {} - for rid, row in rows.items(): - identity = str(row.get(key_field) or "").strip() - if identity: - groups.setdefault(identity, []).append((rid, row)) - removed = 0 - for members in groups.values(): - if len(members) < 2: - continue - keep = min((rid for rid, _row in members), key=lambda r: (not r.isdigit(), int(r) if r.isdigit() else r)) - ordered = sorted(members, key=lambda item: str(item[1].get(newest_by) or ""), reverse=True) \ - if newest_by else members - merged = {} - for _rid, row in ordered: - for key, value in row.items(): - if key not in merged or str(merged.get(key) or "").strip() == "": - merged[key] = value - rows[keep] = merged - for rid, _row in members: - if rid != keep: - rows.pop(rid, None) - removed += 1 - return rows, removed - - -# --------------------------------------------------------------------------------------------- -# USER TABLES — read/write through the RUNTIME (never core.user_tables' module-global key) -# --------------------------------------------------------------------------------------------- - -def _ut_slug(label): - s = re.sub(r"[^a-z0-9]+", "_", str(label or "").strip().lower()).strip("_") - return (s or "table")[:40] - - -def ut_all(rt): - try: - return dict(rt.get(UT_STORE_KEY) or {}) - except Exception: - return {} - - -def disable_for_table(rt, table_key, note="target database deleted"): - """Wave 21 (item 6a, C3): a deleted table's automations are DISABLED loudly, never deleted. - - The definition survives with `schedule.enabled = False` + a `statusNote`, so the rail still - shows what existed and why it stopped — silently deleting a user's automation because its - target died would read as data loss. Returns the ids it touched.""" - key = str(table_key or "") - touched = [] - - def _up(cur): - for aid, d in (cur or {}).items(): - if isinstance(d, dict) and (d.get("config") or {}).get("targetTable") == key: - sch = d.get("schedule") +"""automation_engine.py — wave-18 item 5 (contract C4-AUTO): the automation RUNTIME. + +THE SHAPE, and why it is this shape. An automation is a DEFINITION that lives in the tenant +store and a RUN that lives in this process. The definition is durable, small and rarely written; +the run is hot, chatty and worthless five minutes later. Conflating them is the failure this file +is built to avoid, and it has two halves: + + * **Run state is NEVER persisted while it is running.** A `state: 'running'` written to the + store outlives the process that wrote it, so a container restart mid-run leaves an automation + that can never run again — the 409 sees a `running` nothing will ever clear. Hot state lives + in `_RUNNING` (a process dict, cleared by definition on restart) and the store is written + exactly ONCE per run, at the end. + * **One coalesced store update per run per bucket.** `core/store.py` coalesces on a 20 s + floor per key against a 256-commits/hour repo budget, so a per-ROW write is the wrong shape + by two orders of magnitude — the S&P demo alone is ~500 rows. Every runner below computes + its whole result first and commits it in a single `update` callback. + +WHAT IS VENDORED HERE AND WHY. `.claude/skills/browse/scrape.py` is not on `sys.path` and is not +deployed, so its extraction core is copied into this file (the wave doc's instruction), ported +httpx → requests (the only HTTP dependency this API already carries). The SSRF rail comes with +it, HARDENED rather than merely preserved — see `guard`/`fetch`. That hardening is the point: +the skill version takes a URL from a developer on a CLI; this takes one from a request body. +""" +from __future__ import annotations + +import copy +import datetime as _dt +import hashlib +import hmac +import ipaddress +import json +import os +import re +import socket +import threading +import time +from urllib.parse import urljoin, urlparse + +import requests + +# ⭐ `providers` IS DELIBERATELY NOT IMPORTED HERE ANY MORE (wave 27 item 23). The capability +# router had exactly one consumer in this file — the views top-up inside the Bright Data rung — +# and that rung now lives in `connectors_ig`. So the vendor-routing seam is reached only by the +# connector that routes, which is the shape the split was for: this file no longer knows that +# there is more than one vendor, or that vendors cost money. Re-adding this import is therefore +# a signal, not a convenience — it means something in the automation runtime started making a +# vendor decision, and that belongs one layer down. + +try: # the parser the extraction half needs + from bs4 import BeautifulSoup +except Exception: # pragma: no cover — deploy lag; see mailbox ② + BeautifulSoup = None + +# --------------------------------------------------------------------------------------------- +# THE BUCKET (C4-AUTO) +# --------------------------------------------------------------------------------------------- + +#: The per-tenant store key. Colocated with its reader, the `routes_nav._NAV_PREFS_KEY` pattern. +STORE_KEY = "automations" +#: The user-table bucket. ⚠ Read/written HERE through `TenantRuntime`, never through +#: `core.user_tables` — that module writes the UNPREFIXED module-global key, which is correct for +#: tenant #0 (empty prefix) and silently cross-tenant for every R2 tenant after it. Booked in the +#: session-D mailbox as a finding against A's file rather than fixed from here. +UT_STORE_KEY = "user_tables" +UT_PREFIX = "ut_" + +MAX_AUTOMATIONS = 40 +MAX_RUNS = 20 # trimmed history per automation +MAX_NAME = 80 +#: ⛔ IT DOES **NOT** MIRROR `core.user_tables.MAX_ROWS`, AND THE COMMENT THAT SAID SO WAS A 12x +#: STALE CLAIM — `MAX_ROWS` is 60,000. That sentence is most of what made D-143 hard to see: it +#: read as "the substrate's bound, kept in step", so nobody asked whether a flow was silently +#: walking 5,000 of 32,826 connected rows. It was. +#: **This is the FALLBACK ONLY.** The live answer is per TABLE and comes from +#: `core.user_tables.row_limit` via `_flow_record_cap` — `None` for a connected source (R6: no +#: cap), `MAX_ROWS` for the editable substrate, `0` for a read-through grid. This constant is +#: reached only when that import fails, and it is deliberately the CONSERVATIVE direction. +#: ⚠ Do not "fix" it by raising it to 60,000: a fallback that runs when the evaluator is missing +#: should not also be the widest one. And do not delete it — D-143's own row says the editable +#: substrate still needs a bound. +MAX_UT_ROWS = 5000 +MAX_UT_TABLES = 40 + +#: ⛔⛔ D-112's SENTENCE, AS A CONSTANT, BECAUSE TWO PLACES DEPEND ON IT AGREEING. +#: `enrich_selection` produces it when a step's `fromView` names a view that no longer resolves; +#: `run_flow` tests for it to turn that run `partial` instead of `ok`. Measured live on a real +#: tenant: an enrich action pointed at a deleted view walked ZERO records and reported **`ok`** — +#: indistinguishable on every surface from a run that worked, and a strong candidate for why the +#: owner's enrichment kept appearing to do nothing. +#: ⚠ THE WORDING IS FREE TO CHANGE; the AGREEMENT is not. Both sites read this name, so a better +#: sentence stays a one-line edit instead of a silent regression [[constant-two-features-share]]. +ENRICH_VIEW_UNREADABLE = "the enrich step names a view it cannot read" + +#: ⚠ THE APPEND TABLES NEED A DIFFERENT CAP, and the reason is arithmetic rather than taste. +#: `MAX_UT_ROWS` was sized for a scraped LIST — a page of ~500 companies that is re-read, so the +#: row count is bounded by the page. The `ut_ig_*` tables are the opposite shape: every pull +#: INSERTS a timestamped row (see `SNAPSHOT_FIELDS` and the compound keys below), so at R1's +#: 1k-profiles-daily the snapshot table crosses 5000 rows on **day five** — and the old behaviour +#: was a SILENT `skipped++`, i.e. the table would quietly stop growing and the run would still +#: report success. A time series that stops after five days without saying so is worse than one +#: that was never built. +#: 200k is the owner-directed ceiling (R1's "~200k"). ⚠ IT BUYS VERY DIFFERENT HORIZONS FOR THE +#: TWO APPEND TABLES, and the difference is worth knowing before anyone plans around it: +#: ut_ig_snapshots 1 row per profile per pull -> ~200 days at 1k profiles/day +#: ut_ig_post_snapshots maxPosts rows per pull -> ~8 days at 1k profiles x 24 posts +#: So the post series is the one that fills, and it fills FAST. That is exactly why the breach had +#: to become LOUD (a distinct `capped` count -> a `partial` run naming the table): at this rate the +#: silent version would have flatlined a chart inside a fortnight with a green dot over it. +#: +#: ⚠ MEASURED COST AT THE CEILING (2026-08-04, this box): a full `ut_ig_post_snapshots` serialises +#: to **35.8 MB** of JSON (~1.4 s). `UT_STORE_KEY` is ONE bucket for ALL of a tenant's user tables, +#: so that cost is paid by every unrelated automation write in the tenant too. Booked for the B-3 +#: Postgres tripwires (R2 clause b) rather than papered over — the fix is a row store, not a +#: smaller number. +#: (W19-C; the GENERIC loud-breach for every other table stays booked as DEBT D-11.) +MAX_UT_IG_ROWS = 200_000 +IG_TABLE_PREFIX = "ut_ig_" + +#: ⚠ THE RAISED CEILING BELONGS TO THE **APPEND** TABLES, NOT TO A NAME PREFIX (wave 20). It was +#: keyed off `ut_ig_`, which was exactly right while every `ut_ig_*` table was an append table — +#: and stopped being right the moment discovery added `ut_ig_candidates`, which is UPSERTED BY +#: HANDLE and is bounded by how many accounts exist rather than by how often we look. It would +#: have inherited a 200,000-row ceiling, and with it the measured 35.8 MB single-bucket +#: serialisation cost, purely by an accident of naming. Naming the append tables is the version +#: that stays true when the next `ut_ig_*` table is not one. +IG_SNAPSHOTS_TABLE = "ut_ig_snapshots" +IG_POSTS_TABLE = "ut_ig_posts" +IG_POST_SNAPSHOTS_TABLE = "ut_ig_post_snapshots" +IG_COMMENTS_TABLE = "ut_ig_comments" + +# --------------------------------------------------------------------------------------------- +# ⭐⭐ WAVE 29 (item 7, DEBT D-9, owner rulings R1 + R2) — TIKTOK, AS A PARALLEL FAMILY +# --------------------------------------------------------------------------------------------- +# R1: FULL PARITY — profile AND posts AND comments, not a profile tier. +# R2: a PARALLEL `ut_tt_*` family. `ut_ig_*` is untouched: zero migration, zero risk to the live +# Instagram rows, and the two schemas may DIVERGE where the vendors do. +# +# ⛔ WHY A SECOND FAMILY RATHER THAN A `platform` COLUMN ON THE FIRST, stated here because it is +# the question every reader will ask. `PRESET_PROFILE_FIELDS` carries `platform` and its identity +# is `(platform, handle)` — so a TikTok PROFILE row could already have lived in `ut_ig_profile`. +# The other four tables carry NO discriminator at all: `ut_ig_posts`/`ut_ig_comments` key on +# `shortcode` and the snapshot tables on `influencer_key`, so an Instagram post and a TikTok video +# that happened to share a code would MERGE SILENTLY. Adding `platform` to four live append tables +# holding hundreds of thousands of rows is a migration on production data to buy a shared grid +# nobody asked for. R2 chose the version with no migration. +# +# ⚠ THE ACCEPTED COST, so it is not rediscovered as a defect: the engine, rollups and grids learn +# two families, and a cross-platform view needs a union. In exchange the two schemas can be HONEST +# about their vendors — which is why `ut_tt_posts` has no `plays` column and `ut_tt_profile` says +# `tt_id` rather than inheriting a column labelled "Instagram id". +TT_TABLE_PREFIX = "ut_tt_" +TT_PROFILE_TABLE = "ut_tt_profile" +TT_SNAPSHOTS_TABLE = "ut_tt_snapshots" +TT_POSTS_TABLE = "ut_tt_posts" +TT_POST_SNAPSHOTS_TABLE = "ut_tt_post_snapshots" +TT_COMMENTS_TABLE = "ut_tt_comments" + +AUTOMATION_RECORD_MODE = "automation" +#: ⚠ THE RAISED CEILING FOLLOWS THE APPEND SHAPE, NOT THE PLATFORM. `ut_tt_snapshots` and +#: `ut_tt_post_snapshots` are append tables for exactly the reason their IG twins are — one row per +#: profile per pull, `maxPosts` rows per pull — so they inherit the ceiling by JOINING THIS SET, +#: which is the mechanism the note above says survives the next table that is not an append table. +APPEND_TABLES = frozenset({IG_SNAPSHOTS_TABLE, IG_POST_SNAPSHOTS_TABLE, + TT_SNAPSHOTS_TABLE, TT_POST_SNAPSHOTS_TABLE}) + +#: ⭐ WAVE 24 (owner ruling R6) — `plain` IS WHAT AN AUTOMATION IS NOW, and it is the DEFAULT. +#: The create wizard is deleted, so nobody picks a kind any more: a new automation is a trigger +#: plus actions and NO machine step. The other three are MACHINE kinds — a scrape, an Instagram +#: column, an Instagram search — and they survive on the automations that already use them +#: (`discover_instagram` stays reachable through the `ig_profile_match` trigger; see C-TRIG). +#: +#: ⚠ ADDING A KIND HERE IS THE SMALL HALF, and the reason is worth reading before adding a fifth: +#: TWO dispatches in this file used to end in a bare `else` that belonged to a SPECIFIC kind +#: rather than to a default — `graph()`'s was `scrape_db`'s and `compose_sentence`'s was +#: `discover_instagram`'s. A kind added here alone would have inherited another kind's whole +#: description: three machine nodes it does not have, and a one-line summary about searching +#: Instagram for up to 0 profiles. Both are explicit arms now, and neither has an `else`. +#: ⭐ WAVE 29 (D-9 / R1) — `discover_tiktok` joins, reachable ONLY through the +#: `tiktok_profile_match` trigger (the same law that makes `discover_instagram` reachable). It +#: lands here IN THE SAME CHANGE as its `RUNNERS` entry and its flip law: a kind in this tuple +#: with no runner is a control that must refuse. +KINDS = ("plain", "scrape_db", "field_instagram", "discover_instagram", "discover_tiktok") +#: ⭐⭐ WAVE 30 · T04 — THE DISCOVERY KINDS UNDER ONE NAME, and this constant is a bug fix rather +#: than tidying. `discover_tiktok` shipped in wave 29 by being added to `KINDS`, `RUNNERS`, +#: `clean_config` and `TRIGGER_*` — four sites that were found — while FIVE more tested the string +#: `"discover_instagram"` directly and were not. The visible symptom was the owner's: picking the +#: TikTok trigger 400'd with *"say how many profiles to fetch"* on the FIRST save, because the seed +#: below was one of the five. The other four are silent — a canvas with no `find` panel, a flow +#: table resolving to the wrong default, and a summary sentence announcing the automation would +#: "do nothing yet". +#: +#: ⛔ SO THE RULE IS: A DISCOVERY BRANCH TESTS MEMBERSHIP OF THIS TUPLE, NEVER A KIND STRING. +#: Three parallel string comparisons is precisely how the third platform gets missed twice more, +#: and the misses are individually invisible — each one degrades a different surface, none of them +#: raises, and every gate stays green (this whole family shipped green in wave 29). +#: ⚠ The kind ↔ platform facts that genuinely DIFFER — the dataset id, the default target table, +#: the field map — stay resolved per kind where they are used. This tuple answers *"is this a +#: corpus search?"* and nothing else; widening it into a platform registry would just move the +#: problem somewhere with a longer name. +DISCOVERY_KINDS = ("discover_instagram", "discover_tiktok") +#: ⭐ WAVE 30 · T06 — THE TRIGGER HALF, and it is a SEPARATE map because the two are NOT +#: interchangeable, which cost a red to learn. Law 1 makes a discovery TRIGGER choose its kind, so +#: `trigger ⇒ kind` always holds — but the converse does NOT: a definition can carry +#: `kind: "discover_instagram"` with no trigger at all (a direct API create does exactly that, and +#: `verify_automation`'s `_discover_defn` fixture is one). Testing the KIND where the rule is about +#: the trigger therefore fires on definitions the picker could never produce — measured: it spawned +#: a preset database under a dry-run fixture that asserts none exists. +#: ⛔ SO: "did somebody PICK a corpus search in the picker?" reads THIS. "Is this stored definition a +#: corpus search?" reads `DISCOVERY_KINDS`. Two questions, two maps, and the gate asserts this one +#: agrees with law 1 rather than trusting that it does. +DISCOVERY_TRIGGER_KIND = {"ig_profile_match": "discover_instagram", + "tiktok_profile_match": "discover_tiktok"} +#: ⭐ WAVE 30 · T08 — the ENRICH action kinds, one per network. Same argument as `DISCOVERY_KINDS` +#: one paragraph up: these two share a validator, a selection, a cooldown and a run summary, and the +#: only things that differ are which connector answers and which tables the rows land in. +#: ⛔ ONE VALIDATOR, NOT TWO. `clean_actions`' enrich branch is ~40 lines of clamps whose comments +#: record why each one is shaped the way it is (`submitted=False` because the client re-posts the +#: whole action list; `limit` clamped twice because the run sees STORED configs). A second copy for +#: TikTok would start identical and drift, and the way it fails is that one network silently accepts +#: a limit the other refuses. +ENRICH_KINDS = ("enrich_instagram", "enrich_tiktok") +#: What a definition with no kind becomes (C-TRIG law 2) — the shape `POST /automations` stores +#: when the body names none, which after R6 is every create the client makes. +DEFAULT_KIND = "plain" +#: R6: no NEW automation may be either of these. They still RUN, still PATCH and still validate — +#: the ruling retires the door, not the two automations behind it (see `create`). +#: ⚠ `discover_instagram` is NOT here: it stays creatable, through the `ig_profile_match` trigger. +RETIRED_KINDS = ("scrape_db", "field_instagram") +#: ⛔ DEBT D-65 — WHAT TO DO INSTEAD, said at every door that refuses one of these. The kinds were +#: not deleted, they were REPLACED by actions any flow can take, and a refusal that does not name +#: the replacement sends somebody looking for a bug in a decision made on purpose. One sentence +#: per kind, in one place, because `create` and `clean_definition` both say it. +RETIRED_KIND_REPLACEMENT = { + "field_instagram": "Add the 'Enrich Instagram profile' action to any automation instead", + "scrape_db": "Add a scrape step to any automation instead", +} +#: ⛔ D-65 — THE RETIRED KINDS THAT ACTUALLY HAVE SOMEWHERE ELSE TO GO, and the distinction is the +#: whole reason this is a second, narrower tuple rather than a reuse of `RETIRED_KINDS`. +#: `field_instagram` was REPLACED: W25/R4 shipped `enrich_instagram` as a `ready:true` action any +#: flow can take, so refusing the kind costs a person nothing but a different click. +#: ⚠ `scrape_db` IS DELIBERATELY ABSENT. Its replacement — the five `web_*` actions — is declared +#: `ready:false` and is DEBT D-51, so refusing a PATCH to it would delete the only way to build a +#: scrape automation and call it tidying up. W24/R6 retired its CREATE door and kept the patch +#: door open on purpose, and there is a gate check that says so in those words. A kind may only be +#: walled off at a door once something else answers the same need. +#: (`REPLACED_KINDS` was here and is deleted with the refusal it gated — see `clean_definition`.) +#: ⭐ WAVE 34 · R12 — `plain` READS "Agent". The owner renamed the module to Agents, so the KIND +#: that means "an ordinary one of these" is an Agent, singular (the module is the plural). +#: ⛔ THE KEY `"plain"` IS UNTOUCHED and so is every other key here: D-65's rule, restated three +#: times in this file, is that a kind is permanent — the LABELS are display and may move freely, +#: the keys are stored on every automation ever created and may not. +KIND_LABELS = {"plain": "Agent", "scrape_db": "Web page to database", + "field_instagram": "Instagram profile column", + "discover_instagram": "Find Instagram profiles", + "discover_tiktok": "Find TikTok profiles"} +#: ⭐⭐ WAVE 34 · CONTRACTS C3 + C4 — WHY A SYSTEM AGENT CANNOT BE DELETED, one sentence per slug. +#: +#: ⛔ ONE PLACE, because two doors ask this question (the DELETE route refuses with it, and the +#: Canvas explains why its controls are read-only) and a refusal that says something different +#: from the surface it refuses on is worse than either sentence alone. D-65's own remedy in a new +#: family: a refusal that does not name the alternative sends somebody looking for a bug in a +#: decision made on purpose. +SYSTEM_AGENT_REASON = { + "field_agent": "this agent is an AI enrichment column. Delete the column on its database and " + "the agent goes with it", + "odoo_sync": "this is the Odoo connection's own sync schedule. Disconnect Odoo in Connectors " + "to stop it, or change how often it runs on this page", +} + + +def system_agent_refusal(slug): + """The sentence for a `system:` slug, or a general one for a slug nobody has written yet. + + ⚠ IT NEVER RETURNS EMPTY. A refusal with no sentence is a 409 a person cannot act on, and the + fallback is the one branch that will be reached by a slug added later and forgotten here. + """ + return SYSTEM_AGENT_REASON.get(str(slug or ""), "this agent is part of the workspace setup " + "and cannot be deleted here") + + +EXTRACTS = ("table", "jsonld") +STATES = ("idle", "running", "ok", "error", "partial") +#: Which capture rung a `field_instagram` automation is allowed to reach for (R1's hybrid). +#: `anonymous` = the $0 ladder only. `brightdata` = the paid rung FIRST, then the ladder as a +#: fallback (unless the fallback is switched off — see `graph`'s `fallback` toggle). +TIERS = ("anonymous", "brightdata") +#: ⚠ STORED CONFIGS SAY `hiker`, AND THEY MEAN "THE PAID RUNG" (wave-20 D-21). The vendor swap +#: must not silently answer that request with the free ladder: `clean_config` refuses an unknown +#: tier by falling back to `anonymous`, so without this alias every existing Instagram automation +#: would quietly stop reaching for exact counts and nothing would say so. Mapping FORWARD keeps +#: the user's expressed intent (they turned the paid step ON) at a cost of ~$0.0015 a profile. +TIER_ALIASES = {"hiker": "brightdata"} + + +def clean_tier(raw): + """A stored/posted tier → a tier this engine runs, or '' when it is neither.""" + t = str(raw or "").strip().lower() + t = TIER_ALIASES.get(t, t) + return t if t in TIERS else "" + + +def row_cap(table_key): + """The row ceiling for ONE table. Per-table rather than global — see `MAX_UT_IG_ROWS`. + + An APPEND table (one row per subject per pull, forever) gets the raised ceiling; everything + else — including the `ut_ig_` table that is an UPSERT — keeps the list-table one. + """ + return MAX_UT_IG_ROWS if str(table_key or "") in APPEND_TABLES else MAX_UT_ROWS + +#: Schedule presets the editor offers. Kept here (not in the client) so the vocabulary the UI +#: shows and the vocabulary the parser accepts cannot drift. +CRON_PRESETS = [ + {"cron": "*/15 * * * *", "label": "Every 15 minutes"}, + {"cron": "0 * * * *", "label": "Hourly"}, + {"cron": "0 6 * * *", "label": "Daily at 06:00"}, + {"cron": "0 6 * * 1", "label": "Weekly (Monday 06:00)"}, + {"cron": "0 6 1 * *", "label": "Monthly (1st, 06:00)"}, +] + +UA = "Mozilla/5.0 (compatible; AIOS-automation/1.0; +https://aios.local/automation)" + + +def _now(): + return _dt.datetime.now() + + +def _stamp(dt=None): + return (dt or _now()).strftime("%Y-%m-%d %H:%M") + + +def _iso(dt=None): + """An ISO stamp **WITH its UTC offset** — `2026-08-05T14:03:11+07:00` (DEBT D-18). + + ⚠ THE OFFSET IS NOT COSMETIC. These stamps are the time axis of the `ut_ig_*` append tables + and they are rendered by a browser, which can only subtract from an instant it can LOCATE. A + naive `2026-08-05T14:03:11` is read as the *reader's* local time, so a container running UTC + minted cells a Jakarta browser would place seven hours in the future — and "2 minutes ago" is + not expressible at all. With the offset the same string is an instant, and relative rendering + becomes possible without migrating a single stored row. + + ⚠ `_parse_iso` reads it BACK as naive local ON PURPOSE. Every cron / `is_due` comparison in + this module is against a naive `_now()`, and mixing aware and naive datetimes raises + `TypeError` — so the offset rides on the WIRE and never enters the arithmetic. + """ + dt = dt or _now() + if dt.tzinfo is None: + dt = dt.astimezone() # a naive stamp from this process IS local time + return dt.isoformat(timespec="seconds") + + +#: ⭐ WAVE 26 · R3 — the DAY out of any stamp we have ever written, for the `date`-typed columns. +#: +#: ⛔ IT MUST READ EVERY SHAPE THE STORE HOLDS, which is the same trap `_parse_iso` documents one +#: function down: `first_found` cells exist as post-D-18 offset stamps +#: (`2026-08-05T14:03:11+07:00`), as pre-D-18 naive ones (`2026-08-05T14:03:11`), as `_stamp()`'s +#: space-separated minute form (`2026-08-05 14:03`) and, after this wave, as bare days. A +#: converter that understood only the newest shape would blank the oldest rows — and a migration +#: that empties cells is indistinguishable from one that moved them. +#: ⚠ Returns "" for anything it cannot read rather than guessing a day. The migration treats "" +#: as LEAVE ALONE, never as a value to write, so an unparseable cell keeps its original text and +#: shows up as itself instead of disappearing. +def _day(s): + """`2026-08-05T14:03:11+07:00` → `2026-08-05`. "" when there is no day in there.""" + raw = str(s or "").strip() + if not raw: + return "" + head = raw.replace("T", " ").split(" ")[0] + try: + _dt.date.fromisoformat(head) + except ValueError: + dt = _parse_iso(raw) + return dt.strftime("%Y-%m-%d") if dt else "" + return head + + +#: ⭐ WAVE 26 · AMENDMENT C1-a — the vendor's 0–1 engagement fraction → this product's 0–100 `pct`. +#: +#: MEASURED on corpus rows: `0.0074`, `0.0656`, `0.0014`, `0.0148`, `0.0274`, `0.0091`. Our `pct` +#: renderer prints the stored number and appends `%`, so storing the raw fraction would show every +#: creator in the book as `0.0%` — a measurement replaced by a wrong measurement, which is worse +#: than the blank cell it came from ([[analyst-chart-library]]: this repo already carries a 0–1 +#: dialect and a 0–100 dialect, and they meet here). +#: ⚠ BLANK STAYS BLANK. `""` means the vendor did not send one — both scrape probe rows were null +#: — and `0.0` would claim we measured zero engagement. +def _pct100(v): + """A 0–1 engagement fraction → a 0–100 percentage. "" when there is nothing to convert.""" + if v is None or (isinstance(v, str) and not v.strip()): + return "" + try: + return _s(round(float(v) * 100.0, 4)) + except (TypeError, ValueError): + return "" + + +def _parse_iso(s): + """A stamp → a NAIVE LOCAL datetime, with or without an offset. + + Both shapes exist in the store simultaneously and always will: every run committed before + D-18 wrote a naive stamp, and nothing rewrites history. A parser that understood only the new + shape would silently return None for them — and `is_due` reads None as "never ran", which + would re-fire every schedule once. Reading both is what makes the change additive. + """ + raw = str(s or "").strip().replace("Z", "+00:00") + dt = None + try: + dt = _dt.datetime.fromisoformat(raw) + except Exception: # noqa: BLE001 + try: + dt = _dt.datetime.strptime(raw[:19], "%Y-%m-%dT%H:%M:%S") + except Exception: # noqa: BLE001 + return None + return dt.astimezone().replace(tzinfo=None) if dt.tzinfo is not None else dt + + +# --------------------------------------------------------------------------------------------- +# THE SSRF RAIL — vendored from scrape.py `guard`, then hardened for a SERVER-SIDE fetcher +# --------------------------------------------------------------------------------------------- +# The skill version guards the URL a developer typed. This one guards a URL that arrived in a +# request body, which changes the threat model in two ways the original does not cover: +# +# 1. REDIRECTS. `httpx.Client(follow_redirects=True)` / `requests.get(allow_redirects=True)` +# never re-enter the guard, so `http://evil.example/x` → 302 → `http://169.254.169.254/…` +# sails straight past a guard that only ever saw the first URL. The negative control ("a +# localhost URL is refused") would still pass while the rail was wide open. `fetch` below +# therefore takes the hops MANUALLY and re-guards every one. +# 2. DNS. A perfectly public hostname may resolve to a private address. `guard` resolves the +# host and checks EVERY answer, not just the literal. +# +# ⚠ STATED, NOT HIDDEN: this is check-then-connect, so a DNS-rebinding attacker who flips the +# record between the guard and the socket is not stopped by it. Closing that needs a pinned +# connection to the validated IP (a custom adapter). Out of scope for v1 and recorded here rather +# than implied away — the rail refuses the realistic cases and says what it does not cover. + +class Refused(ValueError): + """The rail refused a URL. A distinct type so a refusal is never logged as a fetch error.""" + + +def _ip_public(ip): + return not (ip.is_private or ip.is_loopback or ip.is_link_local + or ip.is_reserved or ip.is_unspecified or ip.is_multicast) + + +def guard(url, allowed=()): + """PUBLIC-WEB-ONLY rail: http(s) only, no loopback/private/link-local host, DNS answers + checked too, plus the optional `--allowed` domain rail scrape.py carries.""" + p = urlparse(url or "") + if p.scheme not in ("http", "https"): + raise Refused(f"only http(s) allowed, not {p.scheme!r}") + host = (p.hostname or "").strip() + if not host: + raise Refused("no host in the URL") + low = host.lower() + if (low in ("localhost", "localhost.localdomain") + or low.endswith(".local") or low.endswith(".internal") + or low.endswith(".localhost")): + raise Refused(f"refused non-public host {host!r} (public web only)") + try: # a bare-IP host is decided on the literal + ip = ipaddress.ip_address(low) + except ValueError: + ip = None + if ip is not None and not _ip_public(ip): + raise Refused(f"refused non-public IP {host} (public web only)") + if allowed and not any(low == d.lower() or low.endswith("." + d.lower()) for d in allowed): + raise Refused(f"host {host!r} not in the allowed domains {list(allowed)}") + if ip is None: + try: + infos = socket.getaddrinfo(host, None) + except Exception as e: + raise Refused(f"could not resolve {host!r}: {type(e).__name__}") + for info in infos: + addr = info[4][0] + try: + resolved = ipaddress.ip_address(addr.split("%")[0]) + except ValueError: + continue + if not _ip_public(resolved): + raise Refused( + f"refused {host!r}: it resolves to the non-public address {resolved}") + return True + + +def fetch(url, timeout=20.0, max_kb=2048, allowed=(), max_hops=5, headers=None): + """GET `url`, taking redirects BY HAND so every hop passes `guard`. + + Returns `(status, final_url, body_bytes)`. A hop chain longer than `max_hops` is a refusal, + not a silent truncation — an endless redirect is indistinguishable from an attempt to walk + the fetcher somewhere it was told not to go. + """ + current = url + hdrs = {"User-Agent": UA, "Accept-Language": "en-US,en;q=0.9"} + hdrs.update(headers or {}) + for _hop in range(max_hops + 1): + guard(current, allowed) + r = requests.get(current, timeout=timeout, headers=hdrs, + allow_redirects=False, stream=True) + if r.status_code in (301, 302, 303, 307, 308): + loc = r.headers.get("Location") + r.close() + if not loc: + raise Refused(f"redirect {r.status_code} with no Location header") + current = urljoin(current, loc) + continue + body = r.raw.read(max_kb * 1024, decode_content=True) or b"" + status, final = r.status_code, str(r.url) + r.close() + return status, final, body + raise Refused(f"more than {max_hops} redirects. Refusing to follow further") + + +def fetch_json(url, body, timeout=60.0, max_kb=8192, headers=None): + """POST a JSON body through the SAME guard. Returns `(status, body_bytes)`. + + ⛔ NO REDIRECT FOLLOWING, AND THAT IS THE WHOLE DIFFERENCE FROM `fetch`. `fetch` walks hops by + hand because a redirected GET is ordinary. A redirected POST is not: re-sending a + credential-bearing body to a Location the *server* chose is precisely the hop the SSRF rail + exists to refuse, and `requests`' own `allow_redirects=True` would do it without ever + re-entering `guard`. So a 3xx here is a REFUSAL with the reason on it, never a second request. + (The vendor calls below are the only POSTs this module makes, and they carry the API key.) + """ + guard(url) + hdrs = {"User-Agent": UA, "Content-Type": "application/json", "Accept": "application/json"} + hdrs.update(headers or {}) + r = requests.post(url, timeout=timeout, headers=hdrs, allow_redirects=False, + data=json.dumps(body if body is not None else {}), stream=True) + try: + if r.status_code in (301, 302, 303, 307, 308): + raise Refused(f"the POST answered {r.status_code}. A redirected POST is refused, " + f"never re-sent to a location the server picked") + return r.status_code, (r.raw.read(max_kb * 1024, decode_content=True) or b"") + finally: + r.close() + + +# --------------------------------------------------------------------------------------------- +# EXTRACTION — vendored from scrape.py, unchanged in behaviour +# --------------------------------------------------------------------------------------------- + +def _soup(body): + if BeautifulSoup is None: + raise RuntimeError( + "beautifulsoup4 is not installed in this environment. The automation engine's " + "extraction half needs it (see the wave-18 mailbox: add beautifulsoup4 + lxml to " + "aios-web/requirements.txt).") + try: + return BeautifulSoup(body, "lxml") + except Exception: + return BeautifulSoup(body, "html.parser") + + +def tables(soup): + """Every HTML table as rows. Dict rows when the first row looks like a header.""" + out = [] + for t in soup.find_all("table"): + rows = [] + for tr in t.find_all("tr"): + cells = [c.get_text(" ", strip=True) for c in tr.find_all(["th", "td"])] + if cells: + rows.append(cells) + if not rows: + continue + head, body = rows[0], rows[1:] + if body and len(head) == len(body[0]) and all(head): + out.append([dict(zip(head, r)) for r in body if len(r) == len(head)]) + else: + out.append(rows) + return out + + +def jsonld(soup): + out = [] + for s in soup.find_all("script", attrs={"type": "application/ld+json"}): + try: + out.append(json.loads(s.string or s.get_text())) + except Exception: + pass + return out + + +def meta(soup): + m = {} + if soup.title and soup.title.string: + m["title"] = soup.title.string.strip() + for tag in soup.find_all("meta"): + k = tag.get("name") or tag.get("property") + v = tag.get("content") + if k and v and (k in ("description", "keywords", "author") + or k.startswith(("og:", "twitter:"))): + m[k] = v.strip() + return m + + +def preview(url, extract="table", table_index=0, allowed=()): + """The field-map preview the editor calls BEFORE anything is created: what columns does this + page actually offer, and what do the first rows look like? Never writes.""" + status, final, body = fetch(url, allowed=allowed) + if not (200 <= status < 300): + return {"ok": False, "status": status, "url": final, + "note": f"the page answered {status}. It may be bot-gated or gone", + "columns": [], "sample": [], "rowCount": 0} + soup = _soup(body) + rows = [] + if extract == "jsonld": + blocks = jsonld(soup) + flat = [] + for b in blocks: + if isinstance(b, list): + flat.extend([x for x in b if isinstance(x, dict)]) + elif isinstance(b, dict): + items = b.get("itemListElement") + flat.extend([x for x in items if isinstance(x, dict)] if isinstance(items, list) + else [b]) + rows = [{k: _scalar(v) for k, v in d.items()} for d in flat] + else: + found = tables(soup) + idx = max(0, min(int(table_index or 0), len(found) - 1)) if found else 0 + picked = found[idx] if found else [] + rows = [r for r in picked if isinstance(r, dict)] + cols, seen = [], set() + for r in rows[:50]: + for k in r: + if k not in seen: + seen.add(k) + cols.append(k) + return {"ok": True, "status": status, "url": final, "columns": cols, + "sample": rows[:8], "rowCount": len(rows), + "tableCount": len(tables(soup)) if extract != "jsonld" else 0, + "title": (meta(soup) or {}).get("title", "")} + + +def _scalar(v): + if isinstance(v, (str, int, float)) and not isinstance(v, bool): + return str(v) + if isinstance(v, bool): + return "1" if v else "" + if isinstance(v, dict): + return str(v.get("name") or v.get("@id") or "") + if isinstance(v, list): + return ", ".join(_scalar(x) for x in v[:8]) + return "" + + +# --------------------------------------------------------------------------------------------- +# CRON — a 5-field parser and a "what was the last fire time" walk +# --------------------------------------------------------------------------------------------- +# Deliberately NOT `croniter` (a dependency for ~60 lines) and deliberately NOT a forward +# scheduler. The question a tick asks is backwards-looking — *"was there a scheduled minute +# between the last run and now?"* — and answering it that way is what makes a missed tick +# self-healing: a container that was asleep for two hours runs once on wake, not eleven times and +# not never. + +_FIELD_RANGES = [(0, 59), (0, 23), (1, 31), (1, 12), (0, 6)] + + +def _parse_field(spec, lo, hi): + out = set() + for part in str(spec).split(","): + part = part.strip() + if not part: + raise ValueError("empty cron field part") + step = 1 + if "/" in part: + part, _, s = part.partition("/") + step = int(s) + if step < 1: + raise ValueError("cron step must be >= 1") + if part in ("*", "?"): + a, b = lo, hi + elif "-" in part.lstrip("-"): + a_s, _, b_s = part.partition("-") + a, b = int(a_s), int(b_s) + else: + a = b = int(part) + if a < lo or b > hi or a > b: + raise ValueError(f"cron value {part!r} out of range {lo}..{hi}") + out.update(range(a, b + 1, step)) + return out + + +def parse_cron(expr): + """`'m h dom mon dow'` → `(minutes, hours, doms, months, dows, dom_restricted, dow_restricted)`. + + Raises ValueError on anything malformed — a schedule that cannot be parsed must be refused at + WRITE time, because a cron string nobody can evaluate is an automation that silently never + runs and reports no error anywhere. + """ + parts = str(expr or "").split() + if len(parts) != 5: + raise ValueError("a cron schedule has exactly 5 fields: minute hour day month weekday") + sets = [_parse_field(p, lo, hi) for p, (lo, hi) in zip(parts, _FIELD_RANGES)] + dom_r = parts[2].strip() not in ("*", "?") + dow_r = parts[4].strip() not in ("*", "?") + return (sets[0], sets[1], sets[2], sets[3], sets[4] | ({0} if 7 in sets[4] else set()), + dom_r, dow_r) + + +def _day_matches(day, doms, months, dows, dom_r, dow_r): + if day.month not in months: + return False + # POSIX rule: with BOTH day-of-month and weekday restricted the match is the UNION, not the + # intersection. `0 6 1 * 1` means "the 1st, and every Monday" — getting this backwards makes + # a schedule that looks right fire almost never. + dow = (day.weekday() + 1) % 7 # python Mon=0 -> cron Sun=0 + if dom_r and dow_r: + return day.day in doms or dow in dows + if dom_r: + return day.day in doms + if dow_r: + return dow in dows + return True + + +def prev_fire(expr, now=None, lookback_days=400): + """The most recent scheduled minute at or before `now`, or None inside the lookback. + + Walks by DAY (≤400 iterations) rather than by minute (≥500k) — the day fields decide first, + and only a matching day needs its hours/minutes searched. + """ + minutes, hours, doms, months, dows, dom_r, dow_r = parse_cron(expr) + now = (now or _now()).replace(second=0, microsecond=0) + for back in range(lookback_days + 1): + day = (now - _dt.timedelta(days=back)).date() + if not _day_matches(day, doms, months, dows, dom_r, dow_r): + continue + same_day = back == 0 + for h in sorted(hours, reverse=True): + if same_day and h > now.hour: + continue + for m in sorted(minutes, reverse=True): + if same_day and h == now.hour and m > now.minute: + continue + return _dt.datetime(day.year, day.month, day.day, h, m) + return None + + +def is_due(defn, now=None): + """Should the scheduler run this automation right now? + + Due when a scheduled minute exists strictly AFTER the reference point (the last run, else the + moment the schedule was enabled, else creation) and at or before now. Anchoring on + `enabledAt` rather than firing on the first tick is what stops "enable a daily 06:00 job at + 14:00" from running immediately and looking like a bug. + """ + if not isinstance(defn, dict): + return False + sched = defn.get("schedule") or {} + if not sched.get("enabled"): + return False + try: + fire = prev_fire(sched.get("cron"), now=now) + except ValueError: + return False + if fire is None: + return False + since = (_parse_iso((defn.get("status") or {}).get("lastRunAt")) + or _parse_iso(sched.get("enabledAt")) + or _parse_iso(defn.get("created"))) + return since is None or fire > since + + +def next_fire(expr, now=None, lookahead_days=400): + """The next scheduled minute strictly after `now` — display only (the rail is `is_due`).""" + try: + minutes, hours, doms, months, dows, dom_r, dow_r = parse_cron(expr) + except ValueError: + return None + now = (now or _now()).replace(second=0, microsecond=0) + for ahead in range(lookahead_days + 1): + day = (now + _dt.timedelta(days=ahead)).date() + if not _day_matches(day, doms, months, dows, dom_r, dow_r): + continue + same_day = ahead == 0 + for h in sorted(hours): + if same_day and h < now.hour: + continue + for m in sorted(minutes): + if same_day and h == now.hour and m <= now.minute: + continue + return _dt.datetime(day.year, day.month, day.day, h, m) + return None + + +# --------------------------------------------------------------------------------------------- +# THE UPSERT — pure, so the arithmetic is testable without a store or a network +# --------------------------------------------------------------------------------------------- + +#: ⛔⛔ D-116 — THE SIX CELLS A CORPUS RE-FIND MUST NOT OVERWRITE ONCE AN ENRICHMENT MEASURED THEM. +#: A discovery re-find emits these on EVERY match, not only on insert, so the moment a scheduled +#: search re-matched an account somebody had PAID to enrich, six exact measurements were replaced by +#: pre-collected corpus values of unknown vintage — no error, no visible change other than the +#: number. Measured 2026-08-10 on a live tenant: 0 of 105 profiles had been re-found yet, so this +#: was a defect waiting for its first scheduled re-run rather than one anybody had seen. +#: ⚠ SCALARS ONLY, and that is the ruling. The OBSERVATION is appended either way — the corpus +#: genuinely saw that account at that follower count, and the snapshot series is where a corpus read +#: belongs. Suppressing the observation would be the same defect arriving from the other side. +CORPUS_SOFT_KEYS = ("followers", "following", "avg_engagement", "verified", "category", "bio") + + +def corpus_protect(before, _src): + """Which keys this incoming CORPUS row may not overwrite on `before`. D-116's precedence rule. + + ⛔ THE TEST IS `enriched_at`, i.e. "did an exact read ever write this row", NOT "is the cell + non-empty". The exit condition rules the second one out by name, and rightly: *"NOT by making + `upsert_rows` skip non-empty cells — that would break every re-scrape in the product."* A + re-scrape SHOULD move a number the corpus owns; what it may not do is move one an exact read + owns. + """ + return CORPUS_SOFT_KEYS if str((before or {}).get("enriched_at") or "").strip() else () + + +def upsert_rows(existing, incoming, key_field, cap=None, protect=None): + """Merge scraped rows into a user table's rows BY KEY. Returns `(rows, counts)`. + + `protect` is an optional `f(before, src) -> keys` naming, PER ROW, the keys this incoming row + may not overwrite. Default `None` = the old behaviour exactly, so the other nine call sites are + untouched — the precedence rule belongs to the CALLER that knows its data's provenance, not to + the merge, and a rule baked in here would apply to nine paths that never asked for one. + + THE RULE THAT MATTERS: **an orphan is COUNTED, NEVER DELETED.** A row that has stopped + appearing on the source page has not necessarily stopped existing — the page changed its + filter, the fetch was partial, the site paginated. Deleting on absence turns any upstream + hiccup into silent data loss, and the row may be carrying user-typed overlay values in + columns the scrape never touches. So the run reports `orphans: N` and leaves them alone. + + Only MAPPED keys are written: a re-run never clobbers a column a user added by hand. + + ⚠ CALL THIS ONCE PER TABLE PER RUN, NOT ONCE PER ROW. It rebuilds the whole row dict on + entry, so it is O(existing) per call — fine once, quadratic in a loop. The IG runner used to + call it per POST, which was survivable only because the cap was 5000; against `MAX_UT_IG_ROWS` + that same loop is hundreds of millions of dict copies and the automation simply never + finishes. Raising a cap and batching the writer are ONE change, not two. (W19-C.) + + ⚠ `capped` IS ITS OWN COUNT, deliberately not folded into `skipped`. They are different + facts: `skipped` means "this row had no key, so it could not be upserted" — a property of the + DATA, and usually benign. `capped` means "this table is full and the run is now losing rows" — + a property of the SYSTEM, and never benign. One number for both meant a table hitting its + ceiling was indistinguishable from a page with a few blank cells, which is how a time series + stops silently. The runners turn any `capped` into a `partial` run that NAMES the table. + """ + cap = MAX_UT_ROWS if cap is None else int(cap) + rows = {str(k): dict(v or {}) for k, v in (existing or {}).items()} + counts = {"inserted": 0, "updated": 0, "unchanged": 0, + "skipped": 0, "duplicates": 0, "orphans": 0, "capped": 0} + by_key, dupe_ids = {}, set() + for rid, row in rows.items(): + kv = str(row.get(key_field, "") or "").strip() + if not kv: + continue + if kv in by_key: + dupe_ids.add(rid) # a pre-existing duplicate: first id wins, second left + continue + by_key[kv] = rid + next_id = max((int(r) for r in rows if str(r).isdigit()), default=0) + 1 + seen_keys, incoming_dupes = set(), 0 + for src in incoming or []: + kv = str((src or {}).get(key_field, "") or "").strip() + if not kv: + counts["skipped"] += 1 # no key -> cannot be upserted; never guessed + continue + if kv in seen_keys: + incoming_dupes += 1 # the SOURCE listed it twice; first wins + continue + seen_keys.add(kv) + rid = by_key.get(kv) + if rid is None: + if len(rows) >= cap: + counts["capped"] += 1 # LOUD: the runner turns this into a partial run + continue + rid = str(next_id) + next_id += 1 + rows[rid] = dict(src) + by_key[kv] = rid + counts["inserted"] += 1 + continue + before = rows[rid] + # ⛔ D-116's PRECEDENCE RULE, applied per ROW because provenance is a property of the row. + # `held` counts the cells a lower-provenance source was refused, so the run can SAY it + # rather than quietly doing the right thing — a protection nobody is told about is + # indistinguishable from a source that happened to agree. + keep = set(protect(before, src) or ()) if protect else set() + use = {k: v for k, v in src.items() if k not in keep} if keep else src + if keep: + counts["held"] = counts.get("held", 0) + sum( + 1 for k in keep if k in src and str(before.get(k, "")) != str(src.get(k))) + changed = {k: v for k, v in use.items() if str(before.get(k, "")) != str(v)} + if changed: + before.update(use) + counts["updated"] += 1 + else: + counts["unchanged"] += 1 + counts["duplicates"] = incoming_dupes + len(dupe_ids) + counts["orphans"] = sum( + 1 for kv, rid in by_key.items() if kv not in seen_keys and rid not in dupe_ids) + return rows, counts + + +def dedupe_canonical_rows(existing, key_field, newest_by=""): + """Collapse duplicate logical rows while preserving the lowest stable row id. + + Canonical entity tables use this before every upsert. Snapshot tables deliberately do not: + repeated shortcodes there are new timestamped observations, not duplicate posts. Values are + taken newest-first and then filled from older rows, so a sparse fresh projection does not + erase a field an earlier row knew. + """ + rows = {str(k): dict(v or {}) for k, v in (existing or {}).items()} + groups = {} + for rid, row in rows.items(): + identity = str(row.get(key_field) or "").strip() + if identity: + groups.setdefault(identity, []).append((rid, row)) + removed = 0 + for members in groups.values(): + if len(members) < 2: + continue + keep = min((rid for rid, _row in members), key=lambda r: (not r.isdigit(), int(r) if r.isdigit() else r)) + ordered = sorted(members, key=lambda item: str(item[1].get(newest_by) or ""), reverse=True) \ + if newest_by else members + merged = {} + for _rid, row in ordered: + for key, value in row.items(): + if key not in merged or str(merged.get(key) or "").strip() == "": + merged[key] = value + rows[keep] = merged + for rid, _row in members: + if rid != keep: + rows.pop(rid, None) + removed += 1 + return rows, removed + + +# --------------------------------------------------------------------------------------------- +# USER TABLES — read/write through the RUNTIME (never core.user_tables' module-global key) +# --------------------------------------------------------------------------------------------- + +def _ut_slug(label): + s = re.sub(r"[^a-z0-9]+", "_", str(label or "").strip().lower()).strip("_") + return (s or "table")[:40] + + +def ut_all(rt): + try: + return dict(rt.get(UT_STORE_KEY) or {}) + except Exception: + return {} + + +def disable_for_table(rt, table_key, note="target database deleted"): + """Wave 21 (item 6a, C3): a deleted table's automations are DISABLED loudly, never deleted. + + The definition survives with `schedule.enabled = False` + a `statusNote`, so the rail still + shows what existed and why it stopped — silently deleting a user's automation because its + target died would read as data loss. Returns the ids it touched.""" + key = str(table_key or "") + touched = [] + + def _up(cur): + for aid, d in (cur or {}).items(): + if isinstance(d, dict) and (d.get("config") or {}).get("targetTable") == key: + sch = d.get("schedule") if not isinstance(sch, dict): sch = d["schedule"] = {} sch["enabled"] = False @@ -963,12355 +963,12727 @@ def disable_for_table(rt, table_key, note="target database deleted"): if isinstance(trg, dict): trg["paused"] = True d["statusNote"] = note - touched.append(str(aid)) - return cur - - if key: - _store_update(rt, _up, flush="sync") - return touched - - -def ut_get(rt, key): - return ut_all(rt).get(str(key)) - - -def retire_automation_stage_fields(rt, tables=None): - """Delete obsolete Board-only fields and their hidden cells, never user fields. - - The authoritative selector is the engine's own ``automation.stageField`` / ``cyclesField`` - metadata — labels such as “Stage” are ordinary user vocabulary and are not touched. Old - generated timestamp/cycle cells are cleared too, including rows whose field definition was - removed by an interrupted earlier migration. Re-running this migration is a no-op. - - ⭐ WAVE 29 (W29-T01) — ``tables`` LETS A CALLER LEND ITS SNAPSHOT. The SCAN below is - O(all row-cells in the tenant) over a bucket whose documented ceiling is 35.8 MB / ~1.4 s to - deep-copy (see this module's header), and `GET /automations` was paying for THREE independent - copies of it per request. The scan is read-only, so borrowing the caller's copy is free; the - WRITE below still goes through `rt.update`, which re-reads under the store lock, so a lent - snapshot can never be the thing that gets written back. - """ - tables = ut_all(rt) if tables is None else tables - stage_keys = set() - for table in tables.values(): - if not isinstance(table, dict): - continue - for field in table.get("fields") or []: - auto = field.get("automation") if isinstance(field, dict) else None - if isinstance(auto, dict) and (auto.get("stageField") or auto.get("cyclesField")): - stage_keys.add(str(field.get("key") or "")) - for row in (table.get("rows") or {}).values(): - for key in (row or {}): - text = str(key) - if text.startswith("stage_auto_"): - stage_keys.add(text.removesuffix("_at").removesuffix("_cycles")) - stage_keys.discard("") - if not stage_keys: - return {"tables": 0, "fields": 0, "cells": 0} - - changed = {"tables": set(), "fields": 0, "cells": 0} - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - for table_key, table in cur.items(): - if not isinstance(table, dict): - continue - kept, removed = [], set() - for field in table.get("fields") or []: - auto = field.get("automation") if isinstance(field, dict) else None - key = str(field.get("key") or "") if isinstance(field, dict) else "" - if key in stage_keys and isinstance(auto, dict) and \ - (auto.get("stageField") or auto.get("cyclesField")): - removed.add(key) - changed["fields"] += 1 - continue - kept.append(field) - if removed: - table["fields"] = kept - changed["tables"].add(str(table_key)) - for row in (table.get("rows") or {}).values(): - if not isinstance(row, dict): - continue - for stage_key in stage_keys: - for key in (stage_key, stage_key + "_at", stage_key + "_cycles"): - if key in row: - row.pop(key, None) - changed["cells"] += 1 - changed["tables"].add(str(table_key)) - return cur - - rt.update(UT_STORE_KEY, _up, flush="sync") - return {"tables": len(changed["tables"]), "fields": changed["fields"], - "cells": changed["cells"]} - - -def bind_unbound_fields(rt): - """C8's migration (wave 22, stated choice): existing automation bags WITHOUT a `flowId` - are bound to the definition that already writes them — matched on the (targetTable, - fieldKey) pair the definition carries, which is the binding W21-C2 established. A bag no - definition references is DISABLED with the reason on it rather than deleted or guessed: - the column was already dead (nothing runs a column no flow names), and now it says so. - Returns `(bound, disabled)`; costs zero store commits when there is nothing to migrate.""" - by_binding = {} - for aid, d in all_definitions(rt).items(): - cfg = d.get("config") or {} - if cfg.get("fieldKey") and cfg.get("targetTable"): - by_binding[(str(cfg["targetTable"]), str(cfg["fieldKey"]))] = str(aid) - plan = {} - for tk, t in ut_all(rt).items(): - for f in (t.get("fields") or []): - a = f.get("automation") - # An already-DISABLED bag is a decision this migration made on a previous pass — - # re-planning it every call would turn the once-per-process sweep into a write - # per read. - if isinstance(a, dict) and not a.get("flowId") and not a.get("stageField") \ - and not a.get("disabled"): - plan[(tk, str(f.get("key")))] = by_binding.get((tk, str(f.get("key")))) - if not plan: - return 0, 0 - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - for (tk, fk), aid in plan.items(): - for f in ((cur.get(tk) or {}).get("fields") or []): - if f.get("key") == fk and isinstance(f.get("automation"), dict): - if aid: - f["automation"]["flowId"] = aid - else: - f["automation"]["disabled"] = True - f["automation"]["statusNote"] = ( - "not bound to any flow. Create an automation for this column " - "or delete it") - return cur - - rt.update(UT_STORE_KEY, _up, flush="sync") - bound = sum(1 for v in plan.values() if v) - return bound, len(plan) - bound - - -def ut_key_for(label, key=None): - """The table key a label resolves to. Factored out of `ut_ensure` so the DRY-RUN path (which - must not create anything) resolves the same key by construction rather than by copying the - derivation and drifting from it.""" - k = str(key or (UT_PREFIX + _ut_slug(label))) - return k if k.startswith(UT_PREFIX) else UT_PREFIX + k - - -#: Machine names, mirrored from `core.user_tables.MACHINE_OWNERS`. A LOCAL literal for the same -#: reason `UT_FIELD_TYPES` is one over there — this module must stay importable without dragging -#: `core` into the API's boot path — and the two are held in step by a gate check. -MACHINE_OWNERS = ("automation", "scheduler") - - -def ut_ensure(rt, label, fields, username="automation", key=None, flow_tag="", - record_mode="", lock_fields=False, tables=None): - """Create the table if it is missing; return its key. Idempotent — a re-run of an automation - that owns a table must not spawn `ut_x_2`, so the key is DERIVED from the label (or given) - and an existing table with that key is adopted, not duplicated. - - ⭐⭐ WAVE 31 · T30 — `tables` LETS A CALLER LEND THE SNAPSHOT IT IS ALREADY HOLDING, and it is - the SKIP TEST below that it pays for. `rt.get` deep-copies the whole tenant document on every - call (documented ceiling 35.8 MB / ~1.4 s), so a caller that ensures FOUR tables in one pass - paid four copies to answer four questions about one document. Owner item 7, verbatim: *"It still - takes forever to change the Config for automation as well. It just say Saving..."* — measured - **7,912 ms** for one save on `4258a93`. - - ⚠ THE LEND IS READ-ONLY AND SAFE BY INSPECTION, not by hope — the same argument - `retire_automation_stage_fields` (this module, same parameter name, same contract) already - makes: only the `have` lookup below reads it, and the WRITE still goes through `rt.update`, - whose `_up` re-reads the live document under the store lock. **A lent snapshot can therefore - never be the thing that gets written back**, and the worst a stale one can do is spend a commit - that would have been skipped — never write a wrong value. Callers that ensure two tables - under the SAME key in one pass pass `tables=None` for the second (see `_spawn_presets`). - - Fields are MERGED, never replaced: a user who added a column to an automation's table keeps - it, and a new source column joins on the next run. - - `flow_tag` (wave 22, C7/item 5): every field THIS call adds is stamped - `automation: {flowId: }` — the pre-set columns an IG automation spawns carry their - provenance. Fields already on the table keep whatever tag they have (first flow wins; - shared tables like ut_ig_snapshots are fed by many flows and the tag is provenance, not - ownership). - - ⛔ AND IT STAMPS A HUMAN OWNER, WHICH IT DID NOT (wave 20, item 3). `createdBy` was whoever - or WHATEVER ran the automation, so the same table belonged to a person if its first run was - manual and to `"scheduler"` if the schedule got there first — and `user_tables.may_open` - admits only the creator or an admin, so **whether you could open your own Instagram - snapshots depended on a race you never saw**. The owner is now the automation's creator, and - a table already stamped with a machine name is ADOPTED the next time a run knows a human - one. Adoption is a repair, not a widening: the automation's creator is the person who asked - for the table in the first place. - """ - key = ut_key_for(label, key) - # ⭐ WAVE 26 — THE MIGRATION RIDES THE WRITE PATH, and that placement is the point. - # - # `ut_ensure` MERGES fields by key and never re-types an existing column, so on its own it - # would leave every table already in production on the old `text` schema forever while new - # ones got the honest types — the split schema R3's note warned about, arriving through the - # very function the note was written on. Migrating here means a table is brought forward - # immediately BEFORE anything appends to it, so no caller has to remember anything and no - # tenant is left behind by a script nobody ran. - # ⚠ Cheap by construction: `migrate_ig_tables` returns without a write when the table is - # already current, which after the first run is every call. - if key and {f.get("key") for f in (fields or [])} & set(PRESET_PROFILE_KEYS): - try: - migrate_ig_tables(rt, log=lambda *_a: None, only=key) - except Exception: # noqa: BLE001 - # A migration that cannot run must not stop the automation from writing its rows. - # The old schema still reads; a refused write loses the pull we just paid for. - pass - wanted = [] - for raw_field in (fields or []): - field = dict(raw_field) - if flow_tag or lock_fields: - automation = dict(field.get("automation") or {}) - if flow_tag: - automation.setdefault("flowId", str(flow_tag)) - if lock_fields: - automation["preset"] = True - field["automation"] = automation - wanted.append(field) - created = _iso() - human = username if username and username not in MACHINE_OWNERS else "" - - # ⚠ SKIP THE WRITE WHEN NOTHING WOULD CHANGE. Without this, every re-run spends a store - # commit re-writing an identical definition — against a 20 s flush floor and a 256/hr repo - # budget, an idempotent helper that always writes is the same defect as a per-row insert. - have = ut_get(rt, key) if tables is None else tables.get(str(key)) - have_fields = {str(f.get("key") or ""): f for f in ((have or {}).get("fields") or [])} - missing = [f for f in wanted if f.get("key") not in have_fields] - missing_locks = [f for f in wanted - if lock_fields and f.get("key") in have_fields - and (not isinstance(have_fields[f.get("key")].get("automation"), dict) - or have_fields[f.get("key")]["automation"].get("preset") is not True)] - if (have is not None and not missing and not missing_locks - and not (record_mode and have.get("recordMode") != record_mode) - and not (human and (have.get("createdBy") or "") in MACHINE_OWNERS)): - return key - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - t = cur.get(key) - if t is None: - if len(cur) >= MAX_UT_TABLES: - return cur - cur[key] = {"key": key, "label": str(label)[:60], "source": "Automation", - "createdBy": username, "created": created, - "fields": wanted, "rows": {}} - if record_mode: - cur[key]["recordMode"] = record_mode - return cur - have = {f.get("key") for f in (t.get("fields") or [])} - for f in wanted: - if f.get("key") not in have: - t.setdefault("fields", []).append(f) - have.add(f.get("key")) - elif lock_fields: - stored = next((g for g in (t.get("fields") or []) - if g.get("key") == f.get("key")), None) - if stored is not None: - automation = dict(stored.get("automation") or {}) - if flow_tag: - automation.setdefault("flowId", str(flow_tag)) - automation["preset"] = True - stored["automation"] = automation - # ADOPTION: a machine name is not an owner. It never overwrites a human one. - if human and (t.get("createdBy") or "") in MACHINE_OWNERS: - t["createdBy"] = human - # An automation's table SAYS an automation owns it — the nav badge reads from this. - t["source"] = t.get("source") or "Automation" - if record_mode: - t["recordMode"] = record_mode - return cur - - rt.update(UT_STORE_KEY, _up, flush="sync") - return key - - -def ut_write_rows(rt, key, rows): - """ONE store update for the WHOLE row set — the flush-ceiling rule (see the module header).""" - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - t = cur.get(key) - if t is not None: - t["rows"] = rows - return cur - rt.update(UT_STORE_KEY, _up, flush="sync") - - -def ut_set_cell(rt, key, row_id, field_key, value, extra_rows=None): - """Write one automation-owned cell (+ optional whole extra tables) in ONE update.""" - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - t = cur.get(key) - if t is not None: - t.setdefault("rows", {}).setdefault(str(row_id), {})[field_key] = value - for k, rws in (extra_rows or {}).items(): - tt = cur.get(k) - if tt is not None: - tt["rows"] = rws - return cur - rt.update(UT_STORE_KEY, _up, flush="sync") - - -IG_FIELD_DESCRIPTIONS = { - "alt_text": "Accessibility text attached to the Instagram post.", - # ⚠ WIDENED 2026-08-10, because the column gained a second writer and the old sentence would - # have made it lie. It was written for the anonymous rung, whose numbers are genuinely ROUNDED - # ("10.4K followers"). A DISCOVERY observation is exact-looking and STALE instead — read off - # the vendor's pre-collected corpus at a collection time we are not told. Both mean the same - # thing to a reader ("do not treat this as an exact count taken at Pulled at"), and a - # description that named only the first would have quietly excluded the second. - "approx": "Checked when the counts are rounded, or were read from a pre-collected corpus " - "rather than measured at the time shown.", - "avg_comments_12": "Average comments across the 12 most recent captured posts.", - "avg_engagement": "Average engagement rate reported for the profile.", - "avg_likes_12": "Average likes across the 12 most recent captured posts.", - "avg_plays_12": "Average plays across the 12 most recent captured posts.", - "avg_views_12": "Average views across the 12 most recent captured posts.", - "views": "View count for the post, as Instagram displays it.", - "video_duration": "Length of the video in seconds.", - "comments_disabled": "Checked when the creator has turned comments off for this post.", - "plays": "Times the video started playing, including replays.", - "bio": "Biography shown on the Instagram profile.", - "bio_hashtags": "Hashtags listed in the profile biography.", - "business_category": "Instagram's business category for the account.", - "caption": "Caption published with the Instagram post.", - "category": "Instagram's category for the profile.", - "comment_key": "Unique ID for the captured comment record.", - "commented_at": "Date the comment was posted.", - "comments": "Comment count reported for the post.", - "comments_captured": "Number of distinct comment records linked to this profile.", - "comments_link": "Comment records linked to this record.", - "country_code": "Country code reported for the profile.", - "created_by": "User whose automation first added this lead.", - "enriched_at": "Date the profile was last enriched.", - # ⭐ ITEM 16. The description says GUESS out loud, because the column is one and the number - # beside it is the only thing that says how much of one. - "location_guess": "Most common city tagged across this profile's captured posts - a guess, " - "not a stated location.", - "location_confidence": "Share of this profile's geotagged posts that agree on the guessed " - "city.", - "external_url": "Website linked from the Instagram biography.", - "external_url_title": "Title Instagram shows for the biography link.", - "fbid": "Facebook ID associated with the Instagram account.", - "first_found": "Date an automation first found this profile.", - "followers": "Latest reported follower count.", - "following": "Latest reported number of accounts followed.", - "found_count": "Number of times discovery automations found this profile.", - "full_name": "Display name shown on the Instagram profile.", - "handle": "Instagram username without the @ symbol.", - "has_channel": "Checked when the profile has an Instagram channel.", - "hashtags": "Hashtags extracted from the post caption.", - "highlights_count": "Total story highlight collections reported for the profile.", - "ig_id": "Instagram's internal ID for the account.", - "influencer_key": "Normalized handle linking this row to its profile.", - "is_business": "Checked when Instagram marks the account as a business.", - "is_joined_recently": "Checked when Instagram marks the account as recently joined.", - "is_private": "Checked when the Instagram profile is private.", - "is_professional": "Checked when Instagram marks the account as professional.", - "last_found": "Date an automation most recently found this profile.", - "likes": "Like count reported for this record.", - "measured_at": "Date engagement metrics on this post were last read.", - "measurements_captured": "Number of measurement rows stored for this post.", - "paid_partnership": "Checked when Instagram marks the post as a paid partnership.", - "partner": "Brand named in the post's paid partnership metadata.", - "partner_id": "Partner ID reported for the Instagram profile.", - "platform": "Social network for this profile.", - "plays": "Video plays, including repeat plays, reported for the post.", - "post_link": "Post record linked to this measurement or comment.", - "post_measurements_captured": "Number of distinct post measurements linked to this profile.", - "post_snapshot_key": "Unique ID for this post measurement.", - "post_snapshots_link": "Engagement measurement rows linked to this record.", - "posted_at": "Date the Instagram post was published.", - "posts_captured": "Number of distinct post records captured for this profile.", - "posts_count": "Total posts reported for the Instagram profile.", - "posts_link": "Post records linked to this profile.", - "profile_name": "Profile name returned by the data provider.", - "profile_reads": "Number of profile snapshots stored for this profile.", - "profile_snapshots_link": "Profile snapshot rows linked to this profile.", - "profile_url": "Direct URL to the Instagram profile.", - "pronouns": "Pronouns shown on the Instagram profile.", - "pulled_at": "Date this snapshot was collected.", - "related_accounts": "Accounts Instagram suggests alongside this profile.", - "replies": "Reply count reported for the comment.", - "shortcode": "Instagram shortcode that uniquely identifies the post.", - "source_payload": "Full source response for this record.", - "snapshot_key": "Unique ID for this profile snapshot.", - "source": "Method or provider used to read this profile.", - "tagged_location": "Location tagged on the post; it is not the creator's residence.", - "text": "What the comment says. The commenter's name is not stored as a column.", - "type": "Instagram media type: image, video, or carousel.", - "url": "Direct URL to the Instagram post.", - "verified": "Checked when Instagram marks the profile as verified.", -} - - -#: ⭐ WAVE 29 (R2) — TIKTOK'S OWN DESCRIPTIONS, and the reason this exists rather than reusing the -#: map above is one line of `field_def`: it defaults `description` to -#: `IG_FIELD_DESCRIPTIONS.get(key)`. Nine `ut_tt_*` columns share a KEY with an Instagram column -#: (`shortcode`, `type`, `url`, `verified`, `caption`, `likes`, `comments`, `replies`, -#: `tagged_location`), so a TikTok video's Type column would have shipped explaining "Instagram -#: media type" — a wrong sentence in the header tooltip of a column nobody would think to check. -#: ⚠ A KEY ABSENT HERE GETS NO DESCRIPTION AT ALL, deliberately: silence is honest, and inheriting -#: the Instagram sentence is the failure this map exists to prevent. -TT_FIELD_DESCRIPTIONS = { - "platform": "Network this handle is on.", - "handle": "TikTok @name; the unique account handle.", - "full_name": "Display name shown on the TikTok profile.", - "tt_id": "TikTok's own numeric id for the account.", - "profile_url": "Direct URL to the TikTok profile.", - "bio": "Profile biography text.", - "external_url": "Link in the TikTok bio.", - "verified": "Checked when TikTok marks the account as verified.", - "is_private": "Checked when the account is private.", - "followers": "Follower count at the time of the pull.", - "following": "Accounts this profile follows.", - "posts_count": "Videos published by this account.", - "likes_received": "Total likes this account's videos have received.", - "avg_engagement": "Average engagement rate, stored as a percentage.", - "comment_engagement": "Comment engagement rate, stored as a percentage.", - "like_engagement": "Like engagement rate, stored as a percentage.", - "is_business": "Approximate: set when TikTok flags the account as a commerce user.", - "country_code": "Two-letter country code reported for the account.", - "predicted_lang": "Language TikTok predicts for this account.", - "account_created_at": "When the TikTok account itself was created.", - "region": "Region reported for the account.", - "shortcode": "TikTok's numeric id for the video; unique per post.", - "type": "TikTok post type: video or image.", - "url": "Direct URL to the TikTok post.", - "caption": "Post description text.", - "posted_at": "When the creator published the post.", - "likes": "Likes reported for the post.", - "comments": "Comment count reported for the post.", - "views": "Play count TikTok reports for the video.", - "shares": "Times the post was shared.", - "saves": "Times the post was saved to a collection.", - "video_duration": "Video length in seconds.", - "hashtags": "Hashtags used in the post.", - "tagged_location": "Commerce location reported for the post; it is not the creator's home.", - "influencer_key": "Handle of the account this record belongs to.", - "comment_key": "Unique ID for this comment.", - "commented_at": "When the comment was posted.", - "text": "What the comment says. The commenter's name is not stored as a column.", - "replies": "Reply count reported for the comment.", - "snapshot_key": "Unique ID for this profile snapshot.", - "post_snapshot_key": "Unique ID for this post measurement.", - "pulled_at": "When this measurement was read.", - "enriched_at": "When this row was last enriched.", - "source": "Method or provider used to read this record.", - "source_payload": "Full source response for this record.", -} - - -def field_def(text_key, label, ftype="text", **extra): - """One machine-spawned column definition. - - ⭐ 2026-08-07 — `**extra` carries the per-field DECLARATIONS the preset set needs (`pinned`, - `profile`), and it stays ONE constructor rather than growing a second for "special" fields. - ⛔ Every key passed here must survive `core.user_tables._clean_field` UNCHANGED — `verify_api`'s - W25-1 section pins the drift at ZERO, so a key the validator drops or rewrites turns the whole - preset list red rather than failing quietly ([[default-must-pass-its-own-guard]]). - - ⭐ WAVE 25 — `editRole` IS DECLARED HERE, and adding it closes a bypass rather than adding a - feature. `ut_ensure` writes these dicts STRAIGHT into the `user_tables` bucket, so - `core.user_tables._clean_field` — the single validator every field created through the ordinary - door passes — has never judged a single field this engine spawned. The difference was exactly - one key: `_clean_field` emits `editRole: 'admins'` and this did not, so `_clean_field(f) != f` - for every automation column in the product. - ⚠ NOTHING CHANGES BEHAVIOURALLY — `may_edit_field` asks `editRole == 'everyone'`, and an - absent key was already not that, so the bypass was fail-closed and therefore silent. It is - stated now, and `verify_automation` asserts the equality field-by-field, so the next key - `_clean_field` grows cannot go unnoticed here ([[default-must-pass-its-own-guard]]). - """ - description = " ".join(str(extra.pop( - "description", IG_FIELD_DESCRIPTIONS.get(text_key, "")) or "").split()) - out = {"key": text_key, "label": label, "type": ftype, "source": "overlay", - "editRole": "admins"} - if description: - out["description"] = description - out.update(extra) - return out - - -def tt_field_def(text_key, label, ftype="text", **extra): - """One `ut_tt_*` column. `field_def` with TikTok's description map bound (wave 29, R2). - - ⛔ `description` IS ALWAYS SUPPLIED, even when blank, so `field_def`'s Instagram default can - never be reached from here. An explicit empty string makes `field_def` omit the key, which is - the same shape an undescribed column already has — the point is that the sentence a TikTok - column carries is one somebody wrote about TikTok, or none at all. - """ - extra.setdefault("description", TT_FIELD_DESCRIPTIONS.get(text_key, "")) - return field_def(text_key, label, ftype, **extra) - - -# --------------------------------------------------------------------------------------------- -# DEFINITIONS — validation and the bucket's read/write half -# --------------------------------------------------------------------------------------------- - -def _s(v, n=200): - return str(v if v is not None else "")[:n] - - -#: ⭐ WAVE 26 · C5 / R2 — HOW MANY POSTS ONE PULL MAY KEEP, and the ceiling is the VENDOR'S. -#: -#: ⛔ MEASURED 2026-08-05 and recorded at `_bd_posts_count`: **a Profiles row carries the TOP 12 -#: posts — a cap, not a count.** Asking for more does not fetch more; it just makes the config -#: disagree with what the run can possibly do. -#: ⚠ THIS REPLACES A SILENT `max(1, min(n, 200))` AT TWO CALL SITES, and the clamp was the defect -#: rather than the number: a person typing 50 got a stored 50, a UI that read back 50, and twelve -#: posts — with nothing anywhere saying why. A control that accepts a value it cannot honour is -#: worse than one that refuses it, because the refusal is the only place the ceiling can be -#: taught. So this REFUSES, and the sentence names the cap and who set it. -#: -#: ⭐⭐ 2026-08-09 — RAISED TO 30 (owner: *"move the max limit to 30 posts per profile"*), and the -#: paragraph above needed correcting to do it honestly: **12 was OUR ROUTE'S cap, not the -#: vendor's.** Re-measured the same day — the PROFILE record still returns 12 however many you ask -#: for (asked 40, got 12), but the documented discover-by-url route answered `num_of_posts: 30` -#: with exactly 30 rows in 90 s (23 Reels + 7 Carousels, @theresalearns). The ceiling was a -#: property of the call we happened to make. -#: ⛔ SO THE NUMBER MOVED AND THE CAPTURE MUST FOLLOW. Until `bd_profile_posts` is wired ahead of -#: the views top-up, a `maxPosts` above 12 is honoured by the CONFIG and bounded by the PROFILE -#: read at run time — the exact accept-what-you-cannot-honour shape this constant exists to -#: prevent, now surviving in one place instead of two. It is recorded rather than hidden, and it -#: is why `clean_post_groups` bounds a group by `maxPosts` rather than by 12. -MAX_POSTS_PER_PULL = 30 -DEFAULT_POSTS_PER_PULL = 10 - -#: ⭐⭐ THE WINDOW THE `avg_*_12` PRESET ROLLUPS AVERAGE OVER — ITS OWN CONSTANT, deliberately -#: NOT `MAX_POSTS_PER_PULL`. -#: -#: ⛔ THEY WERE THE SAME NUMBER AND THAT WAS A LATENT BUG, caught by the gate the moment the cap -#: moved: four columns are NAMED `avg_views_12` and LABELLED "Avg views · last 12 posts", so -#: raising the capture cap to 30 silently made every one of them average thirty posts under a -#: label that says twelve. Nobody would have looked at those columns again to check. -#: ⚠ Raising the CAPTURE ceiling and redefining an EXISTING named measure are two different -#: decisions, and only the first one was made. To widen the average, change this constant AND the -#: four keys and labels together — a column whose meaning changes underneath its own name is -#: worse than one that is merely narrow. -AVG_WINDOW_POSTS = 12 - - -#: The post kinds a group filter may name — exactly what `_bd_type` maps a vendor row onto, so a -#: filter cannot ask for a category that can never match a stored row. -POST_TYPES = ("video", "image", "carousel") -#: ⭐ W29-T09 — the words a PERSON reads for those three keys, server-owned for the same reason -#: `KIND_LABELS` and `TRIGGER_LABELS` are. The owner asked for *"the last 12 reels"*, and `video` -#: is the stored key: a client that translated it locally would be a second copy of this -#: vocabulary, free to drift the day a fourth kind appears or a name changes. -#: ⚠ "Reels & videos", not "Reels": `_bd_type` maps every non-image, non-carousel post here, so a -#: label naming only reels would over-promise on a plain video post. -POST_TYPE_LABELS = {"video": "Reels & videos", "image": "Photos", "carousel": "Carousels"} - - -def clean_post_groups(raw, max_posts): - """`(groups, error)` for `config.postGroups` — "last N reels, last M images". - - ⭐ 2026-08-09 (owner: *"not just last 12 but by group also"*). Shape: - `[{"type": "video", "limit": 12}, {"type": "image", "limit": 6}]`. - - ⛔ ABSENT IS OFF, and off must stay the default forever: an enrich action stored before today - carries no such key, and inventing a filter for it would silently start DROPPING posts those - automations have always captured. Empty list and missing are the same answer. - - ⚠ A MALFORMED GROUP IS REFUSED, not dropped — the opposite of `tier`/`noFallback`, and the - difference is legitimate. Those are RETIRED keys that live in stored data, so refusing them - would 400 old automations forever (D-65). This key is NEW: nothing stored can carry a broken - one, so the only way to see one is a person typing it, and a filter that silently ignores the - type you asked for is how you end up paying for reels and storing carousels. - """ - if raw in (None, "", [], {}): - return [], None - if not isinstance(raw, list): - return None, "the post groups have to be a list of {type, limit} entries" - out, seen = [], set() - for item in raw: - if not isinstance(item, dict): - return None, "each post group has to be a {type, limit} entry" - t = str(item.get("type") or "").strip().lower() - if t not in POST_TYPES: - return None, (f"'{t or 'blank'}' is not a post type. Use one of " - f"{', '.join(POST_TYPES)}") - if t in seen: - return None, f"the post groups name '{t}' twice. One limit per type" - seen.add(t) - try: - lim = int(item.get("limit")) - except (TypeError, ValueError): - return None, f"how many {t} posts to keep has to be a whole number" - if lim < 1: - return None, f"a {t} group has to keep at least one post" - # ⛔ BOUNDED BY WHAT THE RUN ACTUALLY BUYS. A group asking for 30 when the pull captures - # 12 is not an error a person can act on — it is a promise the run cannot keep — so it is - # refused HERE, where the number they typed is still on the screen in front of them. - if lim > max_posts: - return None, (f"this enrichment captures {max_posts} posts per profile, so a {t} " - f"group cannot keep {lim}. Raise the post count first") - out.append({"type": t, "limit": lim}) - return out, None - - -def clean_max_posts(raw, default=DEFAULT_POSTS_PER_PULL, submitted=True): - """`(maxPosts, error)` — refuses out-of-range rather than clamping into range. - - ⛔ `submitted=False` CLAMPS INSTEAD OF REFUSING, and that asymmetry is the whole reason this - takes a flag. **Every automation stored before wave 26 carries `maxPosts: 24`** — the old - clamp's default, which this cap now forbids. If an inherited value were refused the same way - a typed one is, every one of those automations would 400 on its next Save **forever**, for a - number nobody on that screen chose or can see. That is D-65's lesson exactly: a validator - that refuses stored data does not protect the user from it, it locks them out of their own - automation. - So: a value the panel SENT is the user asking for it, and gets the sentence. A value merely - INHERITED gets silently brought inside the cap it was already effectively subject to at the - vendor — the run was returning 12 either way ([[default-must-pass-its-own-guard]]). - """ - if raw is None or (isinstance(raw, str) and not raw.strip()): - return default, None - try: - n = int(raw) - except (TypeError, ValueError): - if not submitted: - return default, None - return None, "the post limit must be a whole number" - if not submitted: - return max(1, min(n, MAX_POSTS_PER_PULL)), None - if n < 1: - return None, "keep at least one post per profile, or turn the post capture off" - if n > MAX_POSTS_PER_PULL: - return None, (f"a profile pull returns at most {MAX_POSTS_PER_PULL} posts. That is the " - f"vendor's cap, not ours, so {n} would store the same " - f"{MAX_POSTS_PER_PULL} and read as more") - return n, None - - -def clean_config(kind, raw, previous=None): - """Validate a kind's config. Returns `(config, error)`; error is a user-facing sentence. - - ⚠ THE NODE-SWITCH FLAGS FALL BACK TO `previous` WHEN THE KEY IS ABSENT, and that is a rail - rather than a nicety. `patch` replaces the whole config, and the canvas's config panels do not - edit `postMetrics` / `commentMetrics` / `dryRun` — those are node SWITCHES. So a plain "Save" - from a panel that never knew about them would silently turn off the dry run, or turn ON a - per-post purchase: a save that quietly changes what the automation costs. Absent ⇒ keep; - present ⇒ take it, including `false`. - """ - raw = raw if isinstance(raw, dict) else {} - prev = previous if isinstance(previous, dict) else {} - - def flag(name): - return bool(raw[name]) if name in raw else bool(prev.get(name)) - if kind == "plain": - # A plain automation has no machine step, so it only needs the database its records walk. - # - # ⚠ THE TARGET IS OPTIONAL ON PURPOSE. An automation triggered by `record_updated` on - # `ut_foo` already knows its table from the trigger — `_flow_table` resolves target-else- - # trigger — so requiring a second copy of that fact here would be a mandatory field with - # exactly one legal answer, and a Save that refuses until you retype what you just picked. - # There is no `dryRun`: dry-run means "run the machine step but do not write", and there - # is no machine step to run. - target = _s(raw.get("targetTable"), 60).strip() - if target and not target.startswith(UT_PREFIX): - target = UT_PREFIX + target - return {"targetTable": target, - "targetLabel": _s(raw.get("targetLabel"), 60).strip()}, None - if kind == "scrape_db": - url = _s(raw.get("url"), 2000).strip() - if not url: - return None, "a source URL is required" - try: - guard(url) - except Refused as e: - return None, str(e) - extract = raw.get("extract") if raw.get("extract") in EXTRACTS else "table" - fmap = {} - for k, v in list((raw.get("fieldMap") or {}).items())[:60]: - tk = re.sub(r"[^a-z0-9_]+", "_", _s(v, 60).strip().lower()).strip("_") - if tk: - fmap[_s(k, 120)] = tk[:60] - if not fmap: - return None, "map at least one source column to a field" - key_field = _s(raw.get("keyField"), 60).strip() - if key_field not in fmap.values(): - return None, "the key field must be one of the mapped fields" - target = _s(raw.get("targetTable"), 60).strip() - if target and not target.startswith(UT_PREFIX): - target = UT_PREFIX + target - return {"url": url, "extract": extract, - "tableIndex": max(0, min(int(raw.get("tableIndex") or 0), 50)), - "fieldMap": fmap, "keyField": key_field, - "targetTable": target, - "dryRun": flag("dryRun"), - "targetLabel": _s(raw.get("targetLabel"), 60).strip() or "Scraped table"}, None - if kind == "field_instagram": - table = _s(raw.get("targetTable"), 60).strip() - if not table.startswith(UT_PREFIX): - return None, "an Instagram automation runs against a blank database (ut_*)" - fkey = _s(raw.get("fieldKey"), 80).strip() - if not fkey: - return None, "pick the automation column this run writes into" - # ⛔⛔ `tier` AND `noFallback` ARE ACCEPTED AND IGNORED (wave 28 / R5, contract C2). - # They are read from nothing and written to nothing: a stored definition carrying either - # still SAVES — it simply loses them on the next write — and neither is ever a reason to - # refuse. That asymmetry is D-65's law and it is not squeamishness: refusing an unknown - # key would 400 every automation a tenant stored before this wave, forever, on a screen - # that gives them no way to remove it. Dropping a retired key is a migration; refusing it - # is an outage. - # ⚠ `clean_tier`/`TIERS`/`bd_ready` KEEP THEIR NAMES. They are VENDOR vocabulary - # (`brightdata` is a provider, and `verify_automation` fences the name), not the retired - # USER concept — renaming them would be a second, unrelated change wearing this one's - # justification. - # C5: absent ⇒ keep whatever is stored, like every other switch in this branch — a panel - # that does not edit the post count must not reset it to the default on Save. And an - # INHERITED value is clamped rather than refused; see `clean_max_posts`. - sent = "maxPosts" in raw - max_posts, perr = clean_max_posts( - raw.get("maxPosts") if sent else prev.get("maxPosts"), submitted=sent) - if perr: - return None, perr - return {"targetTable": table, "fieldKey": fkey, - "urlField": _s(raw.get("urlField"), 80).strip(), - # ⛔ THE ONE FLAG THAT MULTIPLIES THE BILL BY THE POST COUNT. Off unless asked - # for: the Profiles row carries post IDENTITY for free but NO engagement - # (measured 2026-08-05), so likes/comments cost one extra vendor record PER POST. - "postMetrics": flag("postMetrics"), - # The full Comments dataset can bill many rows per post. It is an explicit, - # independent opt-in and never follows the per-post switch automatically. - "commentMetrics": flag("commentMetrics"), - "dryRun": flag("dryRun"), - "maxPosts": max_posts}, None - # ⭐ WAVE 29 (D-9 / R1) — BOTH discovery kinds share this branch, because they ask the vendor - # the same question of two different corpora: a record ceiling, a predicate list, a join word - # and where to write. ⛔ THE ONLY DIFFERENCE IS THE DEFAULT TABLE, and it is resolved from the - # kind rather than hard-coded — a TikTok search falling back to `ut_ig_profile` would write - # TikTok rows into the Instagram family, which is precisely what R2's parallel family exists - # to prevent, and it would do it silently. - if kind in DISCOVERY_KINDS: - # ⭐ WAVE 30 · T05 — the inline tuple became the named one, and the two defaults below now - # come from `discovery_facts` rather than being spelled out here. They were correct; they - # were also the FIFTH copy of "which table does this kind write to", and the other four - # were the ones wave 29 forgot to update. - _, _disc_table, _disc_label, _ = discovery_facts(kind) - limit = int(raw.get("recordsLimit") or 0) - if limit < 1: - return None, ("say how many profiles to fetch. An UNBOUNDED discovery query is the " - "one shape the vendor refuses outright (NOT_ENOUGH_FUNDS)") - if limit > BD_MAX_RECORDS: - return None, f"a single discovery run may ask for at most {BD_MAX_RECORDS} profiles" - # ⚠ THE JOIN IS RESOLVED BEFORE THE PREDICATES ARE JUDGED, because it is part of what - # makes them broad or narrow (see `narrowing_refusal`). Validating them first and reading - # the operator afterwards is how the OR hole survived: the guard was handed the branches - # and never told they were a union. - join = "or" if str(raw.get("operator") or "").lower() == "or" else "and" - # ⭐ WAVE 32 · T46 (D-167) — AND THE KIND GOES WITH THEM. `clean_config` already knows which - # corpus this automation searches; passing it is what stops a `discover_tiktok` being built - # on the 16 Instagram fields TikTok's dataset does not carry — a search that does not error, - # returns nothing, and reads as "no such creators exist" after the money is spent. - preds, perr = clean_predicates(raw.get("predicates"), join, kind) - if perr: - return None, perr - target = _s(raw.get("targetTable"), 60).strip() or _disc_table - if not target.startswith(UT_PREFIX): - target = UT_PREFIX + target - seed, serr = clean_seed(raw.get("seed") if "seed" in raw else prev.get("seed")) - if serr: - return None, serr - return {"recordsLimit": limit, "predicates": preds, - "operator": join, - "targetTable": target, - "targetLabel": _s(raw.get("targetLabel"), 60).strip() or _disc_label, - # C6/R5: WHERE the conditions above came from, when they were derived. Stored so - # the surface can say "these were filled in from the 'Florists' view, over 12 - # records" instead of presenting them as if somebody typed them. - "seed": seed, - "dryRun": flag("dryRun")}, None - return None, f"unknown automation kind {kind!r}" - - -def clean_seed(raw): - """C6: validate `config.seed`. Returns `({}, None)` when there is none — a discovery filter - somebody typed by hand has no seed, and that is the ordinary case. - - ⚠ `derived` AND `basis` ARE STORED AS PROVENANCE, NOT AS A SECOND FILTER. R5 is explicit that - the derived conditions are "written into `config.predicates` as ordinary conditions — it is a - filling-in, not a parallel filter", so the RUN never reads this bag: it reads `predicates`, - like every other search. Keeping it means the surface can say where those rows came from, and - a user editing them freely is exactly what is supposed to happen. - """ - if raw in (None, "", {}): - return {}, None - if not isinstance(raw, dict): - return None, "the seed must be an object" - source = _s(raw.get("source"), 12).strip().lower() - if source not in SEED_SOURCES: - return None, f"{source or 'that seed'!r} is not one of: " + ", ".join(SEED_SOURCES) - table = _s(raw.get("table"), 60).strip() - if table and not table.startswith(UT_PREFIX): - return None, f"a seed reads a blank database (ut_*). {table!r} is not one" - basis = raw.get("basis") if isinstance(raw.get("basis"), dict) else {} - derived, _err = clean_predicates(raw.get("derived") or [], "and") - return {"source": source, "table": table, "id": _s(raw.get("id"), 80).strip(), - # ⛔ A MALFORMED `derived` IS DROPPED, NEVER A REFUSAL. This bag is a RECORD of what - # was suggested; the conditions that matter are already in `predicates` and were - # validated there. Refusing a Save because a stored provenance note aged badly would - # block the user from editing the very filter it describes. - "derived": derived or [], - "basis": {"rows": max(0, int(basis.get("rows") or 0)), - "fields": [f for f in (basis.get("fields") or []) - if isinstance(f, dict)][:SEED_MAX_PREDICATES], - "related": basis.get("related") - if isinstance(basis.get("related"), dict) else {}, - "note": _s(basis.get("note"), 200)}}, None - - -def clean_schedule(raw, previous=None): - """Validate `{cron, enabled}` and stamp `enabledAt` on the OFF→ON edge (the anchor `is_due` - measures from — without it, enabling a daily job fires it immediately).""" - raw = raw if isinstance(raw, dict) else {} - cron = _s(raw.get("cron"), 120).strip() or "0 6 * * *" - parse_cron(cron) # raises -> the route answers 400 - enabled = bool(raw.get("enabled")) - prev = previous or {} - out = {"cron": cron, "enabled": enabled} - if enabled: - out["enabledAt"] = (prev.get("enabledAt") if prev.get("enabled") else None) or _iso() - return out - - -# ── WAVE 23 · C5 — the ENDING, and the cycle counter that makes a loop countable. ───────────── -# The owner's words: "we can make this automation a loop, so the ending should always be defined, -# either it ends somewhere deterministic like Closed/Failed, or it goes to reset automatically, -# or the user have to click a button to reset, or after a certain amount of time it can -# automatically reset to first cycle." -# -# ⛔ `terminal` IS THE DEFAULT and every existing automation gets it, because a stored definition -# that predates this field must not start moving records on its own the day the code ships. A -# loop is a thing somebody turns on. -def clean_flow(raw, previous=None, notes=None, rt=None): - """Validate the builder's ordered action list. `(flow, error)`. - - ⚠ `rt` is the tenant wall for gated action kinds (W35-T35 / C8) and is simply forwarded. - """ - raw = raw if isinstance(raw, dict) else {} - prev = previous if isinstance(previous, dict) else {} - actions, err = clean_actions( - raw.get("actions") if "actions" in raw else prev.get("actions"), notes=notes, rt=rt) - if err: - return None, err - return {"actions": actions}, None - - -#: AMENDMENT A1 — what the `ig_profile_match` kind-flip seeds as `recordsLimit` when the -#: definition carries none. -#: -#: ⚠ 25 BECAUSE THE INPUT SAYS 25 (2026-08-06). It was 10 — the size wave 21 proved live at -#: $0.15 — while `AutomationDetail` initialises its own box to 25, and the two never met: the -#: flow node read "Up to 10 profiles · about $0.025" beside a field reading 25, on a freshly -#: created automation. Two numbers for one fact, in the panel, before anybody had typed -#: anything. **Read off the screenshot; every assertion in the battery was green.** -#: -#: This is the same species as the default-versus-guard split fixed the same day: a value -#: decided in one file and a value decided in another, describing the same thing. Aligning the -#: constants closes the only window in which they can disagree — the seed — because every later -#: state comes from the stored config. -DISCOVER_SEED_RECORDS = 25 - - -def discovery_seed_spec(kind): - """-> (default table, enrich action kind) for a discovery KIND. - - ⭐⭐ WAVE 30, OWNER REPORT 2026-08-12, verbatim: *"'When a Tiktok profile fits a criteria' - should ALWAYS have a 'Create record' EXACTLY like the Instagram one … THE ONLY DIFFERENCE IS - THE COLUMNS AND DATA SCHEMA"*. They were right, and the seeding path was the one family of - sites this wave widened everywhere ELSE: `clean_definition` planted step 1 and step 2 only when - the trigger was `ig_profile_match`, so a TikTok search stored `flow.actions = []` and had - nowhere to put what it found — MEASURED live against `auto_1` (Instagram: create_record + - enrich) versus a fresh TikTok discovery (empty). - - ⭐ THIS IS A LOOKUP, NOT A SECOND SEEDER, and that is the whole point of the ruling. One - builder plants both platforms' steps; the only things that vary are the table the record lands - in and which enrich action reads it — literally "the columns and data schema". A forked - `_ensure_tt_action` would start identical and drift, which is the failure `ENRICH_KINDS` and - `DISCOVERY_KINDS` were introduced to prevent one screen up. - - ⚠ Resolved in a FUNCTION rather than a module-level dict because `DISCOVER_TABLE` is defined - far below this line; a dict here would raise at import. - """ - if kind == "discover_tiktok": - return TT_PROFILE_TABLE, "enrich_tiktok" - return DISCOVER_TABLE, "enrich_instagram" - - -def _ensure_ig_action(flow_raw, table, enrich_kind="enrich_instagram"): - """Create record is PERMANENT step 1 on a DISCOVERY trigger — it cannot be deleted or moved. - - ⚠ THE NAME IS INSTAGRAM'S AND THE BEHAVIOUR IS BOTH PLATFORMS' (wave 30). Renaming it would - churn six gate references for no behaviour change; `enrich_kind` is what makes it general, and - it defaults to Instagram's so every pre-existing caller keeps its exact meaning. - - ⭐ OWNER RULING 2026-08-06, AND IT REVERSES WAVE 24's LAW 3. That law seeded this action once, - on the edge into the trigger, and ended "deleting it is their call, not a refusal" — with a - comment warning that re-seeding on every clean would be "a control that will not take no for - an answer". The owner's answer: *"should ALWAYS have a 'Create record' Step 1, that can't be - deleted, because the nature of that automation is that it needs to first create a record in a - database somewhere from the list of profiles to fetch."* - - Which is right, and the earlier reasoning had the category wrong: a flow whose TRIGGER - produces rows has nowhere to put them until something writes them, so an Instagram search - with no Create record is not a customised automation — it is a search whose results are - discarded. That is not a preference to respect. - - ⚠ IT KEEPS THE USER'S EDITS. Presence and POSITION are guaranteed; the table it writes to and - the values it maps are theirs. An existing `create_record` further down is MOVED to the front - rather than duplicated — re-seeding a second one would quietly double every run's writes. - """ - flow = dict(flow_raw or {}) if isinstance(flow_raw, dict) else {} - actions = [a for a in (flow.get("actions") or []) if isinstance(a, dict)] - if actions and actions[0].get("kind") == "create_record": - return _pin_unique(_ensure_enrich_step( - flow_raw if isinstance(flow_raw, dict) else flow, enrich_kind)) - at = next((i for i, a in enumerate(actions) if a.get("kind") == "create_record"), -1) - if at > 0: - actions.insert(0, actions.pop(at)) - else: - # The seed must be a config `clean_actions` ACCEPTS, or the builder shows a red banner - # and no card: the wave-23 header records four kinds that shipped with illegal seeds and - # did exactly that. `create_record` needs a ut_-prefixed table and at least one value. - actions.insert(0, { - "id": "act_1", "kind": "create_record", "enabled": True, "when": None, - # `config.label` follows the `review` action's precedent in `_clean_action_config` — - # an action naming itself, inside the untyped config bag, so no shared TS type changes. - "config": {"table": str(table or DISCOVER_TABLE), - "label": "Save the profile", - # C5: the picture and the real write finally agree — see `_pin_unique`. - "uniqueOn": "handle", - "values": {"handle": "{{handle}}"}}, - }) - flow["actions"] = actions - return _pin_unique(_ensure_enrich_step(flow, enrich_kind)) - - -def _ensure_enrich_step(flow_raw, enrich_kind="enrich_instagram"): - """⭐ 2026-08-07 (owner ruling) — ENRICH IS STEP 2 ON A DISCOVERY SEARCH. - - ⚠ WAVE 30: `enrich_kind` selects the network. Instagram's is the default so every pre-existing - caller means exactly what it meant before; TikTok passes `enrich_tiktok` and gets the identical - step shape, which is the owner's *"only the columns and data schema differ"*. - - Owner: *"make it default that this enrichment action is Step 2 always, and under Config of - Step 2 … we can have a toggle on or off."* Which is the same shape as step 1's ruling and for - the same reason: a search that finds profiles and never reads them has done half a job. The - difference is the control — step 1 is permanent because a flow without it discards its - results, while this one is permanent because the TOGGLE is how you turn it off. Deleting and - disabling would be two ways to say one thing, and only one of them survives a re-save. - - ⚠ SEEDED ON, AND ON THE FREE RUNG. `tier: "anonymous"` costs nothing, so a search that gains - this step by upgrading does not quietly start spending; switching the Source to the paid - provider is an explicit choice a person makes in front of the sentence that names the cost. - ⚠ THE COOLDOWN IS SEEDED ON TOO (30 days). On a table nobody has enriched it changes nothing — - a blank `enriched_at` is never "recent" — and the moment there IS history it stops the flow - re-buying the same profile nightly. Off-by-default would make the expensive behaviour the - accident. - - ⛔ THE EXISTENCE TEST WALKS THE FORKS (`walk_actions`). A person who moved enrichment inside an - If/then branch has one; seeding a second at the top level would enrich twice and bill twice, - which is the duplication `apply_actions` already paid for once with the pinned step 1. - """ - flow = dict(flow_raw or {}) if isinstance(flow_raw, dict) else {} - actions = [a for a in (flow.get("actions") or []) if isinstance(a, dict)] - if any(a.get("kind") == enrich_kind for a in walk_actions(actions)): - return flow_raw if isinstance(flow_raw, dict) else flow - seeded = list(actions) - # Index 1 — after the pinned Create record, because there is nothing to enrich until the - # profiles have been written as records. `insert` past the end is a plain append, so a flow - # with only step 1 lands this at the end, which IS step 2. - seeded.insert(1, { - "id": "act_enrich", "kind": enrich_kind, "enabled": True, "when": None, - "config": {"postMetrics": False, "commentMetrics": False, - "dryRun": False, "maxPosts": DEFAULT_POSTS_PER_PULL, - "fromView": "", "sortField": DEFAULT_ENRICH_SORT, "sortDir": "desc", - "limit": DEFAULT_ENRICH_LIMIT, "skipRecent": True, - "skipRecentDays": DEFAULT_ENRICH_COOLDOWN_DAYS}, - }) - return {**flow, "actions": seeded} - - -#: The key the discovery writer really upserts on. Both discovery runners key candidates on -#: `candidate_key(platform, handle)`; `handle` is the half a person can see and the half this action's -#: values carry, which is why the PINNED card shows that rather than the pair — `platform` is -#: supplied by the runner, never typed by a person. -#: ⛔ DEBT D-73 (closed wave 29): this note used to name wave 22's C6 compound — the one that -#: paired the handle with its finder — as the live key. Wave 26 · R4/R5 retired it: the identity is -#: the pair above and the TENANT is the unit, while `created_by` survives as an informational -#: "Found by" stamp that no run may branch on. -#: ⚠ The retired pair is DESCRIBED here and not reproduced, deliberately: a gate asserts this note -#: names the key `_ck` actually takes, and a verbatim quotation of the wrong one reads to that gate -#: exactly like the defect. A stale comment on correct code is how the next session reintroduces a -#: bug with a rationale attached. -IG_PINNED_UNIQUE = "handle" - - -def _pin_unique(flow): - """C5: keep `uniqueOn: "handle"` on the PINNED step 1 across every save. - - ⭐ WHY IT IS RE-STAMPED RATHER THAN MERELY SEEDED. `_clean_action_config` has no `previous` — - the builder posts the whole action list on every save — so a client that omitted the key would - silently reset it to `""`, i.e. back to append. The pinned card is a PICTURE of the engine's - own upsert (`apply_actions` skips it at runtime, see the note there), so the picture claiming - "append" while the engine upserts is exactly the surface-disagrees-with-the-engine defect this - module refuses everywhere else. - - ⛔ AND IT CANNOT MINT A CONFIG THE GUARD REFUSES — HARD RULE 11, which is the whole reason - this is a function and not one line. `_clean_action_config` refuses a `uniqueOn` the action - does not write, so the stamp is applied ONLY when `handle` is among the action's own values. - A user who remaps the card to write different columns keeps their edit and still saves; the - alternative — stamping unconditionally — is a product-seeded default that its own validator - would 400, which is precisely the post-W24 hotfix this rule exists because of. - - ⚠ NON-MUTATING, and that is load-bearing rather than tidiness. `clean_definition` passes - `prev.get("flow")` here when a PATCH carries no flow of its own — that is the STORED - definition's own dict, so writing into it would edit the live store object in memory, before - (and regardless of) any commit. Every touched level is copied instead. - """ - if not isinstance(flow, dict): - return flow - acts = flow.get("actions") - if not isinstance(acts, list) or not acts or not isinstance(acts[0], dict): - return flow - first = acts[0] - if first.get("kind") != "create_record": - return flow - cfg = first.get("config") - if not isinstance(cfg, dict) or IG_PINNED_UNIQUE not in (cfg.get("values") or {}): - return flow - if str(cfg.get("uniqueOn") or "").strip(): - return flow - return {**flow, - "actions": [{**first, "config": {**cfg, "uniqueOn": IG_PINNED_UNIQUE}}] + acts[1:]} - - -#: The two steps `_ensure_ig_action` / `_ensure_enrich_step` plant, as `(kind, id)`. Named here so -#: the seeder and the un-seeder cannot disagree about what "seeded" means. -#: ⚠ WAVE 30 — `enrich_tiktok` shares `act_enrich`. The map is keyed by KIND, and only one enrich -#: step is ever seeded per flow (the network follows the trigger), so the id cannot collide. -_IG_SEED_IDS = {"create_record": "act_1", "enrich_instagram": "act_enrich", - "enrich_tiktok": "act_enrich"} - - -def _is_untouched_ig_seed(action, table, enrich_kind="enrich_instagram"): - """Is this action still EXACTLY what the Instagram trigger planted? (wave 27 item 15) - - ⛔ THE COMPARISON IS AGAINST A FRESHLY BUILT SEED, not against a list of remembered keys, and - that is the only version that stays true: `_ensure_ig_action` and `_ensure_enrich_step` build - the same dicts one screen above, so a step gaining a config key next wave gains it here too. - A remembered key list would quietly start calling every seeded action "edited". - - ⛔ AND THE FRESH SEED GOES THROUGH `clean_actions` FIRST, which cost a red check to learn. The - stored action has been cleaned — `_clean_action_config` normalises it and adds the keys the - kind declares (`profileField: ""` on an enrich step, for one) — so comparing against the RAW - dict `_ensure_enrich_step` writes reports every seeded enrich step as "edited by a person", - and the un-seeding silently never fires for it. Comparing cleaned against cleaned is the only - version where the two sides are the same kind of object. - - ⚠ `enabled` IS DELIBERATELY NOT PART OF THE TEST for the enrich step. Its ruling says the - TOGGLE is how you turn it off — so a person who switched it off has expressed an opinion about - a step they still want, and an off seed is still a seed. - """ - if not isinstance(action, dict): - return False - kind = str(action.get("kind") or "") - if _IG_SEED_IDS.get(kind) != str(action.get("id") or ""): - return False - seeded, _seed_err = clean_actions( - (_ensure_ig_action({"actions": []}, table, enrich_kind).get("actions") or [])) - fresh = {a.get("kind"): a for a in (seeded or [])} - seed = fresh.get(kind) - if not seed: - return False - cfg, seed_cfg = dict(action.get("config") or {}), dict(seed.get("config") or {}) - if kind in ENRICH_KINDS: - cfg.pop("enabled", None) - seed_cfg.pop("enabled", None) - return cfg == seed_cfg and (action.get("when") or None) is None - - -def _drop_ig_seeds(flow_raw, table, enrich_kind="enrich_instagram"): - """Strip the trigger's own seeded steps, keeping any the person has since made theirs. - - ⭐⭐ WAVE 27 ITEM 15 (owner) — *"stale pinned create_record on trigger change"*. THE COMMENT - THAT USED TO SIT AT THE KIND FLIP ARGUED THE OPPOSITE and is rewritten there; it read: *"it - does not delete the seeded actions … a create_record the person has since re-pointed at their - own table with their own values is THEIR action now"*. That reasoning is still correct and is - exactly what this function preserves — but it was applied to EVERY seeded action, including - the ones nobody had ever opened, and the result is what the owner reported: switch an - Instagram search to Manual and you are left holding a "Save the profile" step writing - `{{handle}}` into `ut_ig_candidates` on a flow that no longer produces handles. That is not - somebody's work being protected; it is the machine's own leftover, pointed at a table the - automation has nothing to do with any more. - - ⛔ SO THE TEST IS "DID ANYBODY TOUCH IT", NOT "WAS IT SEEDED" — the same guard shape as - `_vestigial_name_field` and `_retired_tracked_field`, and for the same reason: this is a - branch that destroys something, so the conservative half is the load-bearing half. - """ - flow = dict(flow_raw or {}) if isinstance(flow_raw, dict) else {} - actions = [a for a in (flow.get("actions") or []) if isinstance(a, dict)] - kept = [a for a in actions if not _is_untouched_ig_seed(a, table, enrich_kind)] - if len(kept) == len(actions): - return flow_raw - return {**flow, "actions": kept} - - -def ig_action_pinned(defn, index): - """Is this action the one that cannot be removed? Derived, never stored — a stored `pinned` - flag is a second copy of the rule, and the copy is what a hand-written PATCH omits.""" - # ⭐ WAVE 30 — BOTH discovery kinds. It read `== "discover_instagram"`, so TikTok's step 1 (once - # seeded) would have rendered as an ordinary draggable, deletable card: the same rule the owner - # asked for "EXACTLY", applied to only one network. - return (defn or {}).get("kind") in DISCOVERY_KINDS and index == 0 - - -def clean_definition(raw, previous=None, username="", notes=None, rt=None): - """Whole-definition validation. Returns `(defn, error)`.""" - raw = raw if isinstance(raw, dict) else {} - prev = previous or {} - # ⭐ WAVE 24 · C-TRIG — THE TRIGGER IS RESOLVED FIRST NOW, because law 1 makes it the thing - # that CHOOSES the kind. Reading `raw["trigger"]["key"]` here instead would be a second - # reader of the trigger vocabulary, free to disagree with `clean_trigger` about which keys - # exist and which are refused. Gate-visible consequence, recorded in AMENDMENT A1: a payload - # invalid in BOTH its trigger and its config now answers with the trigger's sentence. - trigger, terr = clean_trigger(raw.get("trigger") if "trigger" in raw - else prev.get("trigger"), prev.get("trigger")) - if terr: - return None, terr - # C-TRIG law 2 (wave 24): a definition that names no kind is a `plain` one. Before R6 this - # refused with "unknown automation kind ''" — correct while a wizard always sent one, and a - # dead end the moment the wizard was deleted and the client's create body became {name}. - kind = _s(raw.get("kind") or prev.get("kind"), 40) or DEFAULT_KIND - drop_ig_seeds = False - if (trigger or {}).get("key") == "ig_profile_match": - # LAW 1: the trigger IS how `discover_instagram` gets chosen now. Unconditional rather - # than "when the kind is unset" — a definition whose trigger says Instagram-discovery and - # whose kind says something else is not a preference to respect, it is two halves of one - # answer disagreeing, and the trigger is the half the person actually picked. - kind = "discover_instagram" - elif (trigger or {}).get("key") == "tiktok_profile_match": - # ⭐ WAVE 29 (D-9 / R1) — LAW 1, TIKTOK'S HALF. Same rule, same reason: the trigger is how - # `discover_tiktok` gets chosen, and it is unconditional for the same reason the Instagram - # arm is — a definition whose trigger says TikTok discovery and whose kind says something - # else is two halves of one answer disagreeing. - kind = "discover_tiktok" - elif (kind == "discover_tiktok" - and ((prev.get("trigger") or {}) or {}).get("key") == "tiktok_profile_match" - and (trigger or {}).get("key") != "tiktok_profile_match"): - # ⛔⛔ LAW 1'S INVERSE, AND ITS ABSENCE ON THE INSTAGRAM SIDE WAS A MONEY BUG (see the arm - # below): with nothing to flip the kind BACK, switching the trigger to Manual left - # `RUNNERS[kind]` pointing at the discovery runner, so **Run now fired a PAID corpus - # search on a flow the person had just made manual.** Shipped here in the same change as - # law 1 rather than discovered the same way twice. - # ⚠ THE TEST IS THAT THE TRIGGER **MOVED** — comparing the RESOLVED key against the - # PREVIOUS one. Asking `"trigger" in raw` would fire on every empty patch, because - # `patch()` builds its raw as `dict(prev)` plus the caller's keys. - kind = DEFAULT_KIND - # ⭐ WAVE 30 — un-seed on the way out, exactly as the Instagram arm below does. This line - # was ABSENT and harmless for as long as TikTok had no seeds to leave behind; the moment - # the seeder above learned TikTok, its absence became wave-27 item 15's defect on the other - # network — a flow switched to Manual still carrying a "Save the profile" step nobody - # planted deliberately. Found by widening the seeder and asking what else assumed only - # Instagram could have seeds. - drop_ig_seeds = True - elif (kind == "discover_instagram" - and ((prev.get("trigger") or {}) or {}).get("key") == "ig_profile_match" - and (trigger or {}).get("key") != "ig_profile_match"): - # ⭐⭐ 2026-08-07 (owner report) — LAW 1 HAS AN INVERSE, AND ITS ABSENCE WAS A MONEY BUG. - # - # Law 1 above flips the kind TO `discover_instagram` when the trigger says Instagram - # discovery. Nothing flipped it BACK. So `kind` was inherited from `prev` forever: switch - # the trigger to Manual and the automation stayed `discover_instagram`, which means - # - # · `RUNNERS[kind]` is still `run_discover_instagram`, so pressing **Run now** on a flow - # the person had just made MANUAL fired a PAID Bright Data corpus search; - # · `ig_action_pinned` keys on the kind, so the seeded "Create record" stayed - # UNDELETABLE — the owner's report, verbatim: *"When a trigger change from the - # instagram trigger, the 'always first' create record just stuck there"*. The server - # would have accepted the delete; the CLIENT hid the control, because both halves ask - # the kind and the kind was lying. - # - # ⛔ THE TEST IS THAT THE TRIGGER **MOVED**, and the first version of this got it wrong in - # a way worth recording. It asked `"trigger" in raw` — but `patch()` builds its raw as - # `merged = dict(prev)` plus the caller's keys, so **`"trigger"` is present on EVERY - # patch**, and an empty `patch(rt, id, {})` flipped a discovery automation to `plain`. - # That turned five `section_bd` checks red and crashed the suite on a `StopIteration` - # three sections later. Comparing the RESOLVED key against the PREVIOUS one is the honest - # question: did this automation stop being an Instagram search? - # - # ⚠ NARROW ON PURPOSE, twice over. It fires only when the automation WAS on the Instagram - # trigger — a `discover_instagram` created with an explicit kind and some other trigger is - # somebody's deliberate state, not a mistake to correct. And only FROM - # `discover_instagram`: `scrape_db` and `field_instagram` are surviving kinds whose - # triggers are their own business, and clobbering them would retire them by accident. - # - # ⭐⭐ WAVE 27 ITEM 15 — IT NOW DROPS THE SEEDS NOBODY TOUCHED, and the paragraph this - # replaces argued the other way, so here is why it was half right. It read: *"it does not - # delete the seeded actions … a create_record the person has since re-pointed at their own - # table with their own values is THEIR action now, and quietly destroying it is the silent - # data loss this module refuses everywhere else."* Every word of that is still true and is - # exactly what `_is_untouched_ig_seed` protects. What it got wrong was applying the - # protection to actions NOBODY HAD EVER OPENED: the owner's report is a flow switched to - # Manual still carrying a "Save the profile" step writing `{{handle}}` into - # `ut_ig_candidates` — the machine's own leftover on a flow that no longer produces - # handles, unpinned but still there, still runnable, and still pointed at a table that has - # nothing to do with this automation. - # ⚠ APPLIED BELOW, at the flow, not here: `flow_raw` is not resolved yet at this line. - kind = DEFAULT_KIND - drop_ig_seeds = True - if kind not in KINDS: - return None, f"unknown automation kind {kind!r}" - # ⛔ DEBT D-65 — THE REFUSAL THAT BELONGS HERE IS NOT SHIPPED, AND THAT IS A DECISION. - # D-65 offers two exits for `field_instagram`: convert a surviving definition to a `plain` - # flow carrying `enrich_instagram`, or REFUSE it here with a sentence naming the replacement. - # The refusal was built and then REVERTED, because W24/R6 deliberately made `patch()` the way - # a retired kind legally comes into existence — `create` refuses, `patch` accepts, and the two - # live automations save through this function every time their owner edits them. The gate's - # own fixture helper says so in the strongest terms available: *"If R6's refusal ever moved - # from `create` into `clean_definition` (the tempting simplification), this helper would go - # red across a dozen sections, which is the alarm that change deserves."* It did, and the - # alarm worked. - # ⚠ WHAT SHIPPED INSTEAD is the half that costs nothing and was the actual complaint: every - # door that DOES refuse now names the replacement (`RETIRED_KIND_REPLACEMENT`), so nobody - # meets "unknown automation kind" for a decision made on purpose. The rest of D-65 is an - # owner-visible behaviour change — either new `field_instagram` mints stop working, or stored - # ones are rewritten under their owner — and that is a ruling, not a refactor. - name = " ".join(_s(raw.get("name") or prev.get("name") or "", MAX_NAME).split()) - if not name: - return None, "name the automation" - cfg_raw = raw.get("config") if isinstance(raw.get("config"), dict) else prev.get("config") - if kind in DISCOVERY_KINDS and prev.get("kind") != kind: - # ⭐⭐ WAVE 30 · T04 — WIDENED FROM `discover_instagram` TO BOTH DISCOVERY KINDS, AND THIS - # ONE LINE WAS THE WHOLE OF THE OWNER'S ITEM 3. Picking "When a TikTok profile fits a - # criteria" 400'd on the very first Save with *"say how many profiles to fetch — an - # UNBOUNDED discovery query is the one shape the vendor refuses outright - # (NOT_ENOUGH_FUNDS)"*, and the trigger was therefore never STORED, which is what the - # owner saw as "the Trigger won't load". - # ⛔ INSTAGRAM WAS NEVER SURVIVING ON ITS OWN MERITS: `AutomationDetail.buildConfig` sends - # no `recordsLimit` for EITHER platform on a fresh automation (its `kind` is still `plain` - # at that moment, so it falls through to `return { targetTable }`). IG worked only because - # this seed caught it. So the defect was never "TikTok is missing something Instagram has" - # — it was one hard-coded string in the single line that rescues both. - # ⚠ `prev.get("kind") != kind` is the EXACT generalisation of the old - # `prev.get("kind") != "discover_instagram"`, not a loosening: it still fires only on a - # kind FLIP, so a later save that carries a real limit is not re-seeded (asserted). - # AMENDMENT A1: the kind FLIP seeds the one field `clean_config` insists on, or the very - # first Save after picking this trigger 400s — against the stored-inert-with- - # `configured:false` pattern the whole picker is built on (see `clean_trigger`'s A3 note). - # ⛔ THE REFUSAL ITSELF IS UNTOUCHED: an explicit 0 still gets its sentence. That guard - # exists because an unbounded query is the one shape the vendor refuses outright, and - # coercing a blank into a number would be exactly the silent widening it protects against. - # Seeding cannot spend — `clean_schedule` defaults `enabled` False, so nothing runs until - # a person presses Run now or arms the schedule. - cfg_raw = dict(cfg_raw or {}) - if not _ig_int(cfg_raw.get("recordsLimit")): - cfg_raw["recordsLimit"] = DISCOVER_SEED_RECORDS - config, err = clean_config(kind, cfg_raw, prev.get("config")) - if err: - return None, err - if trigger: - # A2: re-ask completeness now that the config is validated — `ig_profile_match`'s own - # configuration IS the config, and `clean_trigger` could not see it. - trigger["configured"] = _trigger_configured(trigger, config) - try: - schedule = clean_schedule( - raw.get("schedule") if isinstance(raw.get("schedule"), dict) - else prev.get("schedule"), prev.get("schedule")) - except ValueError as e: - return None, str(e) - # (`clean_trigger` ran at the top — law 1 needs the trigger before the kind.) - flow_raw = raw.get("flow") if "flow" in raw else prev.get("flow") - # ⭐ EVERY CLEAN, NOT ONLY THE EDGE (owner ruling 6 — see `_ensure_ig_action`). The edge-only - # call was what made the action deletable; running it on every save is what makes step 1 - # permanent, and it is deliberate rather than a widened condition nobody noticed. - # ⭐⭐ WAVE 30 (owner, 2026-08-12) — BOTH DISCOVERY TRIGGERS SEED, not just Instagram's. This - # single `==` was the whole of *"why is it not copied EXACTLY"*: a TikTok search stored an - # EMPTY flow, so the product's own rule — a search that finds profiles must have somewhere to - # put them — held for one network and not the other. - if (trigger or {}).get("key") in DISCOVERY_TRIGGER_KIND: - _seed_table, _seed_enrich = discovery_seed_spec(kind) - flow_raw = _ensure_ig_action( - flow_raw, config.get("targetTable") or _seed_table, _seed_enrich) - elif drop_ig_seeds: - # ITEM 15: the trigger just STOPPED being a discovery one. Un-seed what the trigger - # planted and nobody has since made theirs — measured against the table the seeds were - # planted for, which is the PREVIOUS config's, not the one being saved. - # ⚠ And against the PREVIOUS kind's enrich action, for the same reason: the seeds to strip - # are TikTok's if that is what was planted, and asking "is this Instagram's seed?" of a - # TikTok flow answers no and silently leaves the leftover behind. - _prev_table, _prev_enrich = discovery_seed_spec(prev.get("kind")) - flow_raw = _drop_ig_seeds( - flow_raw, (prev.get("config") or {}).get("targetTable") or _prev_table, _prev_enrich) - flow, ferr = clean_flow(flow_raw, prev.get("flow"), notes=notes, rt=rt) - if ferr: - return None, ferr - # ⭐ WAVE 30 — widened with the seeder above: TikTok now HAS a pinned step 1, so the rule that - # its `config.table` is authoritative has to reach it, or the two fields that name one database - # would disagree on exactly the network that just gained the card. - if (trigger or {}).get("key") in DISCOVERY_TRIGGER_KIND: - # ⭐ WAVE 25 · R2 — THE CREATE RECORD ACTION'S `config.table` IS AUTHORITATIVE, and - # `config.targetTable` FOLLOWS IT. Two fields have been naming the same database since - # wave 24: the pinned step 1 says where profiles are written, and the config says where - # the automation's records live — and for a discovery flow those are the same database by - # construction. When they disagreed, every surface picked a different winner (the canvas - # Write node read the config, the actual write read the action), which is a bug you can - # only see by comparing two panels. - # - # ⛔ THE ACTION WINS BECAUSE IT IS THE ONE A PERSON EDITS. R2b puts the preset list on the - # action's own configuration, so the table named there is the one they were looking at. - # - # ⚠ THIS IS ALSO THE MIGRATION, and it is a migration in the shape laws 4/5 established: - # a stored definition carrying two different values is reconciled on its next clean, - # silently and exactly once, because nothing writes the losing value back. A refusal here - # would 400 every Save of a definition that was legal yesterday. - # ⭐⭐ WAVE 27 ITEM 11 — WHICHEVER SIDE MOVED WINS, and R2's "the action wins" is now the - # TIE-BREAK rather than the whole rule. The owner's report: change the database on the - # TRIGGER, save, and the picker springs back to the old one with no message. The cause is - # the branch below read unconditionally — the pinned action still named yesterday's table, - # so it overwrote a change the person had just made in front of it, silently, every time. - # - # ⛔ R2 IS NOT REVERSED, and the distinction is which QUESTION the code asks. R2 settled - # "when the two disagree, whose value is real?" — the action's, because R2b puts the - # preset list on the action's own panel, so that is the one they were looking at. That - # answers a definition at REST. A save is different: it carries an INTENT, and the honest - # question is which side this particular save changed. So: - # · the trigger-side picker moved -> it wins, and the pinned action is re-pointed to - # follow it (leaving the action behind would just re-revert on the next save); - # · only the action moved, or neither did and they were already inconsistent -> R2's - # migration, unchanged, reconciling a stored definition exactly once. - # · BOTH moved in one payload -> the action still wins. That is R2 literally, and it is - # also the only reading that cannot lose a value: the action's table is the one with - # the field mapping attached to it. - first = (flow.get("actions") or [{}])[0] - pinned = str((first.get("config") or {}).get("table") or "") \ - if first.get("kind") == "create_record" else "" - prev_target = str((prev.get("config") or {}).get("targetTable") or "") - prev_pinned = str(((((prev.get("flow") or {}).get("actions") or [{}])[0] - ).get("config") or {}).get("table") or "") - target_moved = bool(prev_target) and config.get("targetTable") != prev_target - action_moved = bool(prev_pinned) and pinned != prev_pinned - if pinned and target_moved and not action_moved: - # The person changed the trigger's database. Carry it INTO the pinned step so the two - # agree, instead of throwing their edit away to keep a stale copy consistent. - cfg1 = dict(first.get("config") or {}) - cfg1["table"] = str(config.get("targetTable") or "") - flow["actions"] = [{**first, "config": cfg1}, *(flow.get("actions") or [])[1:]] - elif pinned and pinned != config.get("targetTable"): - config["targetTable"] = pinned - # The stored label described a DIFFERENT database, so keeping it would caption the new - # one with the old one's name. Blank falls back to the key everywhere, and `ut_ensure` - # never relabels a table that already exists — so an adopted hand-made database keeps - # the name its owner gave it. - config["targetLabel"] = "" - return { - "id": _s(prev.get("id") or raw.get("id"), 40), - "name": name, "kind": kind, "config": config, "schedule": schedule, - "trigger": trigger, - # The Builder's ordered actions. An absent flow is simply an automation with no actions. - "flow": flow, - "status": prev.get("status") or {"state": "idle", "lastRunAt": "", "lastSummary": ""}, - "runs": list(prev.get("runs") or [])[:MAX_RUNS], - # ⚠ ENGINE-OWNED CONTINUATION STATE, and it is NOT config. A discovery snapshot takes ~20 - # minutes to build, so a run persists its id here and the NEXT run collects it. It is - # carried through `patch` untouched because a user pressing Save must not silently orphan - # a set the vendor is already building (and already counting against the funds gate). - "state": dict(prev.get("state") or {}), - "created": prev.get("created") or _iso(), - "createdBy": prev.get("createdBy") or username, - # ⭐⭐ WAVE 35 · T37 — `system` IS STICKY ACROSS A PATCH, AND IT IS **NEVER READ FROM `raw`**. - # - # ⛔ THE BUG THIS CLOSES WAS ALREADY WRITTEN AND WOULD HAVE SHIPPED. `routes_automation. - # delete_automation` refuses any stored row carrying `system`, which is what makes a seeded - # system agent undeletable — and this function is a WHITELIST that did not carry the key, so - # the FIRST EDIT of such an agent silently stripped its marker and made it deletable. The - # seed would then mint it again on the next list call: a delete that appears to work and - # undoes itself. Found by asserting the round trip rather than by reading the code. - # - # ⛔ FROM `prev` ONLY. Reading it from `raw` would let ANY caller POST - # `{"system": "anything"}` and mint themselves an automation nobody can delete — a - # privilege escalation through a field nobody validates. The flag is granted by the seeder - # and inherited from the stored row, never asserted by a payload. - # ⚠ Same shape and same reason as the pre-set FIELD flag, which DESIGN.md §4 already - # describes as "sticky across a PATCH". One idea, one behaviour, two objects. - **({"system": _s(prev.get("system"), 40)} if prev.get("system") else {}), - }, None - - -def set_state(rt, auto_id, patch_state): - """Merge into ONE automation's engine state. A key set to None is removed. - - Its own tiny store write rather than a field on `_commit_run`, because the handoff must - survive a run that ends in `error` — a snapshot the vendor is building does not stop existing - because the run that started it failed afterwards. - """ - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - d = cur.get(str(auto_id)) - if d is None: - return cur - st = dict(d.get("state") or {}) - for k, v in (patch_state or {}).items(): - st.pop(k, None) if v is None else st.__setitem__(k, v) - d["state"] = st - return cur - - # ⚠ W31-T35 — THE ONE `keeps_defs=True` IN THIS MODULE. `set_state` writes `state` and never a - # trigger, and `grid_hook` calls it once per CREATED RECORD; clearing the definitions memo here - # would re-read the whole bucket one row later than before and undo D-134 entirely. Safe - # because `grid_hook` reads exactly one state key (`rcHighwater`) and mirrors its own write - # into the memo in the same statement. Read `_store_update`'s note before adding a second. - _store_update(rt, _up, flush="sync", keeps_defs=True) - - -def _without_retired_board(definition): - """Return a definition with retired Board-only state removed, without mutating the store. - - This protects scheduled/manual execution before the next UI list request persists the - migration. It removes only the former Board configuration, actions, and audit trail; normal - actions and all user data remain intact. - """ - if not isinstance(definition, dict): - return definition, False - out, changed = dict(definition), False - cfg = out.get("config") - if isinstance(cfg, dict) and "lanes" in cfg: - cfg = dict(cfg) - cfg.pop("lanes", None) - out["config"] = cfg - changed = True - - def _actions(actions): - nonlocal changed - clean = [] - for action in actions if isinstance(actions, list) else []: - if not isinstance(action, dict): - clean.append(action) - continue - if action.get("kind") == "review": - changed = True - continue - item = dict(action) - if item.get("kind") == "group" and isinstance(item.get("config"), dict): - group_cfg = dict(item["config"]) - branches = group_cfg.get("branches") - if isinstance(branches, list): - kept = [] - for branch in branches: - if not isinstance(branch, dict): - changed = True - continue - next_actions = _actions(branch.get("actions")) - if not next_actions: - changed = True - continue - kept.append({**branch, "actions": next_actions}) - if not kept: - changed = True - continue - if kept != branches: - changed = True - group_cfg["branches"] = kept - item["config"] = group_cfg - elif isinstance(group_cfg.get("actions"), list): - nested = _actions(group_cfg["actions"]) - if not nested: - changed = True - continue - if nested != group_cfg["actions"]: - group_cfg["actions"] = nested - item["config"] = group_cfg - clean.append(item) - return clean - - flow = out.get("flow") - if isinstance(flow, dict): - next_flow = dict(flow) - actions = _actions(flow.get("actions")) - if actions != flow.get("actions"): - next_flow["actions"] = actions - if "ending" in next_flow: - next_flow.pop("ending", None) - changed = True - out["flow"] = next_flow - if "reviews" in out: - out.pop("reviews", None) - changed = True - return out, changed - - -def retire_automation_board_state(rt, tables=None): - """Persist the idempotent Board retirement, then remove its generated table fields. - - ``tables`` is the optional lent snapshot of the `user_tables` bucket (W29-T01) — it reaches - the stage scan and nothing else. - """ - fields = retire_automation_stage_fields(rt, tables=tables) - try: - stored = dict(rt.get(STORE_KEY) or {}) - except Exception: - return {"definitions": 0, **fields} - plan = {str(aid): cleaned for aid, raw in stored.items() - for cleaned, changed in [_without_retired_board(raw)] if changed} - if not plan: - return {"definitions": 0, **fields} - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - for aid, cleaned in plan.items(): - if aid in cur: - cur[aid] = cleaned - return cur - - _store_update(rt, _up, flush="sync") - return {"definitions": len(plan), **fields} - - -#: ⭐⭐ WAVE 31 · T35 (D-134) — the definitions memo, and the two things that make it safe. -#: `tenant -> (monotonic_at, defs)`. Read ONLY through `all_definitions(..., cached=True)`, which -#: exactly one caller uses (`grid_hook`). -_DEFS_MEMO = {} -#: Deliberately SHORT. This exists to collapse one burst of row events into one read, not to be a -#: cache — an import of 20,000 rows arrives in far less than this, and a stale trigger for two -#: seconds is bounded and recoverable where a stale one for a minute is a mystery. -_DEFS_TTL = 2.0 + touched.append(str(aid)) + return cur + + if key: + _store_update(rt, _up, flush="sync") + return touched + + +def ut_get(rt, key): + return ut_all(rt).get(str(key)) + + +def retire_automation_stage_fields(rt, tables=None): + """Delete obsolete Board-only fields and their hidden cells, never user fields. + + The authoritative selector is the engine's own ``automation.stageField`` / ``cyclesField`` + metadata — labels such as “Stage” are ordinary user vocabulary and are not touched. Old + generated timestamp/cycle cells are cleared too, including rows whose field definition was + removed by an interrupted earlier migration. Re-running this migration is a no-op. + + ⭐ WAVE 29 (W29-T01) — ``tables`` LETS A CALLER LEND ITS SNAPSHOT. The SCAN below is + O(all row-cells in the tenant) over a bucket whose documented ceiling is 35.8 MB / ~1.4 s to + deep-copy (see this module's header), and `GET /automations` was paying for THREE independent + copies of it per request. The scan is read-only, so borrowing the caller's copy is free; the + WRITE below still goes through `rt.update`, which re-reads under the store lock, so a lent + snapshot can never be the thing that gets written back. + """ + tables = ut_all(rt) if tables is None else tables + stage_keys = set() + for table in tables.values(): + if not isinstance(table, dict): + continue + for field in table.get("fields") or []: + auto = field.get("automation") if isinstance(field, dict) else None + if isinstance(auto, dict) and (auto.get("stageField") or auto.get("cyclesField")): + stage_keys.add(str(field.get("key") or "")) + for row in (table.get("rows") or {}).values(): + for key in (row or {}): + text = str(key) + if text.startswith("stage_auto_"): + stage_keys.add(text.removesuffix("_at").removesuffix("_cycles")) + stage_keys.discard("") + if not stage_keys: + return {"tables": 0, "fields": 0, "cells": 0} + + changed = {"tables": set(), "fields": 0, "cells": 0} + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + for table_key, table in cur.items(): + if not isinstance(table, dict): + continue + kept, removed = [], set() + for field in table.get("fields") or []: + auto = field.get("automation") if isinstance(field, dict) else None + key = str(field.get("key") or "") if isinstance(field, dict) else "" + if key in stage_keys and isinstance(auto, dict) and \ + (auto.get("stageField") or auto.get("cyclesField")): + removed.add(key) + changed["fields"] += 1 + continue + kept.append(field) + if removed: + table["fields"] = kept + changed["tables"].add(str(table_key)) + for row in (table.get("rows") or {}).values(): + if not isinstance(row, dict): + continue + for stage_key in stage_keys: + for key in (stage_key, stage_key + "_at", stage_key + "_cycles"): + if key in row: + row.pop(key, None) + changed["cells"] += 1 + changed["tables"].add(str(table_key)) + return cur + + rt.update(UT_STORE_KEY, _up, flush="sync") + return {"tables": len(changed["tables"]), "fields": changed["fields"], + "cells": changed["cells"]} + + +def bind_unbound_fields(rt): + """C8's migration (wave 22, stated choice): existing automation bags WITHOUT a `flowId` + are bound to the definition that already writes them — matched on the (targetTable, + fieldKey) pair the definition carries, which is the binding W21-C2 established. A bag no + definition references is DISABLED with the reason on it rather than deleted or guessed: + the column was already dead (nothing runs a column no flow names), and now it says so. + Returns `(bound, disabled)`; costs zero store commits when there is nothing to migrate.""" + by_binding = {} + for aid, d in all_definitions(rt).items(): + cfg = d.get("config") or {} + if cfg.get("fieldKey") and cfg.get("targetTable"): + by_binding[(str(cfg["targetTable"]), str(cfg["fieldKey"]))] = str(aid) + plan = {} + for tk, t in ut_all(rt).items(): + for f in (t.get("fields") or []): + a = f.get("automation") + # An already-DISABLED bag is a decision this migration made on a previous pass — + # re-planning it every call would turn the once-per-process sweep into a write + # per read. + if isinstance(a, dict) and not a.get("flowId") and not a.get("stageField") \ + and not a.get("disabled"): + plan[(tk, str(f.get("key")))] = by_binding.get((tk, str(f.get("key")))) + if not plan: + return 0, 0 + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + for (tk, fk), aid in plan.items(): + for f in ((cur.get(tk) or {}).get("fields") or []): + if f.get("key") == fk and isinstance(f.get("automation"), dict): + if aid: + f["automation"]["flowId"] = aid + else: + f["automation"]["disabled"] = True + f["automation"]["statusNote"] = ( + "not bound to any flow. Create an automation for this column " + "or delete it") + return cur + + rt.update(UT_STORE_KEY, _up, flush="sync") + bound = sum(1 for v in plan.values() if v) + return bound, len(plan) - bound + + +def ut_key_for(label, key=None): + """The table key a label resolves to. Factored out of `ut_ensure` so the DRY-RUN path (which + must not create anything) resolves the same key by construction rather than by copying the + derivation and drifting from it.""" + k = str(key or (UT_PREFIX + _ut_slug(label))) + return k if k.startswith(UT_PREFIX) else UT_PREFIX + k + + +#: Machine names, mirrored from `core.user_tables.MACHINE_OWNERS`. A LOCAL literal for the same +#: reason `UT_FIELD_TYPES` is one over there — this module must stay importable without dragging +#: `core` into the API's boot path — and the two are held in step by a gate check. +MACHINE_OWNERS = ("automation", "scheduler") + + +def _preset_description_due(stored, wanted): + """The shipped sentence a stored PRESET column is still missing, or None to leave it alone. + + ⭐⭐ WAVE 41 · T22 (R19: *a user's description edit wins forever*) — `IG_FIELD_DESCRIPTIONS` + and `TT_FIELD_DESCRIPTIONS` reach a BRAND-NEW column through `field_def`, which is why an + untouched field has always shown the right sentence. They reach an EXISTING one only through + `_reconcile_ig_graph_fields` — and that pass is Instagram's. **TikTok has no reconciler at + all**, so a `ut_tt_*` column that predates its sentence (or one whose description was cleared) + had nothing anywhere that could put the default back, and R19's *Reset to default* would have + had nothing to restore FROM. `ut_ensure(lock_fields=True)` is the one door BOTH platforms' + preset sets pass through, so the repair belongs there; on Instagram it is a no-op after the + reconciler has already agreed with it. + + ⛔ THE CUSTODY STAMP IS THE WHOLE POINT, and it is `user_tables.user_edited` — the SAME + predicate `_reconcile_ig_graph_fields` reads, never a second mechanism invented beside it. A + human who has taken a column over keeps their sentence forever, and CLEARING the stamp is what + makes the shipped one come back. That is the reset, expressed as the absence of custody rather + than as a second flag somebody has to remember to write. + ⚠ THE STAMP IS FIELD-WIDE, not description-only (`core.user_tables.USER_EDITED_KEY`, written + once in `patch_field`). This function therefore cannot tell a description edit from a label + edit, and deliberately does not try: it reads the custody mark that exists. + ⚠ NORMALISED AND CAPPED exactly as `core.user_tables._clean_field` would, so the value written + here is the value that validator would keep — a preset whose stored sentence disagreed with + its own validator would otherwise be rewritten on every single run. + """ + want = " ".join(str((wanted or {}).get("description") or "").split())[:300] + if not want or _ut().user_edited(stored): + return None + return None if str((stored or {}).get("description") or "") == want else want + + +def ut_ensure(rt, label, fields, username="automation", key=None, flow_tag="", + record_mode="", lock_fields=False, tables=None): + """Create the table if it is missing; return its key. Idempotent — a re-run of an automation + that owns a table must not spawn `ut_x_2`, so the key is DERIVED from the label (or given) + and an existing table with that key is adopted, not duplicated. + + ⭐⭐ WAVE 31 · T30 — `tables` LETS A CALLER LEND THE SNAPSHOT IT IS ALREADY HOLDING, and it is + the SKIP TEST below that it pays for. `rt.get` deep-copies the whole tenant document on every + call (documented ceiling 35.8 MB / ~1.4 s), so a caller that ensures FOUR tables in one pass + paid four copies to answer four questions about one document. Owner item 7, verbatim: *"It still + takes forever to change the Config for automation as well. It just say Saving..."* — measured + **7,912 ms** for one save on `4258a93`. + + ⚠ THE LEND IS READ-ONLY AND SAFE BY INSPECTION, not by hope — the same argument + `retire_automation_stage_fields` (this module, same parameter name, same contract) already + makes: only the `have` lookup below reads it, and the WRITE still goes through `rt.update`, + whose `_up` re-reads the live document under the store lock. **A lent snapshot can therefore + never be the thing that gets written back**, and the worst a stale one can do is spend a commit + that would have been skipped — never write a wrong value. Callers that ensure two tables + under the SAME key in one pass pass `tables=None` for the second (see `_spawn_presets`). + + Fields are MERGED, never replaced: a user who added a column to an automation's table keeps + it, and a new source column joins on the next run. + + `flow_tag` (wave 22, C7/item 5): every field THIS call adds is stamped + `automation: {flowId: }` — the pre-set columns an IG automation spawns carry their + provenance. Fields already on the table keep whatever tag they have (first flow wins; + shared tables like ut_ig_snapshots are fed by many flows and the tag is provenance, not + ownership). + + ⛔ AND IT STAMPS A HUMAN OWNER, WHICH IT DID NOT (wave 20, item 3). `createdBy` was whoever + or WHATEVER ran the automation, so the same table belonged to a person if its first run was + manual and to `"scheduler"` if the schedule got there first — and `user_tables.may_open` + admits only the creator or an admin, so **whether you could open your own Instagram + snapshots depended on a race you never saw**. The owner is now the automation's creator, and + a table already stamped with a machine name is ADOPTED the next time a run knows a human + one. Adoption is a repair, not a widening: the automation's creator is the person who asked + for the table in the first place. + """ + key = ut_key_for(label, key) + # ⭐ WAVE 26 — THE MIGRATION RIDES THE WRITE PATH, and that placement is the point. + # + # `ut_ensure` MERGES fields by key and never re-types an existing column, so on its own it + # would leave every table already in production on the old `text` schema forever while new + # ones got the honest types — the split schema R3's note warned about, arriving through the + # very function the note was written on. Migrating here means a table is brought forward + # immediately BEFORE anything appends to it, so no caller has to remember anything and no + # tenant is left behind by a script nobody ran. + # ⚠ Cheap by construction: `migrate_ig_tables` returns without a write when the table is + # already current, which after the first run is every call. + if key and {f.get("key") for f in (fields or [])} & set(PRESET_PROFILE_KEYS): + try: + migrate_ig_tables(rt, log=lambda *_a: None, only=key) + except Exception: # noqa: BLE001 + # A migration that cannot run must not stop the automation from writing its rows. + # The old schema still reads; a refused write loses the pull we just paid for. + pass + wanted = [] + for raw_field in (fields or []): + field = dict(raw_field) + if flow_tag or lock_fields: + automation = dict(field.get("automation") or {}) + if flow_tag: + automation.setdefault("flowId", str(flow_tag)) + if lock_fields: + automation["preset"] = True + field["automation"] = automation + wanted.append(field) + created = _iso() + human = username if username and username not in MACHINE_OWNERS else "" + + # ⚠ SKIP THE WRITE WHEN NOTHING WOULD CHANGE. Without this, every re-run spends a store + # commit re-writing an identical definition — against a 20 s flush floor and a 256/hr repo + # budget, an idempotent helper that always writes is the same defect as a per-row insert. + have = ut_get(rt, key) if tables is None else tables.get(str(key)) + have_fields = {str(f.get("key") or ""): f for f in ((have or {}).get("fields") or [])} + missing = [f for f in wanted if f.get("key") not in have_fields] + missing_locks = [f for f in wanted + if lock_fields and f.get("key") in have_fields + and (not isinstance(have_fields[f.get("key")].get("automation"), dict) + or have_fields[f.get("key")]["automation"].get("preset") is not True)] + # ⭐ WAVE 41 · T22 — AND THE SKIP TEST HAS TO KNOW ABOUT THE DESCRIPTION REPAIR, or the repair + # in `_up` below can never run: every table that already exists returns HERE, three lines + # before it. A term added to `_up` without a term added here is a change that reads correctly + # and does nothing on precisely the tables it was written for. + # ⚠ It converges: once `_up` has written the sentence, `_preset_description_due` answers None + # for that field and this skip fires again, so the owner's save path pays no extra commit. + stale_descriptions = [f for f in wanted + if lock_fields and f.get("key") in have_fields + and _preset_description_due(have_fields[f.get("key")], f) is not None] + if (have is not None and not missing and not missing_locks and not stale_descriptions + and not (record_mode and have.get("recordMode") != record_mode) + and not (human and (have.get("createdBy") or "") in MACHINE_OWNERS)): + return key + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + t = cur.get(key) + if t is None: + if len(cur) >= MAX_UT_TABLES: + return cur + cur[key] = {"key": key, "label": str(label)[:60], "source": "Automation", + "createdBy": username, "created": created, + "fields": wanted, "rows": {}} + if record_mode: + cur[key]["recordMode"] = record_mode + return cur + have = {f.get("key") for f in (t.get("fields") or [])} + for f in wanted: + if f.get("key") not in have: + t.setdefault("fields", []).append(f) + have.add(f.get("key")) + elif lock_fields: + stored = next((g for g in (t.get("fields") or []) + if g.get("key") == f.get("key")), None) + if stored is not None: + # ⭐⭐ WAVE 41 · T22 (R19) — the shipped sentence, and ONLY on a column no + # human has taken custody of. Read BEFORE the `preset` stamp below, so the + # custody question is asked of the field exactly as it was found. + due = _preset_description_due(stored, f) + if due is not None: + stored["description"] = due + automation = dict(stored.get("automation") or {}) + if flow_tag: + automation.setdefault("flowId", str(flow_tag)) + automation["preset"] = True + stored["automation"] = automation + # ADOPTION: a machine name is not an owner. It never overwrites a human one. + if human and (t.get("createdBy") or "") in MACHINE_OWNERS: + t["createdBy"] = human + # An automation's table SAYS an automation owns it — the nav badge reads from this. + t["source"] = t.get("source") or "Automation" + if record_mode: + t["recordMode"] = record_mode + return cur + + rt.update(UT_STORE_KEY, _up, flush="sync") + return key + + +def ut_write_rows(rt, key, rows): + """ONE store update for the WHOLE row set — the flush-ceiling rule (see the module header).""" + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + t = cur.get(key) + if t is not None: + t["rows"] = rows + return cur + rt.update(UT_STORE_KEY, _up, flush="sync") + + +def ut_set_cell(rt, key, row_id, field_key, value, extra_rows=None): + """Write one automation-owned cell (+ optional whole extra tables) in ONE update.""" + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + t = cur.get(key) + if t is not None: + t.setdefault("rows", {}).setdefault(str(row_id), {})[field_key] = value + for k, rws in (extra_rows or {}).items(): + tt = cur.get(k) + if tt is not None: + tt["rows"] = rws + return cur + rt.update(UT_STORE_KEY, _up, flush="sync") + + +IG_FIELD_DESCRIPTIONS = { + "alt_text": "Accessibility text attached to the Instagram post.", + # ⚠ WIDENED 2026-08-10, because the column gained a second writer and the old sentence would + # have made it lie. It was written for the anonymous rung, whose numbers are genuinely ROUNDED + # ("10.4K followers"). A DISCOVERY observation is exact-looking and STALE instead — read off + # the vendor's pre-collected corpus at a collection time we are not told. Both mean the same + # thing to a reader ("do not treat this as an exact count taken at Pulled at"), and a + # description that named only the first would have quietly excluded the second. + "approx": "Checked when the counts are rounded, or were read from a pre-collected corpus " + "rather than measured at the time shown.", + "avg_comments_12": "Average comments across the 12 most recent captured posts.", + "avg_engagement": "Average engagement rate reported for the profile.", + "avg_likes_12": "Average likes across the 12 most recent captured posts.", + "avg_plays_12": "Average plays across the 12 most recent captured posts.", + "avg_views_12": "Average views across the 12 most recent captured posts.", + "views": "View count for the post, as Instagram displays it.", + "video_duration": "Length of the video in seconds.", + "comments_disabled": "Checked when the creator has turned comments off for this post.", + "plays": "Times the video started playing, including replays.", + "bio": "Biography shown on the Instagram profile.", + "bio_hashtags": "Hashtags listed in the profile biography.", + "business_category": "Instagram's business category for the account.", + "caption": "Caption published with the Instagram post.", + "category": "Instagram's category for the profile.", + "comment_key": "Unique ID for the captured comment record.", + "commented_at": "Date the comment was posted.", + "comments": "Comment count reported for the post.", + "comments_captured": "Number of distinct comment records linked to this profile.", + "comments_link": "Comment records linked to this record.", + "country_code": "Country code reported for the profile.", + "created_by": "User whose automation first added this lead.", + "enriched_at": "Date the profile was last enriched.", + # ⭐ ITEM 16. The description says GUESS out loud, because the column is one and the number + # beside it is the only thing that says how much of one. + "location_guess": "Most common city tagged across this profile's captured posts - a guess, " + "not a stated location.", + "location_confidence": "Share of this profile's geotagged posts that agree on the guessed " + "city.", + "external_url": "Website linked from the Instagram biography.", + "external_url_title": "Title Instagram shows for the biography link.", + "fbid": "Facebook ID associated with the Instagram account.", + "first_found": "Date an automation first found this profile.", + "followers": "Latest reported follower count.", + "following": "Latest reported number of accounts followed.", + "found_count": "Number of times discovery automations found this profile.", + "full_name": "Display name shown on the Instagram profile.", + "handle": "Instagram username without the @ symbol.", + "has_channel": "Checked when the profile has an Instagram channel.", + "hashtags": "Hashtags extracted from the post caption.", + "highlights_count": "Total story highlight collections reported for the profile.", + "ig_id": "Instagram's internal ID for the account.", + "influencer_key": "Normalized handle linking this row to its profile.", + "is_business": "Checked when Instagram marks the account as a business.", + "is_joined_recently": "Checked when Instagram marks the account as recently joined.", + "is_private": "Checked when the Instagram profile is private.", + "is_professional": "Checked when Instagram marks the account as professional.", + "last_found": "Date an automation most recently found this profile.", + "likes": "Like count reported for this record.", + "measured_at": "Date engagement metrics on this post were last read.", + "measurements_captured": "Number of measurement rows stored for this post.", + "paid_partnership": "Checked when Instagram marks the post as a paid partnership.", + "partner": "Brand named in the post's paid partnership metadata.", + "partner_id": "Partner ID reported for the Instagram profile.", + "platform": "Social network for this profile.", + "plays": "Video plays, including repeat plays, reported for the post.", + "post_link": "Post record linked to this measurement or comment.", + "post_measurements_captured": "Number of distinct post measurements linked to this profile.", + "post_snapshot_key": "Unique ID for this post measurement.", + "post_snapshots_link": "Engagement measurement rows linked to this record.", + "posted_at": "Date the Instagram post was published.", + "posts_captured": "Number of distinct post records captured for this profile.", + "posts_count": "Total posts reported for the Instagram profile.", + "posts_link": "Post records linked to this profile.", + "profile_name": "Profile name returned by the data provider.", + "profile_reads": "Number of profile snapshots stored for this profile.", + "profile_snapshots_link": "Profile snapshot rows linked to this profile.", + "profile_url": "Direct URL to the Instagram profile.", + "pronouns": "Pronouns shown on the Instagram profile.", + "pulled_at": "Date this snapshot was collected.", + "related_accounts": "Accounts Instagram suggests alongside this profile.", + "replies": "Reply count reported for the comment.", + "shortcode": "Instagram shortcode that uniquely identifies the post.", + "source_payload": "Full source response for this record.", + "snapshot_key": "Unique ID for this profile snapshot.", + "source": "Method or provider used to read this profile.", + "tagged_location": "Location tagged on the post; it is not the creator's residence.", + "text": "What the comment says. The commenter's name is not stored as a column.", + "type": "Instagram media type: image, video, or carousel.", + "url": "Direct URL to the Instagram post.", + "verified": "Checked when Instagram marks the profile as verified.", +} + + +#: ⭐ WAVE 29 (R2) — TIKTOK'S OWN DESCRIPTIONS, and the reason this exists rather than reusing the +#: map above is one line of `field_def`: it defaults `description` to +#: `IG_FIELD_DESCRIPTIONS.get(key)`. Nine `ut_tt_*` columns share a KEY with an Instagram column +#: (`shortcode`, `type`, `url`, `verified`, `caption`, `likes`, `comments`, `replies`, +#: `tagged_location`), so a TikTok video's Type column would have shipped explaining "Instagram +#: media type" — a wrong sentence in the header tooltip of a column nobody would think to check. +#: ⚠ A KEY ABSENT HERE GETS NO DESCRIPTION AT ALL, deliberately: silence is honest, and inheriting +#: the Instagram sentence is the failure this map exists to prevent. +TT_FIELD_DESCRIPTIONS = { + "platform": "Network this handle is on.", + "handle": "TikTok @name; the unique account handle.", + "full_name": "Display name shown on the TikTok profile.", + "tt_id": "TikTok's own numeric id for the account.", + "profile_url": "Direct URL to the TikTok profile.", + "bio": "Profile biography text.", + "external_url": "Link in the TikTok bio.", + "verified": "Checked when TikTok marks the account as verified.", + "is_private": "Checked when the account is private.", + "followers": "Follower count at the time of the pull.", + "following": "Accounts this profile follows.", + "posts_count": "Videos published by this account.", + "likes_received": "Total likes this account's videos have received.", + "avg_engagement": "Average engagement rate, stored as a percentage.", + "comment_engagement": "Comment engagement rate, stored as a percentage.", + "like_engagement": "Like engagement rate, stored as a percentage.", + "is_business": "Approximate: set when TikTok flags the account as a commerce user.", + "country_code": "Two-letter country code reported for the account.", + "predicted_lang": "Language TikTok predicts for this account.", + "account_created_at": "When the TikTok account itself was created.", + "region": "Region reported for the account.", + "shortcode": "TikTok's numeric id for the video; unique per post.", + "type": "TikTok post type: video or image.", + "url": "Direct URL to the TikTok post.", + "caption": "Post description text.", + "posted_at": "When the creator published the post.", + "likes": "Likes reported for the post.", + "comments": "Comment count reported for the post.", + "views": "Play count TikTok reports for the video.", + "shares": "Times the post was shared.", + "saves": "Times the post was saved to a collection.", + "video_duration": "Video length in seconds.", + "hashtags": "Hashtags used in the post.", + "tagged_location": "Commerce location reported for the post; it is not the creator's home.", + "influencer_key": "Handle of the account this record belongs to.", + "comment_key": "Unique ID for this comment.", + "commented_at": "When the comment was posted.", + "text": "What the comment says. The commenter's name is not stored as a column.", + "replies": "Reply count reported for the comment.", + "snapshot_key": "Unique ID for this profile snapshot.", + "post_snapshot_key": "Unique ID for this post measurement.", + "pulled_at": "When this measurement was read.", + "enriched_at": "When this row was last enriched.", + "source": "Method or provider used to read this record.", + "source_payload": "Full source response for this record.", +} + + +def field_def(text_key, label, ftype="text", **extra): + """One machine-spawned column definition. + + ⭐ 2026-08-07 — `**extra` carries the per-field DECLARATIONS the preset set needs (`pinned`, + `profile`), and it stays ONE constructor rather than growing a second for "special" fields. + ⛔ Every key passed here must survive `core.user_tables._clean_field` UNCHANGED — `verify_api`'s + W25-1 section pins the drift at ZERO, so a key the validator drops or rewrites turns the whole + preset list red rather than failing quietly ([[default-must-pass-its-own-guard]]). + + ⭐ WAVE 25 — `editRole` IS DECLARED HERE, and adding it closes a bypass rather than adding a + feature. `ut_ensure` writes these dicts STRAIGHT into the `user_tables` bucket, so + `core.user_tables._clean_field` — the single validator every field created through the ordinary + door passes — has never judged a single field this engine spawned. The difference was exactly + one key: `_clean_field` emits `editRole: 'admins'` and this did not, so `_clean_field(f) != f` + for every automation column in the product. + ⚠ NOTHING CHANGES BEHAVIOURALLY — `may_edit_field` asks `editRole == 'everyone'`, and an + absent key was already not that, so the bypass was fail-closed and therefore silent. It is + stated now, and `verify_automation` asserts the equality field-by-field, so the next key + `_clean_field` grows cannot go unnoticed here ([[default-must-pass-its-own-guard]]). + """ + description = " ".join(str(extra.pop( + "description", IG_FIELD_DESCRIPTIONS.get(text_key, "")) or "").split()) + out = {"key": text_key, "label": label, "type": ftype, "source": "overlay", + "editRole": "admins"} + if description: + out["description"] = description + out.update(extra) + return out + + +def tt_field_def(text_key, label, ftype="text", **extra): + """One `ut_tt_*` column. `field_def` with TikTok's description map bound (wave 29, R2). + + ⛔ `description` IS ALWAYS SUPPLIED, even when blank, so `field_def`'s Instagram default can + never be reached from here. An explicit empty string makes `field_def` omit the key, which is + the same shape an undescribed column already has — the point is that the sentence a TikTok + column carries is one somebody wrote about TikTok, or none at all. + """ + extra.setdefault("description", TT_FIELD_DESCRIPTIONS.get(text_key, "")) + return field_def(text_key, label, ftype, **extra) + + +# --------------------------------------------------------------------------------------------- +# DEFINITIONS — validation and the bucket's read/write half +# --------------------------------------------------------------------------------------------- + +def _s(v, n=200): + return str(v if v is not None else "")[:n] + + +#: ⭐ WAVE 26 · C5 / R2 — HOW MANY POSTS ONE PULL MAY KEEP, and the ceiling is the VENDOR'S. +#: +#: ⛔ MEASURED 2026-08-05 and recorded at `_bd_posts_count`: **a Profiles row carries the TOP 12 +#: posts — a cap, not a count.** Asking for more does not fetch more; it just makes the config +#: disagree with what the run can possibly do. +#: ⚠ THIS REPLACES A SILENT `max(1, min(n, 200))` AT TWO CALL SITES, and the clamp was the defect +#: rather than the number: a person typing 50 got a stored 50, a UI that read back 50, and twelve +#: posts — with nothing anywhere saying why. A control that accepts a value it cannot honour is +#: worse than one that refuses it, because the refusal is the only place the ceiling can be +#: taught. So this REFUSES, and the sentence names the cap and who set it. +#: +#: ⭐⭐ 2026-08-09 — RAISED TO 30 (owner: *"move the max limit to 30 posts per profile"*), and the +#: paragraph above needed correcting to do it honestly: **12 was OUR ROUTE'S cap, not the +#: vendor's.** Re-measured the same day — the PROFILE record still returns 12 however many you ask +#: for (asked 40, got 12), but the documented discover-by-url route answered `num_of_posts: 30` +#: with exactly 30 rows in 90 s (23 Reels + 7 Carousels, @theresalearns). The ceiling was a +#: property of the call we happened to make. +#: ⛔ SO THE NUMBER MOVED AND THE CAPTURE MUST FOLLOW. Until `bd_profile_posts` is wired ahead of +#: the views top-up, a `maxPosts` above 12 is honoured by the CONFIG and bounded by the PROFILE +#: read at run time — the exact accept-what-you-cannot-honour shape this constant exists to +#: prevent, now surviving in one place instead of two. It is recorded rather than hidden, and it +#: is why `clean_post_groups` bounds a group by `maxPosts` rather than by 12. +MAX_POSTS_PER_PULL = 30 +DEFAULT_POSTS_PER_PULL = 10 + +#: ⭐⭐ THE WINDOW THE `avg_*_12` PRESET ROLLUPS AVERAGE OVER — ITS OWN CONSTANT, deliberately +#: NOT `MAX_POSTS_PER_PULL`. +#: +#: ⛔ THEY WERE THE SAME NUMBER AND THAT WAS A LATENT BUG, caught by the gate the moment the cap +#: moved: four columns are NAMED `avg_views_12` and LABELLED "Avg views · last 12 posts", so +#: raising the capture cap to 30 silently made every one of them average thirty posts under a +#: label that says twelve. Nobody would have looked at those columns again to check. +#: ⚠ Raising the CAPTURE ceiling and redefining an EXISTING named measure are two different +#: decisions, and only the first one was made. To widen the average, change this constant AND the +#: four keys and labels together — a column whose meaning changes underneath its own name is +#: worse than one that is merely narrow. +AVG_WINDOW_POSTS = 12 + + +#: The post kinds a group filter may name — exactly what `_bd_type` maps a vendor row onto, so a +#: filter cannot ask for a category that can never match a stored row. +POST_TYPES = ("video", "image", "carousel") +#: ⭐ W29-T09 — the words a PERSON reads for those three keys, server-owned for the same reason +#: `KIND_LABELS` and `TRIGGER_LABELS` are. The owner asked for *"the last 12 reels"*, and `video` +#: is the stored key: a client that translated it locally would be a second copy of this +#: vocabulary, free to drift the day a fourth kind appears or a name changes. +#: ⚠ "Reels & videos", not "Reels": `_bd_type` maps every non-image, non-carousel post here, so a +#: label naming only reels would over-promise on a plain video post. +POST_TYPE_LABELS = {"video": "Reels & videos", "image": "Photos", "carousel": "Carousels"} + + +def clean_post_groups(raw, max_posts): + """`(groups, error)` for `config.postGroups` — "last N reels, last M images". + + ⭐ 2026-08-09 (owner: *"not just last 12 but by group also"*). Shape: + `[{"type": "video", "limit": 12}, {"type": "image", "limit": 6}]`. + + ⛔ ABSENT IS OFF, and off must stay the default forever: an enrich action stored before today + carries no such key, and inventing a filter for it would silently start DROPPING posts those + automations have always captured. Empty list and missing are the same answer. + + ⚠ A MALFORMED GROUP IS REFUSED, not dropped — the opposite of `tier`/`noFallback`, and the + difference is legitimate. Those are RETIRED keys that live in stored data, so refusing them + would 400 old automations forever (D-65). This key is NEW: nothing stored can carry a broken + one, so the only way to see one is a person typing it, and a filter that silently ignores the + type you asked for is how you end up paying for reels and storing carousels. + """ + if raw in (None, "", [], {}): + return [], None + if not isinstance(raw, list): + return None, "the post groups have to be a list of {type, limit} entries" + out, seen = [], set() + for item in raw: + if not isinstance(item, dict): + return None, "each post group has to be a {type, limit} entry" + t = str(item.get("type") or "").strip().lower() + if t not in POST_TYPES: + return None, (f"'{t or 'blank'}' is not a post type. Use one of " + f"{', '.join(POST_TYPES)}") + if t in seen: + return None, f"the post groups name '{t}' twice. One limit per type" + seen.add(t) + try: + lim = int(item.get("limit")) + except (TypeError, ValueError): + return None, f"how many {t} posts to keep has to be a whole number" + if lim < 1: + return None, f"a {t} group has to keep at least one post" + # ⛔ BOUNDED BY WHAT THE RUN ACTUALLY BUYS. A group asking for 30 when the pull captures + # 12 is not an error a person can act on — it is a promise the run cannot keep — so it is + # refused HERE, where the number they typed is still on the screen in front of them. + if lim > max_posts: + return None, (f"this enrichment captures {max_posts} posts per profile, so a {t} " + f"group cannot keep {lim}. Raise the post count first") + out.append({"type": t, "limit": lim}) + return out, None + + +def clean_max_posts(raw, default=DEFAULT_POSTS_PER_PULL, submitted=True): + """`(maxPosts, error)` — refuses out-of-range rather than clamping into range. + + ⛔ `submitted=False` CLAMPS INSTEAD OF REFUSING, and that asymmetry is the whole reason this + takes a flag. **Every automation stored before wave 26 carries `maxPosts: 24`** — the old + clamp's default, which this cap now forbids. If an inherited value were refused the same way + a typed one is, every one of those automations would 400 on its next Save **forever**, for a + number nobody on that screen chose or can see. That is D-65's lesson exactly: a validator + that refuses stored data does not protect the user from it, it locks them out of their own + automation. + So: a value the panel SENT is the user asking for it, and gets the sentence. A value merely + INHERITED gets silently brought inside the cap it was already effectively subject to at the + vendor — the run was returning 12 either way ([[default-must-pass-its-own-guard]]). + """ + if raw is None or (isinstance(raw, str) and not raw.strip()): + return default, None + try: + n = int(raw) + except (TypeError, ValueError): + if not submitted: + return default, None + return None, "the post limit must be a whole number" + if not submitted: + return max(1, min(n, MAX_POSTS_PER_PULL)), None + if n < 1: + return None, "keep at least one post per profile, or turn the post capture off" + if n > MAX_POSTS_PER_PULL: + return None, (f"a profile pull returns at most {MAX_POSTS_PER_PULL} posts. That is the " + f"vendor's cap, not ours, so {n} would store the same " + f"{MAX_POSTS_PER_PULL} and read as more") + return n, None + + +def clean_config(kind, raw, previous=None): + """Validate a kind's config. Returns `(config, error)`; error is a user-facing sentence. + + ⚠ THE NODE-SWITCH FLAGS FALL BACK TO `previous` WHEN THE KEY IS ABSENT, and that is a rail + rather than a nicety. `patch` replaces the whole config, and the canvas's config panels do not + edit `postMetrics` / `commentMetrics` / `dryRun` — those are node SWITCHES. So a plain "Save" + from a panel that never knew about them would silently turn off the dry run, or turn ON a + per-post purchase: a save that quietly changes what the automation costs. Absent ⇒ keep; + present ⇒ take it, including `false`. + """ + raw = raw if isinstance(raw, dict) else {} + prev = previous if isinstance(previous, dict) else {} + + def flag(name): + return bool(raw[name]) if name in raw else bool(prev.get(name)) + if kind == "plain": + # A plain automation has no machine step, so it only needs the database its records walk. + # + # ⚠ THE TARGET IS OPTIONAL ON PURPOSE. An automation triggered by `record_updated` on + # `ut_foo` already knows its table from the trigger — `_flow_table` resolves target-else- + # trigger — so requiring a second copy of that fact here would be a mandatory field with + # exactly one legal answer, and a Save that refuses until you retype what you just picked. + # There is no `dryRun`: dry-run means "run the machine step but do not write", and there + # is no machine step to run. + target = _s(raw.get("targetTable"), 60).strip() + if target and not target.startswith(UT_PREFIX): + target = UT_PREFIX + target + return {"targetTable": target, + "targetLabel": _s(raw.get("targetLabel"), 60).strip()}, None + if kind == "scrape_db": + url = _s(raw.get("url"), 2000).strip() + if not url: + return None, "a source URL is required" + try: + guard(url) + except Refused as e: + return None, str(e) + extract = raw.get("extract") if raw.get("extract") in EXTRACTS else "table" + fmap = {} + for k, v in list((raw.get("fieldMap") or {}).items())[:60]: + tk = re.sub(r"[^a-z0-9_]+", "_", _s(v, 60).strip().lower()).strip("_") + if tk: + fmap[_s(k, 120)] = tk[:60] + if not fmap: + return None, "map at least one source column to a field" + key_field = _s(raw.get("keyField"), 60).strip() + if key_field not in fmap.values(): + return None, "the key field must be one of the mapped fields" + target = _s(raw.get("targetTable"), 60).strip() + if target and not target.startswith(UT_PREFIX): + target = UT_PREFIX + target + return {"url": url, "extract": extract, + "tableIndex": max(0, min(int(raw.get("tableIndex") or 0), 50)), + "fieldMap": fmap, "keyField": key_field, + "targetTable": target, + "dryRun": flag("dryRun"), + "targetLabel": _s(raw.get("targetLabel"), 60).strip() or "Scraped table"}, None + if kind == "field_instagram": + table = _s(raw.get("targetTable"), 60).strip() + if not table.startswith(UT_PREFIX): + return None, "an Instagram automation runs against a blank database (ut_*)" + fkey = _s(raw.get("fieldKey"), 80).strip() + if not fkey: + return None, "pick the automation column this run writes into" + # ⛔⛔ `tier` AND `noFallback` ARE ACCEPTED AND IGNORED (wave 28 / R5, contract C2). + # They are read from nothing and written to nothing: a stored definition carrying either + # still SAVES — it simply loses them on the next write — and neither is ever a reason to + # refuse. That asymmetry is D-65's law and it is not squeamishness: refusing an unknown + # key would 400 every automation a tenant stored before this wave, forever, on a screen + # that gives them no way to remove it. Dropping a retired key is a migration; refusing it + # is an outage. + # ⚠ `clean_tier`/`TIERS`/`bd_ready` KEEP THEIR NAMES. They are VENDOR vocabulary + # (`brightdata` is a provider, and `verify_automation` fences the name), not the retired + # USER concept — renaming them would be a second, unrelated change wearing this one's + # justification. + # C5: absent ⇒ keep whatever is stored, like every other switch in this branch — a panel + # that does not edit the post count must not reset it to the default on Save. And an + # INHERITED value is clamped rather than refused; see `clean_max_posts`. + sent = "maxPosts" in raw + max_posts, perr = clean_max_posts( + raw.get("maxPosts") if sent else prev.get("maxPosts"), submitted=sent) + if perr: + return None, perr + return {"targetTable": table, "fieldKey": fkey, + "urlField": _s(raw.get("urlField"), 80).strip(), + # ⛔ THE ONE FLAG THAT MULTIPLIES THE BILL BY THE POST COUNT. Off unless asked + # for: the Profiles row carries post IDENTITY for free but NO engagement + # (measured 2026-08-05), so likes/comments cost one extra vendor record PER POST. + "postMetrics": flag("postMetrics"), + # The full Comments dataset can bill many rows per post. It is an explicit, + # independent opt-in and never follows the per-post switch automatically. + "commentMetrics": flag("commentMetrics"), + "dryRun": flag("dryRun"), + "maxPosts": max_posts}, None + # ⭐ WAVE 29 (D-9 / R1) — BOTH discovery kinds share this branch, because they ask the vendor + # the same question of two different corpora: a record ceiling, a predicate list, a join word + # and where to write. ⛔ THE ONLY DIFFERENCE IS THE DEFAULT TABLE, and it is resolved from the + # kind rather than hard-coded — a TikTok search falling back to `ut_ig_profile` would write + # TikTok rows into the Instagram family, which is precisely what R2's parallel family exists + # to prevent, and it would do it silently. + if kind in DISCOVERY_KINDS: + # ⭐ WAVE 30 · T05 — the inline tuple became the named one, and the two defaults below now + # come from `discovery_facts` rather than being spelled out here. They were correct; they + # were also the FIFTH copy of "which table does this kind write to", and the other four + # were the ones wave 29 forgot to update. + _, _disc_table, _disc_label, _ = discovery_facts(kind) + limit = int(raw.get("recordsLimit") or 0) + if limit < 1: + return None, ("say how many profiles to fetch. An UNBOUNDED discovery query is the " + "one shape the vendor refuses outright (NOT_ENOUGH_FUNDS)") + if limit > BD_MAX_RECORDS: + return None, f"a single discovery run may ask for at most {BD_MAX_RECORDS} profiles" + # ⚠ THE JOIN IS RESOLVED BEFORE THE PREDICATES ARE JUDGED, because it is part of what + # makes them broad or narrow (see `narrowing_refusal`). Validating them first and reading + # the operator afterwards is how the OR hole survived: the guard was handed the branches + # and never told they were a union. + join = "or" if str(raw.get("operator") or "").lower() == "or" else "and" + # ⭐ WAVE 32 · T46 (D-167) — AND THE KIND GOES WITH THEM. `clean_config` already knows which + # corpus this automation searches; passing it is what stops a `discover_tiktok` being built + # on the 16 Instagram fields TikTok's dataset does not carry — a search that does not error, + # returns nothing, and reads as "no such creators exist" after the money is spent. + preds, perr = clean_predicates(raw.get("predicates"), join, kind) + if perr: + return None, perr + target = _s(raw.get("targetTable"), 60).strip() or _disc_table + if not target.startswith(UT_PREFIX): + target = UT_PREFIX + target + seed, serr = clean_seed(raw.get("seed") if "seed" in raw else prev.get("seed")) + if serr: + return None, serr + return {"recordsLimit": limit, "predicates": preds, + "operator": join, + "targetTable": target, + "targetLabel": _s(raw.get("targetLabel"), 60).strip() or _disc_label, + # C6/R5: WHERE the conditions above came from, when they were derived. Stored so + # the surface can say "these were filled in from the 'Florists' view, over 12 + # records" instead of presenting them as if somebody typed them. + "seed": seed, + "dryRun": flag("dryRun")}, None + return None, f"unknown automation kind {kind!r}" + + +def clean_seed(raw): + """C6: validate `config.seed`. Returns `({}, None)` when there is none — a discovery filter + somebody typed by hand has no seed, and that is the ordinary case. + + ⚠ `derived` AND `basis` ARE STORED AS PROVENANCE, NOT AS A SECOND FILTER. R5 is explicit that + the derived conditions are "written into `config.predicates` as ordinary conditions — it is a + filling-in, not a parallel filter", so the RUN never reads this bag: it reads `predicates`, + like every other search. Keeping it means the surface can say where those rows came from, and + a user editing them freely is exactly what is supposed to happen. + """ + if raw in (None, "", {}): + return {}, None + if not isinstance(raw, dict): + return None, "the seed must be an object" + source = _s(raw.get("source"), 12).strip().lower() + if source not in SEED_SOURCES: + return None, f"{source or 'that seed'!r} is not one of: " + ", ".join(SEED_SOURCES) + table = _s(raw.get("table"), 60).strip() + if table and not table.startswith(UT_PREFIX): + return None, f"a seed reads a blank database (ut_*). {table!r} is not one" + basis = raw.get("basis") if isinstance(raw.get("basis"), dict) else {} + derived, _err = clean_predicates(raw.get("derived") or [], "and") + return {"source": source, "table": table, "id": _s(raw.get("id"), 80).strip(), + # ⛔ A MALFORMED `derived` IS DROPPED, NEVER A REFUSAL. This bag is a RECORD of what + # was suggested; the conditions that matter are already in `predicates` and were + # validated there. Refusing a Save because a stored provenance note aged badly would + # block the user from editing the very filter it describes. + "derived": derived or [], + "basis": {"rows": max(0, int(basis.get("rows") or 0)), + "fields": [f for f in (basis.get("fields") or []) + if isinstance(f, dict)][:SEED_MAX_PREDICATES], + "related": basis.get("related") + if isinstance(basis.get("related"), dict) else {}, + "note": _s(basis.get("note"), 200)}}, None + + +def clean_schedule(raw, previous=None): + """Validate `{cron, enabled}` and stamp `enabledAt` on the OFF→ON edge (the anchor `is_due` + measures from — without it, enabling a daily job fires it immediately).""" + raw = raw if isinstance(raw, dict) else {} + cron = _s(raw.get("cron"), 120).strip() or "0 6 * * *" + parse_cron(cron) # raises -> the route answers 400 + enabled = bool(raw.get("enabled")) + prev = previous or {} + out = {"cron": cron, "enabled": enabled} + if enabled: + out["enabledAt"] = (prev.get("enabledAt") if prev.get("enabled") else None) or _iso() + return out + + +# ── WAVE 23 · C5 — the ENDING, and the cycle counter that makes a loop countable. ───────────── +# The owner's words: "we can make this automation a loop, so the ending should always be defined, +# either it ends somewhere deterministic like Closed/Failed, or it goes to reset automatically, +# or the user have to click a button to reset, or after a certain amount of time it can +# automatically reset to first cycle." +# +# ⛔ `terminal` IS THE DEFAULT and every existing automation gets it, because a stored definition +# that predates this field must not start moving records on its own the day the code ships. A +# loop is a thing somebody turns on. +def clean_flow(raw, previous=None, notes=None, rt=None): + """Validate the builder's ordered action list. `(flow, error)`. + + ⚠ `rt` is the tenant wall for gated action kinds (W35-T35 / C8) and is simply forwarded. + """ + raw = raw if isinstance(raw, dict) else {} + prev = previous if isinstance(previous, dict) else {} + actions, err = clean_actions( + raw.get("actions") if "actions" in raw else prev.get("actions"), notes=notes, rt=rt) + if err: + return None, err + return {"actions": actions}, None + + +#: AMENDMENT A1 — what the `ig_profile_match` kind-flip seeds as `recordsLimit` when the +#: definition carries none. +#: +#: ⚠ 25 BECAUSE THE INPUT SAYS 25 (2026-08-06). It was 10 — the size wave 21 proved live at +#: $0.15 — while `AutomationDetail` initialises its own box to 25, and the two never met: the +#: flow node read "Up to 10 profiles · about $0.025" beside a field reading 25, on a freshly +#: created automation. Two numbers for one fact, in the panel, before anybody had typed +#: anything. **Read off the screenshot; every assertion in the battery was green.** +#: +#: This is the same species as the default-versus-guard split fixed the same day: a value +#: decided in one file and a value decided in another, describing the same thing. Aligning the +#: constants closes the only window in which they can disagree — the seed — because every later +#: state comes from the stored config. +DISCOVER_SEED_RECORDS = 25 + + +def discovery_seed_spec(kind): + """-> (default table, enrich action kind) for a discovery KIND. + + ⭐⭐ WAVE 30, OWNER REPORT 2026-08-12, verbatim: *"'When a Tiktok profile fits a criteria' + should ALWAYS have a 'Create record' EXACTLY like the Instagram one … THE ONLY DIFFERENCE IS + THE COLUMNS AND DATA SCHEMA"*. They were right, and the seeding path was the one family of + sites this wave widened everywhere ELSE: `clean_definition` planted step 1 and step 2 only when + the trigger was `ig_profile_match`, so a TikTok search stored `flow.actions = []` and had + nowhere to put what it found — MEASURED live against `auto_1` (Instagram: create_record + + enrich) versus a fresh TikTok discovery (empty). + + ⭐ THIS IS A LOOKUP, NOT A SECOND SEEDER, and that is the whole point of the ruling. One + builder plants both platforms' steps; the only things that vary are the table the record lands + in and which enrich action reads it — literally "the columns and data schema". A forked + `_ensure_tt_action` would start identical and drift, which is the failure `ENRICH_KINDS` and + `DISCOVERY_KINDS` were introduced to prevent one screen up. + + ⚠ Resolved in a FUNCTION rather than a module-level dict because `DISCOVER_TABLE` is defined + far below this line; a dict here would raise at import. + """ + if kind == "discover_tiktok": + return TT_PROFILE_TABLE, "enrich_tiktok" + return DISCOVER_TABLE, "enrich_instagram" + + +def _ensure_ig_action(flow_raw, table, enrich_kind="enrich_instagram"): + """Create record is PERMANENT step 1 on a DISCOVERY trigger — it cannot be deleted or moved. + + ⚠ THE NAME IS INSTAGRAM'S AND THE BEHAVIOUR IS BOTH PLATFORMS' (wave 30). Renaming it would + churn six gate references for no behaviour change; `enrich_kind` is what makes it general, and + it defaults to Instagram's so every pre-existing caller keeps its exact meaning. + + ⭐ OWNER RULING 2026-08-06, AND IT REVERSES WAVE 24's LAW 3. That law seeded this action once, + on the edge into the trigger, and ended "deleting it is their call, not a refusal" — with a + comment warning that re-seeding on every clean would be "a control that will not take no for + an answer". The owner's answer: *"should ALWAYS have a 'Create record' Step 1, that can't be + deleted, because the nature of that automation is that it needs to first create a record in a + database somewhere from the list of profiles to fetch."* + + Which is right, and the earlier reasoning had the category wrong: a flow whose TRIGGER + produces rows has nowhere to put them until something writes them, so an Instagram search + with no Create record is not a customised automation — it is a search whose results are + discarded. That is not a preference to respect. + + ⚠ IT KEEPS THE USER'S EDITS. Presence and POSITION are guaranteed; the table it writes to and + the values it maps are theirs. An existing `create_record` further down is MOVED to the front + rather than duplicated — re-seeding a second one would quietly double every run's writes. + """ + flow = dict(flow_raw or {}) if isinstance(flow_raw, dict) else {} + actions = [a for a in (flow.get("actions") or []) if isinstance(a, dict)] + if actions and actions[0].get("kind") == "create_record": + return _pin_unique(_ensure_enrich_step( + flow_raw if isinstance(flow_raw, dict) else flow, enrich_kind)) + at = next((i for i, a in enumerate(actions) if a.get("kind") == "create_record"), -1) + if at > 0: + actions.insert(0, actions.pop(at)) + else: + # The seed must be a config `clean_actions` ACCEPTS, or the builder shows a red banner + # and no card: the wave-23 header records four kinds that shipped with illegal seeds and + # did exactly that. `create_record` needs a ut_-prefixed table and at least one value. + actions.insert(0, { + "id": "act_1", "kind": "create_record", "enabled": True, "when": None, + # `config.label` follows the `review` action's precedent in `_clean_action_config` — + # an action naming itself, inside the untyped config bag, so no shared TS type changes. + "config": {"table": str(table or DISCOVER_TABLE), + "label": "Save the profile", + # C5: the picture and the real write finally agree — see `_pin_unique`. + "uniqueOn": "handle", + "values": {"handle": "{{handle}}"}}, + }) + flow["actions"] = actions + return _pin_unique(_ensure_enrich_step(flow, enrich_kind)) + + +def _ensure_enrich_step(flow_raw, enrich_kind="enrich_instagram"): + """⭐ 2026-08-07 (owner ruling) — ENRICH IS STEP 2 ON A DISCOVERY SEARCH. + + ⚠ WAVE 30: `enrich_kind` selects the network. Instagram's is the default so every pre-existing + caller means exactly what it meant before; TikTok passes `enrich_tiktok` and gets the identical + step shape, which is the owner's *"only the columns and data schema differ"*. + + Owner: *"make it default that this enrichment action is Step 2 always, and under Config of + Step 2 … we can have a toggle on or off."* Which is the same shape as step 1's ruling and for + the same reason: a search that finds profiles and never reads them has done half a job. The + difference is the control — step 1 is permanent because a flow without it discards its + results, while this one is permanent because the TOGGLE is how you turn it off. Deleting and + disabling would be two ways to say one thing, and only one of them survives a re-save. + + ⚠ SEEDED ON, AND ON THE FREE RUNG. `tier: "anonymous"` costs nothing, so a search that gains + this step by upgrading does not quietly start spending; switching the Source to the paid + provider is an explicit choice a person makes in front of the sentence that names the cost. + ⚠ THE COOLDOWN IS SEEDED ON TOO (30 days). On a table nobody has enriched it changes nothing — + a blank `enriched_at` is never "recent" — and the moment there IS history it stops the flow + re-buying the same profile nightly. Off-by-default would make the expensive behaviour the + accident. + + ⛔ THE EXISTENCE TEST WALKS THE FORKS (`walk_actions`). A person who moved enrichment inside an + If/then branch has one; seeding a second at the top level would enrich twice and bill twice, + which is the duplication `apply_actions` already paid for once with the pinned step 1. + """ + flow = dict(flow_raw or {}) if isinstance(flow_raw, dict) else {} + actions = [a for a in (flow.get("actions") or []) if isinstance(a, dict)] + if any(a.get("kind") == enrich_kind for a in walk_actions(actions)): + return flow_raw if isinstance(flow_raw, dict) else flow + seeded = list(actions) + # Index 1 — after the pinned Create record, because there is nothing to enrich until the + # profiles have been written as records. `insert` past the end is a plain append, so a flow + # with only step 1 lands this at the end, which IS step 2. + seeded.insert(1, { + "id": "act_enrich", "kind": enrich_kind, "enabled": True, "when": None, + "config": {"postMetrics": False, "commentMetrics": False, + "dryRun": False, "maxPosts": DEFAULT_POSTS_PER_PULL, + "fromView": "", "sortField": DEFAULT_ENRICH_SORT, "sortDir": "desc", + "limit": DEFAULT_ENRICH_LIMIT, "skipRecent": True, + "skipRecentDays": DEFAULT_ENRICH_COOLDOWN_DAYS}, + }) + return {**flow, "actions": seeded} + + +#: The key the discovery writer really upserts on. Both discovery runners key candidates on +#: `candidate_key(platform, handle)`; `handle` is the half a person can see and the half this action's +#: values carry, which is why the PINNED card shows that rather than the pair — `platform` is +#: supplied by the runner, never typed by a person. +#: ⛔ DEBT D-73 (closed wave 29): this note used to name wave 22's C6 compound — the one that +#: paired the handle with its finder — as the live key. Wave 26 · R4/R5 retired it: the identity is +#: the pair above and the TENANT is the unit, while `created_by` survives as an informational +#: "Found by" stamp that no run may branch on. +#: ⚠ The retired pair is DESCRIBED here and not reproduced, deliberately: a gate asserts this note +#: names the key `_ck` actually takes, and a verbatim quotation of the wrong one reads to that gate +#: exactly like the defect. A stale comment on correct code is how the next session reintroduces a +#: bug with a rationale attached. +IG_PINNED_UNIQUE = "handle" + + +def _pin_unique(flow): + """C5: keep `uniqueOn: "handle"` on the PINNED step 1 across every save. + + ⭐ WHY IT IS RE-STAMPED RATHER THAN MERELY SEEDED. `_clean_action_config` has no `previous` — + the builder posts the whole action list on every save — so a client that omitted the key would + silently reset it to `""`, i.e. back to append. The pinned card is a PICTURE of the engine's + own upsert (`apply_actions` skips it at runtime, see the note there), so the picture claiming + "append" while the engine upserts is exactly the surface-disagrees-with-the-engine defect this + module refuses everywhere else. + + ⛔ AND IT CANNOT MINT A CONFIG THE GUARD REFUSES — HARD RULE 11, which is the whole reason + this is a function and not one line. `_clean_action_config` refuses a `uniqueOn` the action + does not write, so the stamp is applied ONLY when `handle` is among the action's own values. + A user who remaps the card to write different columns keeps their edit and still saves; the + alternative — stamping unconditionally — is a product-seeded default that its own validator + would 400, which is precisely the post-W24 hotfix this rule exists because of. + + ⚠ NON-MUTATING, and that is load-bearing rather than tidiness. `clean_definition` passes + `prev.get("flow")` here when a PATCH carries no flow of its own — that is the STORED + definition's own dict, so writing into it would edit the live store object in memory, before + (and regardless of) any commit. Every touched level is copied instead. + """ + if not isinstance(flow, dict): + return flow + acts = flow.get("actions") + if not isinstance(acts, list) or not acts or not isinstance(acts[0], dict): + return flow + first = acts[0] + if first.get("kind") != "create_record": + return flow + cfg = first.get("config") + if not isinstance(cfg, dict) or IG_PINNED_UNIQUE not in (cfg.get("values") or {}): + return flow + if str(cfg.get("uniqueOn") or "").strip(): + return flow + return {**flow, + "actions": [{**first, "config": {**cfg, "uniqueOn": IG_PINNED_UNIQUE}}] + acts[1:]} + + +#: The two steps `_ensure_ig_action` / `_ensure_enrich_step` plant, as `(kind, id)`. Named here so +#: the seeder and the un-seeder cannot disagree about what "seeded" means. +#: ⚠ WAVE 30 — `enrich_tiktok` shares `act_enrich`. The map is keyed by KIND, and only one enrich +#: step is ever seeded per flow (the network follows the trigger), so the id cannot collide. +_IG_SEED_IDS = {"create_record": "act_1", "enrich_instagram": "act_enrich", + "enrich_tiktok": "act_enrich"} + + +def _is_untouched_ig_seed(action, table, enrich_kind="enrich_instagram"): + """Is this action still EXACTLY what the Instagram trigger planted? (wave 27 item 15) + + ⛔ THE COMPARISON IS AGAINST A FRESHLY BUILT SEED, not against a list of remembered keys, and + that is the only version that stays true: `_ensure_ig_action` and `_ensure_enrich_step` build + the same dicts one screen above, so a step gaining a config key next wave gains it here too. + A remembered key list would quietly start calling every seeded action "edited". + + ⛔ AND THE FRESH SEED GOES THROUGH `clean_actions` FIRST, which cost a red check to learn. The + stored action has been cleaned — `_clean_action_config` normalises it and adds the keys the + kind declares (`profileField: ""` on an enrich step, for one) — so comparing against the RAW + dict `_ensure_enrich_step` writes reports every seeded enrich step as "edited by a person", + and the un-seeding silently never fires for it. Comparing cleaned against cleaned is the only + version where the two sides are the same kind of object. + + ⚠ `enabled` IS DELIBERATELY NOT PART OF THE TEST for the enrich step. Its ruling says the + TOGGLE is how you turn it off — so a person who switched it off has expressed an opinion about + a step they still want, and an off seed is still a seed. + """ + if not isinstance(action, dict): + return False + kind = str(action.get("kind") or "") + if _IG_SEED_IDS.get(kind) != str(action.get("id") or ""): + return False + seeded, _seed_err = clean_actions( + (_ensure_ig_action({"actions": []}, table, enrich_kind).get("actions") or [])) + fresh = {a.get("kind"): a for a in (seeded or [])} + seed = fresh.get(kind) + if not seed: + return False + cfg, seed_cfg = dict(action.get("config") or {}), dict(seed.get("config") or {}) + if kind in ENRICH_KINDS: + cfg.pop("enabled", None) + seed_cfg.pop("enabled", None) + return cfg == seed_cfg and (action.get("when") or None) is None + + +def _drop_ig_seeds(flow_raw, table, enrich_kind="enrich_instagram"): + """Strip the trigger's own seeded steps, keeping any the person has since made theirs. + + ⭐⭐ WAVE 27 ITEM 15 (owner) — *"stale pinned create_record on trigger change"*. THE COMMENT + THAT USED TO SIT AT THE KIND FLIP ARGUED THE OPPOSITE and is rewritten there; it read: *"it + does not delete the seeded actions … a create_record the person has since re-pointed at their + own table with their own values is THEIR action now"*. That reasoning is still correct and is + exactly what this function preserves — but it was applied to EVERY seeded action, including + the ones nobody had ever opened, and the result is what the owner reported: switch an + Instagram search to Manual and you are left holding a "Save the profile" step writing + `{{handle}}` into `ut_ig_candidates` on a flow that no longer produces handles. That is not + somebody's work being protected; it is the machine's own leftover, pointed at a table the + automation has nothing to do with any more. + + ⛔ SO THE TEST IS "DID ANYBODY TOUCH IT", NOT "WAS IT SEEDED" — the same guard shape as + `_vestigial_name_field` and `_retired_tracked_field`, and for the same reason: this is a + branch that destroys something, so the conservative half is the load-bearing half. + """ + flow = dict(flow_raw or {}) if isinstance(flow_raw, dict) else {} + actions = [a for a in (flow.get("actions") or []) if isinstance(a, dict)] + kept = [a for a in actions if not _is_untouched_ig_seed(a, table, enrich_kind)] + if len(kept) == len(actions): + return flow_raw + return {**flow, "actions": kept} + + +def ig_action_pinned(defn, index): + """Is this action the one that cannot be removed? Derived, never stored — a stored `pinned` + flag is a second copy of the rule, and the copy is what a hand-written PATCH omits.""" + # ⭐ WAVE 30 — BOTH discovery kinds. It read `== "discover_instagram"`, so TikTok's step 1 (once + # seeded) would have rendered as an ordinary draggable, deletable card: the same rule the owner + # asked for "EXACTLY", applied to only one network. + return (defn or {}).get("kind") in DISCOVERY_KINDS and index == 0 + + +def clean_definition(raw, previous=None, username="", notes=None, rt=None): + """Whole-definition validation. Returns `(defn, error)`.""" + raw = raw if isinstance(raw, dict) else {} + prev = previous or {} + # ⭐ WAVE 24 · C-TRIG — THE TRIGGER IS RESOLVED FIRST NOW, because law 1 makes it the thing + # that CHOOSES the kind. Reading `raw["trigger"]["key"]` here instead would be a second + # reader of the trigger vocabulary, free to disagree with `clean_trigger` about which keys + # exist and which are refused. Gate-visible consequence, recorded in AMENDMENT A1: a payload + # invalid in BOTH its trigger and its config now answers with the trigger's sentence. + trigger, terr = clean_trigger(raw.get("trigger") if "trigger" in raw + else prev.get("trigger"), prev.get("trigger")) + if terr: + return None, terr + # C-TRIG law 2 (wave 24): a definition that names no kind is a `plain` one. Before R6 this + # refused with "unknown automation kind ''" — correct while a wizard always sent one, and a + # dead end the moment the wizard was deleted and the client's create body became {name}. + kind = _s(raw.get("kind") or prev.get("kind"), 40) or DEFAULT_KIND + drop_ig_seeds = False + if (trigger or {}).get("key") == "ig_profile_match": + # LAW 1: the trigger IS how `discover_instagram` gets chosen now. Unconditional rather + # than "when the kind is unset" — a definition whose trigger says Instagram-discovery and + # whose kind says something else is not a preference to respect, it is two halves of one + # answer disagreeing, and the trigger is the half the person actually picked. + kind = "discover_instagram" + elif (trigger or {}).get("key") == "tiktok_profile_match": + # ⭐ WAVE 29 (D-9 / R1) — LAW 1, TIKTOK'S HALF. Same rule, same reason: the trigger is how + # `discover_tiktok` gets chosen, and it is unconditional for the same reason the Instagram + # arm is — a definition whose trigger says TikTok discovery and whose kind says something + # else is two halves of one answer disagreeing. + kind = "discover_tiktok" + elif (kind == "discover_tiktok" + and ((prev.get("trigger") or {}) or {}).get("key") == "tiktok_profile_match" + and (trigger or {}).get("key") != "tiktok_profile_match"): + # ⛔⛔ LAW 1'S INVERSE, AND ITS ABSENCE ON THE INSTAGRAM SIDE WAS A MONEY BUG (see the arm + # below): with nothing to flip the kind BACK, switching the trigger to Manual left + # `RUNNERS[kind]` pointing at the discovery runner, so **Run now fired a PAID corpus + # search on a flow the person had just made manual.** Shipped here in the same change as + # law 1 rather than discovered the same way twice. + # ⚠ THE TEST IS THAT THE TRIGGER **MOVED** — comparing the RESOLVED key against the + # PREVIOUS one. Asking `"trigger" in raw` would fire on every empty patch, because + # `patch()` builds its raw as `dict(prev)` plus the caller's keys. + kind = DEFAULT_KIND + # ⭐ WAVE 30 — un-seed on the way out, exactly as the Instagram arm below does. This line + # was ABSENT and harmless for as long as TikTok had no seeds to leave behind; the moment + # the seeder above learned TikTok, its absence became wave-27 item 15's defect on the other + # network — a flow switched to Manual still carrying a "Save the profile" step nobody + # planted deliberately. Found by widening the seeder and asking what else assumed only + # Instagram could have seeds. + drop_ig_seeds = True + elif (kind == "discover_instagram" + and ((prev.get("trigger") or {}) or {}).get("key") == "ig_profile_match" + and (trigger or {}).get("key") != "ig_profile_match"): + # ⭐⭐ 2026-08-07 (owner report) — LAW 1 HAS AN INVERSE, AND ITS ABSENCE WAS A MONEY BUG. + # + # Law 1 above flips the kind TO `discover_instagram` when the trigger says Instagram + # discovery. Nothing flipped it BACK. So `kind` was inherited from `prev` forever: switch + # the trigger to Manual and the automation stayed `discover_instagram`, which means + # + # · `RUNNERS[kind]` is still `run_discover_instagram`, so pressing **Run now** on a flow + # the person had just made MANUAL fired a PAID Bright Data corpus search; + # · `ig_action_pinned` keys on the kind, so the seeded "Create record" stayed + # UNDELETABLE — the owner's report, verbatim: *"When a trigger change from the + # instagram trigger, the 'always first' create record just stuck there"*. The server + # would have accepted the delete; the CLIENT hid the control, because both halves ask + # the kind and the kind was lying. + # + # ⛔ THE TEST IS THAT THE TRIGGER **MOVED**, and the first version of this got it wrong in + # a way worth recording. It asked `"trigger" in raw` — but `patch()` builds its raw as + # `merged = dict(prev)` plus the caller's keys, so **`"trigger"` is present on EVERY + # patch**, and an empty `patch(rt, id, {})` flipped a discovery automation to `plain`. + # That turned five `section_bd` checks red and crashed the suite on a `StopIteration` + # three sections later. Comparing the RESOLVED key against the PREVIOUS one is the honest + # question: did this automation stop being an Instagram search? + # + # ⚠ NARROW ON PURPOSE, twice over. It fires only when the automation WAS on the Instagram + # trigger — a `discover_instagram` created with an explicit kind and some other trigger is + # somebody's deliberate state, not a mistake to correct. And only FROM + # `discover_instagram`: `scrape_db` and `field_instagram` are surviving kinds whose + # triggers are their own business, and clobbering them would retire them by accident. + # + # ⭐⭐ WAVE 27 ITEM 15 — IT NOW DROPS THE SEEDS NOBODY TOUCHED, and the paragraph this + # replaces argued the other way, so here is why it was half right. It read: *"it does not + # delete the seeded actions … a create_record the person has since re-pointed at their own + # table with their own values is THEIR action now, and quietly destroying it is the silent + # data loss this module refuses everywhere else."* Every word of that is still true and is + # exactly what `_is_untouched_ig_seed` protects. What it got wrong was applying the + # protection to actions NOBODY HAD EVER OPENED: the owner's report is a flow switched to + # Manual still carrying a "Save the profile" step writing `{{handle}}` into + # `ut_ig_candidates` — the machine's own leftover on a flow that no longer produces + # handles, unpinned but still there, still runnable, and still pointed at a table that has + # nothing to do with this automation. + # ⚠ APPLIED BELOW, at the flow, not here: `flow_raw` is not resolved yet at this line. + kind = DEFAULT_KIND + drop_ig_seeds = True + if kind not in KINDS: + return None, f"unknown automation kind {kind!r}" + # ⛔ DEBT D-65 — THE REFUSAL THAT BELONGS HERE IS NOT SHIPPED, AND THAT IS A DECISION. + # D-65 offers two exits for `field_instagram`: convert a surviving definition to a `plain` + # flow carrying `enrich_instagram`, or REFUSE it here with a sentence naming the replacement. + # The refusal was built and then REVERTED, because W24/R6 deliberately made `patch()` the way + # a retired kind legally comes into existence — `create` refuses, `patch` accepts, and the two + # live automations save through this function every time their owner edits them. The gate's + # own fixture helper says so in the strongest terms available: *"If R6's refusal ever moved + # from `create` into `clean_definition` (the tempting simplification), this helper would go + # red across a dozen sections, which is the alarm that change deserves."* It did, and the + # alarm worked. + # ⚠ WHAT SHIPPED INSTEAD is the half that costs nothing and was the actual complaint: every + # door that DOES refuse now names the replacement (`RETIRED_KIND_REPLACEMENT`), so nobody + # meets "unknown automation kind" for a decision made on purpose. The rest of D-65 is an + # owner-visible behaviour change — either new `field_instagram` mints stop working, or stored + # ones are rewritten under their owner — and that is a ruling, not a refactor. + name = " ".join(_s(raw.get("name") or prev.get("name") or "", MAX_NAME).split()) + if not name: + return None, "name the automation" + cfg_raw = raw.get("config") if isinstance(raw.get("config"), dict) else prev.get("config") + if kind in DISCOVERY_KINDS and prev.get("kind") != kind: + # ⭐⭐ WAVE 30 · T04 — WIDENED FROM `discover_instagram` TO BOTH DISCOVERY KINDS, AND THIS + # ONE LINE WAS THE WHOLE OF THE OWNER'S ITEM 3. Picking "When a TikTok profile fits a + # criteria" 400'd on the very first Save with *"say how many profiles to fetch — an + # UNBOUNDED discovery query is the one shape the vendor refuses outright + # (NOT_ENOUGH_FUNDS)"*, and the trigger was therefore never STORED, which is what the + # owner saw as "the Trigger won't load". + # ⛔ INSTAGRAM WAS NEVER SURVIVING ON ITS OWN MERITS: `AutomationDetail.buildConfig` sends + # no `recordsLimit` for EITHER platform on a fresh automation (its `kind` is still `plain` + # at that moment, so it falls through to `return { targetTable }`). IG worked only because + # this seed caught it. So the defect was never "TikTok is missing something Instagram has" + # — it was one hard-coded string in the single line that rescues both. + # ⚠ `prev.get("kind") != kind` is the EXACT generalisation of the old + # `prev.get("kind") != "discover_instagram"`, not a loosening: it still fires only on a + # kind FLIP, so a later save that carries a real limit is not re-seeded (asserted). + # AMENDMENT A1: the kind FLIP seeds the one field `clean_config` insists on, or the very + # first Save after picking this trigger 400s — against the stored-inert-with- + # `configured:false` pattern the whole picker is built on (see `clean_trigger`'s A3 note). + # ⛔ THE REFUSAL ITSELF IS UNTOUCHED: an explicit 0 still gets its sentence. That guard + # exists because an unbounded query is the one shape the vendor refuses outright, and + # coercing a blank into a number would be exactly the silent widening it protects against. + # Seeding cannot spend — `clean_schedule` defaults `enabled` False, so nothing runs until + # a person presses Run now or arms the schedule. + cfg_raw = dict(cfg_raw or {}) + if not _ig_int(cfg_raw.get("recordsLimit")): + cfg_raw["recordsLimit"] = DISCOVER_SEED_RECORDS + config, err = clean_config(kind, cfg_raw, prev.get("config")) + if err: + return None, err + if trigger: + # A2: re-ask completeness now that the config is validated — `ig_profile_match`'s own + # configuration IS the config, and `clean_trigger` could not see it. + trigger["configured"] = _trigger_configured(trigger, config) + try: + schedule = clean_schedule( + raw.get("schedule") if isinstance(raw.get("schedule"), dict) + else prev.get("schedule"), prev.get("schedule")) + except ValueError as e: + return None, str(e) + # (`clean_trigger` ran at the top — law 1 needs the trigger before the kind.) + flow_raw = raw.get("flow") if "flow" in raw else prev.get("flow") + # ⭐ EVERY CLEAN, NOT ONLY THE EDGE (owner ruling 6 — see `_ensure_ig_action`). The edge-only + # call was what made the action deletable; running it on every save is what makes step 1 + # permanent, and it is deliberate rather than a widened condition nobody noticed. + # ���⭐ WAVE 30 (owner, 2026-08-12) — BOTH DISCOVERY TRIGGERS SEED, not just Instagram's. This + # single `==` was the whole of *"why is it not copied EXACTLY"*: a TikTok search stored an + # EMPTY flow, so the product's own rule — a search that finds profiles must have somewhere to + # put them — held for one network and not the other. + if (trigger or {}).get("key") in DISCOVERY_TRIGGER_KIND: + _seed_table, _seed_enrich = discovery_seed_spec(kind) + flow_raw = _ensure_ig_action( + flow_raw, config.get("targetTable") or _seed_table, _seed_enrich) + elif drop_ig_seeds: + # ITEM 15: the trigger just STOPPED being a discovery one. Un-seed what the trigger + # planted and nobody has since made theirs — measured against the table the seeds were + # planted for, which is the PREVIOUS config's, not the one being saved. + # ⚠ And against the PREVIOUS kind's enrich action, for the same reason: the seeds to strip + # are TikTok's if that is what was planted, and asking "is this Instagram's seed?" of a + # TikTok flow answers no and silently leaves the leftover behind. + _prev_table, _prev_enrich = discovery_seed_spec(prev.get("kind")) + flow_raw = _drop_ig_seeds( + flow_raw, (prev.get("config") or {}).get("targetTable") or _prev_table, _prev_enrich) + flow, ferr = clean_flow(flow_raw, prev.get("flow"), notes=notes, rt=rt) + if ferr: + return None, ferr + # ⭐ WAVE 30 — widened with the seeder above: TikTok now HAS a pinned step 1, so the rule that + # its `config.table` is authoritative has to reach it, or the two fields that name one database + # would disagree on exactly the network that just gained the card. + if (trigger or {}).get("key") in DISCOVERY_TRIGGER_KIND: + # ⭐ WAVE 25 · R2 — THE CREATE RECORD ACTION'S `config.table` IS AUTHORITATIVE, and + # `config.targetTable` FOLLOWS IT. Two fields have been naming the same database since + # wave 24: the pinned step 1 says where profiles are written, and the config says where + # the automation's records live — and for a discovery flow those are the same database by + # construction. When they disagreed, every surface picked a different winner (the canvas + # Write node read the config, the actual write read the action), which is a bug you can + # only see by comparing two panels. + # + # ⛔ THE ACTION WINS BECAUSE IT IS THE ONE A PERSON EDITS. R2b puts the preset list on the + # action's own configuration, so the table named there is the one they were looking at. + # + # ⚠ THIS IS ALSO THE MIGRATION, and it is a migration in the shape laws 4/5 established: + # a stored definition carrying two different values is reconciled on its next clean, + # silently and exactly once, because nothing writes the losing value back. A refusal here + # would 400 every Save of a definition that was legal yesterday. + # ⭐⭐ WAVE 27 ITEM 11 — WHICHEVER SIDE MOVED WINS, and R2's "the action wins" is now the + # TIE-BREAK rather than the whole rule. The owner's report: change the database on the + # TRIGGER, save, and the picker springs back to the old one with no message. The cause is + # the branch below read unconditionally — the pinned action still named yesterday's table, + # so it overwrote a change the person had just made in front of it, silently, every time. + # + # ⛔ R2 IS NOT REVERSED, and the distinction is which QUESTION the code asks. R2 settled + # "when the two disagree, whose value is real?" — the action's, because R2b puts the + # preset list on the action's own panel, so that is the one they were looking at. That + # answers a definition at REST. A save is different: it carries an INTENT, and the honest + # question is which side this particular save changed. So: + # · the trigger-side picker moved -> it wins, and the pinned action is re-pointed to + # follow it (leaving the action behind would just re-revert on the next save); + # · only the action moved, or neither did and they were already inconsistent -> R2's + # migration, unchanged, reconciling a stored definition exactly once. + # · BOTH moved in one payload -> the action still wins. That is R2 literally, and it is + # also the only reading that cannot lose a value: the action's table is the one with + # the field mapping attached to it. + first = (flow.get("actions") or [{}])[0] + pinned = str((first.get("config") or {}).get("table") or "") \ + if first.get("kind") == "create_record" else "" + prev_target = str((prev.get("config") or {}).get("targetTable") or "") + prev_pinned = str(((((prev.get("flow") or {}).get("actions") or [{}])[0] + ).get("config") or {}).get("table") or "") + target_moved = bool(prev_target) and config.get("targetTable") != prev_target + action_moved = bool(prev_pinned) and pinned != prev_pinned + if pinned and target_moved and not action_moved: + # The person changed the trigger's database. Carry it INTO the pinned step so the two + # agree, instead of throwing their edit away to keep a stale copy consistent. + cfg1 = dict(first.get("config") or {}) + cfg1["table"] = str(config.get("targetTable") or "") + flow["actions"] = [{**first, "config": cfg1}, *(flow.get("actions") or [])[1:]] + elif pinned and pinned != config.get("targetTable"): + config["targetTable"] = pinned + # The stored label described a DIFFERENT database, so keeping it would caption the new + # one with the old one's name. Blank falls back to the key everywhere, and `ut_ensure` + # never relabels a table that already exists — so an adopted hand-made database keeps + # the name its owner gave it. + config["targetLabel"] = "" + return { + "id": _s(prev.get("id") or raw.get("id"), 40), + "name": name, "kind": kind, "config": config, "schedule": schedule, + "trigger": trigger, + # The Builder's ordered actions. An absent flow is simply an automation with no actions. + "flow": flow, + "status": prev.get("status") or {"state": "idle", "lastRunAt": "", "lastSummary": ""}, + "runs": list(prev.get("runs") or [])[:MAX_RUNS], + # ⚠ ENGINE-OWNED CONTINUATION STATE, and it is NOT config. A discovery snapshot takes ~20 + # minutes to build, so a run persists its id here and the NEXT run collects it. It is + # carried through `patch` untouched because a user pressing Save must not silently orphan + # a set the vendor is already building (and already counting against the funds gate). + "state": dict(prev.get("state") or {}), + "created": prev.get("created") or _iso(), + "createdBy": prev.get("createdBy") or username, + # ⭐⭐ WAVE 35 · T37 — `system` IS STICKY ACROSS A PATCH, AND IT IS **NEVER READ FROM `raw`**. + # + # ⛔ THE BUG THIS CLOSES WAS ALREADY WRITTEN AND WOULD HAVE SHIPPED. `routes_automation. + # delete_automation` refuses any stored row carrying `system`, which is what makes a seeded + # system agent undeletable — and this function is a WHITELIST that did not carry the key, so + # the FIRST EDIT of such an agent silently stripped its marker and made it deletable. The + # seed would then mint it again on the next list call: a delete that appears to work and + # undoes itself. Found by asserting the round trip rather than by reading the code. + # + # ⛔ FROM `prev` ONLY. Reading it from `raw` would let ANY caller POST + # `{"system": "anything"}` and mint themselves an automation nobody can delete — a + # privilege escalation through a field nobody validates. The flag is granted by the seeder + # and inherited from the stored row, never asserted by a payload. + # ⚠ Same shape and same reason as the pre-set FIELD flag, which DESIGN.md §4 already + # describes as "sticky across a PATCH". One idea, one behaviour, two objects. + **({"system": _s(prev.get("system"), 40)} if prev.get("system") else {}), + }, None + + +def set_state(rt, auto_id, patch_state): + """Merge into ONE automation's engine state. A key set to None is removed. + + Its own tiny store write rather than a field on `_commit_run`, because the handoff must + survive a run that ends in `error` — a snapshot the vendor is building does not stop existing + because the run that started it failed afterwards. + """ + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + d = cur.get(str(auto_id)) + if d is None: + return cur + st = dict(d.get("state") or {}) + for k, v in (patch_state or {}).items(): + st.pop(k, None) if v is None else st.__setitem__(k, v) + d["state"] = st + return cur + + # ⚠ W31-T35 — THE ONE `keeps_defs=True` IN THIS MODULE. `set_state` writes `state` and never a + # trigger, and `grid_hook` calls it once per CREATED RECORD; clearing the definitions memo here + # would re-read the whole bucket one row later than before and undo D-134 entirely. Safe + # because `grid_hook` reads exactly one state key (`rcHighwater`) and mirrors its own write + # into the memo in the same statement. Read `_store_update`'s note before adding a second. + _store_update(rt, _up, flush="sync", keeps_defs=True) + + +def _without_retired_board(definition): + """Return a definition with retired Board-only state removed, without mutating the store. + + This protects scheduled/manual execution before the next UI list request persists the + migration. It removes only the former Board configuration, actions, and audit trail; normal + actions and all user data remain intact. + """ + if not isinstance(definition, dict): + return definition, False + out, changed = dict(definition), False + cfg = out.get("config") + if isinstance(cfg, dict) and "lanes" in cfg: + cfg = dict(cfg) + cfg.pop("lanes", None) + out["config"] = cfg + changed = True + + def _actions(actions): + nonlocal changed + clean = [] + for action in actions if isinstance(actions, list) else []: + if not isinstance(action, dict): + clean.append(action) + continue + if action.get("kind") == "review": + changed = True + continue + item = dict(action) + if item.get("kind") == "group" and isinstance(item.get("config"), dict): + group_cfg = dict(item["config"]) + branches = group_cfg.get("branches") + if isinstance(branches, list): + kept = [] + for branch in branches: + if not isinstance(branch, dict): + changed = True + continue + next_actions = _actions(branch.get("actions")) + if not next_actions: + changed = True + continue + kept.append({**branch, "actions": next_actions}) + if not kept: + changed = True + continue + if kept != branches: + changed = True + group_cfg["branches"] = kept + item["config"] = group_cfg + elif isinstance(group_cfg.get("actions"), list): + nested = _actions(group_cfg["actions"]) + if not nested: + changed = True + continue + if nested != group_cfg["actions"]: + group_cfg["actions"] = nested + item["config"] = group_cfg + clean.append(item) + return clean + + flow = out.get("flow") + if isinstance(flow, dict): + next_flow = dict(flow) + actions = _actions(flow.get("actions")) + if actions != flow.get("actions"): + next_flow["actions"] = actions + if "ending" in next_flow: + next_flow.pop("ending", None) + changed = True + out["flow"] = next_flow + if "reviews" in out: + out.pop("reviews", None) + changed = True + return out, changed + + +def retire_automation_board_state(rt, tables=None): + """Persist the idempotent Board retirement, then remove its generated table fields. + + ``tables`` is the optional lent snapshot of the `user_tables` bucket (W29-T01) — it reaches + the stage scan and nothing else. + """ + fields = retire_automation_stage_fields(rt, tables=tables) + try: + stored = dict(rt.get(STORE_KEY) or {}) + except Exception: + return {"definitions": 0, **fields} + plan = {str(aid): cleaned for aid, raw in stored.items() + for cleaned, changed in [_without_retired_board(raw)] if changed} + if not plan: + return {"definitions": 0, **fields} + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + for aid, cleaned in plan.items(): + if aid in cur: + cur[aid] = cleaned + return cur + + _store_update(rt, _up, flush="sync") + return {"definitions": len(plan), **fields} + + +#: ⭐⭐ WAVE 31 · T35 (D-134) — the definitions memo, and the two things that make it safe. +#: `tenant -> (monotonic_at, defs)`. Read ONLY through `all_definitions(..., cached=True)`, which +#: exactly one caller uses (`grid_hook`). +_DEFS_MEMO = {} +#: Deliberately SHORT. This exists to collapse one burst of row events into one read, not to be a +#: cache — an import of 20,000 rows arrives in far less than this, and a stale trigger for two +#: seconds is bounded and recoverable where a stale one for a minute is a mystery. +_DEFS_TTL = 2.0 + +# The durable external scheduler has a different read pattern from a row-event burst. A live +# Space is one process / one replica today, and every automation-definition write in this module +# passes through `_store_update`, so the scheduler can retain definitions until a REAL write +# invalidates them. This is the cost boundary for Neon: an idle 15-minute wake-up must not turn +# into 96 reads/day of the same tenant JSON value merely because a timer fired. +_TICK_DEFS_MEMO = {} +_TICK_CACHE_LOCK = threading.RLock() + + +def _store_update(rt, fn, flush="sync", keeps_defs=False): + """THE one door every write to the automations bucket goes through — and the ONLY reason it + exists is that the memo's invalidation must be DERIVED rather than remembered. + + ⚠ `keeps_defs=True` IS AN EXPLICIT, SINGLE-SITE OPT-OUT, and it is here because the safe + default would otherwise make the memo useless on the exact path it was built for. `set_state` + writes `state` and NEVER a trigger, and it is called by `grid_hook` itself once per created + record — so clearing on it would mean a 20,000-row import re-read the bucket 20,000 times + anyway, one row later than before. The opt-out is safe because `grid_hook` reads exactly one + state key (`rcHighwater`) and mirrors its own write into the memo in the same statement. + ⛔ The DEFAULT is the safe one, so a thirteenth writer added next wave gets invalidation + without knowing this exists; only a caller that has read this paragraph can opt out. + + ⛔ THE ALTERNATIVE WAS TWELVE CALL SITES. `rt.update(STORE_KEY, …)` appeared twelve times in + this module; hanging an invalidation on each would be a hand-maintained list, and the way that + fails is silent: a thirteenth writer added next wave leaves `grid_hook` firing triggers off a + definition set that no longer exists, with nothing red. Same defect class as `patch`'s + hand-maintained patchable-key list, which this module already documents as *"the silent-drop + seat"* [[a-constant-two-features-share]]. + + ⚠ IT CLEARS THE WHOLE MEMO, not this tenant's row, on purpose: identifying the tenant here + would mean trusting `getattr(rt, 'key')` at a WRITE, and a wrong answer there is a stale + trigger set for another tenant. The memo holds at most a handful of entries and the correct, + boring thing costs nothing. + """ + if not keeps_defs: + _DEFS_MEMO.clear() + with _TICK_CACHE_LOCK: + _TICK_DEFS_MEMO.clear() + return rt.update(STORE_KEY, fn, flush=flush) + + +def all_definitions(rt, cached=False): + """Every automation definition for this tenant. + + ⭐⭐ `cached=True` IS D-134's FIX, AND IT IS OPT-IN FOR A REASON. `grid_hook` runs ONCE PER ROW + EVENT and called this unconditionally, so a 20,000-row import performed 20,000 whole-document + deep copies of the automations bucket — `Store.get` re-serialises on every call, hit or miss, + under the store lock. Every other caller is a request handler that reads it once, so widening + the memo to all of them would trade a real guarantee (a request sees the current store) for + nothing measurable. + + ⛔ THE MEMO IS THE SAME OBJECT ON A HIT, and `grid_hook` MUTATES it deliberately — see the + `rcHighwater` write there. That is not a leak of an implementation detail, it is the fix to the + hazard a naive memo creates: `grid_hook` both READS `state.rcHighwater` and WRITES it through + `set_state`, so a memo that went stale against its own write would re-fire `record_created` + for records it had already handled, breaking A2(4)'s *"once per record EVER — undo-proof"*. + Writes through `_store_update` drop the memo; the one write `grid_hook` makes to its OWN copy + is mirrored into it in the same statement. + """ + if cached: + key = str(getattr(rt, "key", "") or "") + hit = _DEFS_MEMO.get(key) + if hit is not None and (time.monotonic() - hit[0]) < _DEFS_TTL: + return hit[1] + try: + raw = dict(rt.get(STORE_KEY) or {}) + except Exception: + return {} + out = {str(aid): _without_retired_board(defn)[0] for aid, defn in raw.items()} + if cached: + _DEFS_MEMO[str(getattr(rt, "key", "") or "")] = (time.monotonic(), out) + return out + + +def tick_definitions(rt): + """Definitions for the external scheduler, cached until `_store_update` changes them. + + Live has one replica, all product writes use `_store_update`, and a restart naturally drops + this process cache. The existing two-second memo remains dedicated to row-event bursts. + """ + key = str(getattr(rt, "key", "") or "") + with _TICK_CACHE_LOCK: + hit = _TICK_DEFS_MEMO.get(key) + if hit is not None: + return hit + fresh = all_definitions(rt) + with _TICK_CACHE_LOCK: + return _TICK_DEFS_MEMO.setdefault(key, fresh) + + +def invalidate_tick_cache(tenant=None): + """Drop scheduler control-plane state after an out-of-process tenant change.""" + with _TICK_CACHE_LOCK: + if tenant is None: + _TICK_DEFS_MEMO.clear() + _TICK_TENANT_CACHE.clear() + else: + _TICK_DEFS_MEMO.pop(str(tenant), None) + + +def _new_id(existing): + n = 1 + while f"auto_{n}" in existing: + n += 1 + return f"auto_{n}" + + +#: C2's three answers to "which database does this automation work on?" — the FIRST question the +#: create wizard asks (owner item 3: the kind is not the first thing a person picks, the data is). +TARGET_MODES = ("existing", "new", "automated") + + +def resolve_target(rt, raw, username=""): + """C2: turn the wizard's `target` into a bound table key. Returns `(config_patch, error)`. + + - `existing` — bind a database that is already there. + - `new` — mint a blank one now, so the automation has somewhere to write before its first + run instead of conjuring a table the person never agreed to. + - `automated` — leave it to the runner: a scraping automation MINTS its target on first run + (`ut_ensure`), stamped `source: "Automation"`, which is what makes it a machine database. + Nothing is created here, deliberately — an empty table created up front for a scrape that + never runs is litter nobody can explain. + """ + if not isinstance(raw, dict) or not raw: + return {}, None # no target block = the pre-wizard shape + mode = _s(raw.get("mode"), 20).strip() or "existing" + if mode not in TARGET_MODES: + return None, f"{mode!r} is not one of: " + ", ".join(TARGET_MODES) + if mode == "automated": + label = " ".join(_s(raw.get("label"), 60).split()) + return ({"targetLabel": label} if label else {}), None + if mode == "existing": + key = _s(raw.get("table"), 60).strip() + if not key: + return None, "choose the database this automation works on" + t = ut_get(rt, key) + if t is None: + return None, f"{key!r} is not a database in this workspace" + return {"targetTable": key, "targetLabel": t.get("label") or key}, None + label = " ".join(_s(raw.get("label"), 60).split()) + if not label: + return None, "name the new database" + import core.user_tables as _ut_new + key = _ut_new.create(label, username or "automation", st=rt) + if not key: + return None, ("the database could not be created. This workspace may be at its table " + "limit") + return {"targetTable": key, "targetLabel": label}, None + + +def _unmint(rt, key): + """Delete a database this call minted moments ago, because the call is refusing (D-50). + + ⚠ FAIL-QUIET BY DESIGN. The caller is already returning a refusal with a sentence the user + needs to read; a rollback that raised would replace that sentence with a 500 and lose the + reason the create failed in the first place. The worst case of a swallowed failure here is + the orphan we had before — strictly no worse, and the refusal still reaches the person. + """ + if not key: + return + try: + import core.user_tables as _ut_del + _ut_del.delete(key, st=rt) + except Exception: # noqa: BLE001 + pass + + +def create(rt, raw, username="", notes=None): + existing = all_definitions(rt) + if len(existing) >= MAX_AUTOMATIONS: + return None, f"this workspace is at the {MAX_AUTOMATIONS}-automation limit" + raw = dict(raw or {}) + # ⭐ WAVE 24 — DEBT D-50. `resolve_target` MINTS a database for `mode:"new"`, and validation + # happens after it, so a correct refusal ("a source URL is required") used to leave an + # orphaned `ut_*` table behind with no automation pointing at it. MEASURED at the W23 close: + # three refused creates, two orphaned databases, deleted by hand. A person mis-filling a form + # twice silently accumulated empty databases in their nav. + # + # ⛔ THE ROLLBACK IS DERIVED, NOT DECLARED, and that is the point of doing it this way. It + # does not test for `mode == "new"`; it asks "did the table this call's target resolved to + # exist BEFORE this call?" So it closes the hole for any future target mode that mints, and + # a fifth mode cannot reopen it by forgetting a branch. Scoped to THIS call's own target + # rather than "any table that appeared", so a concurrent create in another request is never + # collateral. + before = set(ut_all(rt) or {}) + patch_cfg, terr = resolve_target(rt, raw.pop("target", None), username) + if terr: + return None, terr + minted = str((patch_cfg or {}).get("targetTable") or "") + minted = minted if minted and minted not in before else "" + if patch_cfg: + raw["config"] = {**(raw.get("config") or {}), **patch_cfg} + # 2026-08-10 — point a targetless discovery flow at the profile database this tenant ALREADY + # has, instead of letting the pure validator mint a second empty one. See + # `discover_default_table`; a target the caller named is never rewritten. + raw = _apply_discover_default(rt, raw) + # W35-T35 (C8): `rt=` is the tenant wall for gated action kinds. Both doors pass it; a door + # that forgot would REFUSE the kind, not admit it (`TENANT_GATED_ACTIONS` is fail-closed). + defn, err = clean_definition(raw, None, username, notes=notes, rt=rt) + if err: + _unmint(rt, minted) + return None, err + # ⭐ WAVE 24 · OWNER RULING R6 — "scrape_db and field_instagram survive on the automations + # that already use them and NO NEW ONE CAN BE CREATED." + # + # ⛔ HERE, IN `create`, AND NEVER IN `clean_definition`. A PATCH of one of the two live + # automations runs through `clean_definition` with `prev["kind"]` already set, so refusing + # the kind there would make both of them unsaveable — the ruling retires the door, not the + # automations behind it. Enforced on the server rather than left to the deleted wizard: a + # wall that holds only because the client stopped asking is a convention, not a wall. + if defn["kind"] in RETIRED_KINDS: + _unmint(rt, minted) + return None, (f"{KIND_LABELS.get(defn['kind'], defn['kind'])!r} automations are no " + f"longer created. The ones already using it keep working. " + f"{RETIRED_KIND_REPLACEMENT.get(defn['kind'], '')}") + defn["id"] = _new_id(existing) + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + cur[defn["id"]] = defn + return cur + + # `async` for the reason spelled out at `remove` and applied at `patch` — a create is the same + # blocking commit inside the same request, and the person is waiting on it in the same way. + # ⚠ NO PRESET GUARD HERE, deliberately: `patch` may skip the spawn because it can compare + # against a PREVIOUS definition, and a create has none — R10's columns must exist before a run, + # so this call stays unconditional. + _store_update(rt, _up, flush="async") + _presets_after_write(rt, defn, username) # R10: the columns exist before a run + if defn.get("trigger"): + _seed_event_state(rt, defn) + return defn, None + + +def patch(rt, auto_id, raw, username="", notes=None): + existing = all_definitions(rt) + prev = existing.get(str(auto_id)) + if prev is None: + return None, "no such automation" + merged = dict(prev) + # ⚠ A HAND-MAINTAINED PATCHABLE-KEY LIST, and it is the silent-drop seat of this module: a + # key missing here is accepted by the route, validated by `clean_definition`, and then + # DISCARDED — the client shows a saved flow that the store never received, and nothing goes + # red. `flow` joined it in the same edit that created `flow` (wave 23), which is the only + # ordering that never has a window where the bug exists. + for k in ("name", "config", "schedule", "kind", "trigger", "flow"): + if k in (raw or {}): + merged[k] = raw[k] + # 2026-08-10 — the PATCH door needs it too, and this is the one that actually bit: picking the + # Instagram trigger on an existing automation flips the kind, and the very next Save runs a + # config that has never carried a target through the pure validator, which mints the empty + # `ut_ig_candidates`. `create` alone would have left the commonest path untouched. + merged = _apply_discover_default(rt, merged) + defn, err = clean_definition(merged, prev, username, notes=notes, rt=rt) + if err: + return None, err + defn["id"] = str(auto_id) + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + cur[defn["id"]] = defn + return cur + + # ⛔ WHY THIS IS `async` — owner report 2026-08-12 (wave 30), MEASURED on the deployed build. + # Owner, verbatim: *"debug Automation module, its extremely slow, even renaming an Automation + # takes forever"*. A rename paid a BLOCKING HF commit inside the request, and `Store.update`'s + # sync branch holds `self._lock` across a strict DOWNLOAD **and** the upload — the same lock + # `Store.get` takes — so one rename stalls every OTHER reader of that tenant's store for its + # whole duration. That is the "module is slow" half a per-route fix never reaches. + # The reasoning, the read-your-writes contract and the `user_tables.add_row` precedent are + # written out in full at `remove` above (same wave, same owner report); this is that ruling + # applied to the save path rather than a second argument for it. + _store_update(rt, _up, flush="async") + # ⛔ THE PRESET SPAWN RUNS ONLY WHEN ITS OWN INPUTS MOVED — the other half of the same report. + # MEASURED as `nurilab-admin` on `4258a93`: **7,912 ms** for one rename against a **288 ms** + # warm `GET /automations` on the same connection, i.e. 27x the read this wave just made fast + # (T12/T13). A rename changes `name`; `_presets_after_write` reads `config`, `flow` and + # `trigger` and NOTHING else, then pays a full `user_tables` read (`ut_get`/`ut_ensure`; + # `rt.get` deep-copies by design, documented ceiling 35.8 MB / ~1.4 s) to re-derive columns + # that cannot have changed. ⭐ The generalisable half: T12/T13 took `ut_all` off the automation + # READ path this wave and every sibling WRITE path still carried it — a performance fix scoped + # to "the slow page" leaves the same call in every route beside it + # [[read-a-gates-predicate-for-what-it-excludes]]. + # ⛔ THE DANGEROUS DIRECTION IS SKIPPING, NEVER RUNNING, so the predicate is the INPUT SET of + # the function being skipped — not a hand-listed set of "cheap" edits, which is how a guard + # silently un-ships W25/R10 the next time the spawn learns to read a fourth key. When none of + # them moved the call is a no-op by construction (it re-derives from those same keys), and the + # run-time `ut_ensure` R10 deliberately kept is still the backstop if the TABLE drifted + # underneath — which is not a thing a rename should be repairing. Shaped after the trigger + # comparison directly below, which already compares `prev` against the cleaned definition. + # ⭐⭐ WAVE 31 · T30 — AND THE PREDICATE IS NOW THE SPAWN'S OWN ARGUMENT, which is the fix. + # The four-whole-objects test above was right in principle and inert in practice: `persist` + # sends a REBUILT `config`, so `prev["config"] != defn["config"]` on essentially every save and + # the guard fired every time — sparing only *rename*, the one edit the owner stopped + # complaining about after wave 30. `preset_inputs` is the canonical projection of exactly what + # `_spawn_presets` reads, and `_spawn_presets` is handed nothing else, so this comparison + # cannot go stale behind a future input the way a hand-listed key set would. The full argument, + # including why this is STRONGER than what it replaces rather than a relaxation, is in + # `preset_inputs`' own docstring — read that before widening or narrowing this line. + if preset_inputs(prev) != preset_inputs(defn): + _presets_after_write(rt, defn, username) # R10: also on the save that RE-TARGETS + # A2(1): a NEW or CHANGED condition trigger starts with everything currently matching + # DISARMED — enabling never fires for records already in the state. + if (defn.get("trigger") or {}) != ((prev.get("trigger")) or {}): + _seed_event_state(rt, defn) + return defn, None + + +def _presets_after_write(rt, defn, username): + """⭐ WAVE 25 · R10 — THE PRESET COLUMNS APPEAR ON SAVE, not on the first run. + + Owner ruling R10: choosing the trigger / naming the Create record target spawns the preset + columns immediately, "so R2b's list is real and the database is visible before anything is + spent". That last clause is the point — a person can look at the database, see the sixteen + columns, and decide whether to arm the schedule, instead of paying a vendor to find out what + they are agreeing to. + + ⛔ THE RUN-TIME `ut_ensure` STAYS AS THE BACKSTOP (R10 says so explicitly), and it costs + nothing to keep: `ut_ensure` skips the write entirely when no field would change, so the first + run of a saved automation finds the table already correct and spends no commit. + + ⚠ IT SPAWNS `CANDIDATE_FIELDS`, NOT `PRESET_PROFILE_FIELDS`. The discovery target needs the + search bookkeeping (`found_count`, `first_found`, `created_by`) as well as the + preset set, and spawning a subset here would mean the columns changed between Save and the + first run — two different answers to "what does this database look like", which is the exact + disagreement R10 exists to remove. + + ⚠ A WORKSPACE AT ITS TABLE CEILING SPAWNS NOTHING AND THE SAVE STILL SUCCEEDS. `ut_ensure` + returns the key without creating when `MAX_UT_TABLES` is reached; refusing to save an + automation because the workspace is full would be a worse answer than saving one whose table + arrives later. The run-time path reports that condition LOUDLY (`ut_missing`, D-11), so the + honest failure still has exactly one home. + """ + _spawn_presets(rt, preset_inputs(defn), username) + + +#: The enrich actions whose OWN config decides which preset columns and child databases a save +#: spawns, and the exact keys read off each. DECLARED here rather than spelled inside +#: `preset_inputs`, so `_spawn_presets` and the save-path guard cannot drift apart by one key. +PRESET_ACTION_INPUTS = { + "enrich_instagram": ("profileField",), + "enrich_tiktok": ("profileField", "postMetrics", "commentMetrics"), +} + + +def preset_inputs(defn): + """⭐⭐ WAVE 31 · T30 — EXACTLY the values `_spawn_presets` consumes off a definition, canonical. + + ⛔ THIS IS THE WHOLE POINT OF THE TICKET, so it is worth being precise about what changed and + why it is SAFER than what it replaces, not weaker. Owner item 7, verbatim: *"It still takes + forever to change the Config for automation as well. It just say Saving... and takes a long time + for me to change options and configs etc."* + + Wave 30 guarded the spawn on `any(prev[k] != defn[k] for k in ("config","flow","trigger", + "kind"))` and reasoned — correctly — that *"the predicate is the INPUT SET of the function being + skipped, not a hand-listed set of cheap edits"*. The flaw was not the reasoning; it was that the + input set was stated as four WHOLE OBJECTS while the spawn reads a handful of leaves out of + them. `AutomationDetail.tsx::persist` sends `config: buildConfig()` — a value RECONSTRUCTED from + React state, not the stored object — so the comparison is against something rebuilt on every + save and the guard never fires on the one edit the owner is complaining about. W30's fix spared + *rename* (the client omits `flow`/`trigger` entirely, and `kind`/`name` are unchanged), which is + exactly the edit the owner stopped complaining about after wave 30. + + ⭐ WHY NARROWING IS SAFE HERE AND WAS NOT SAFE THEN: this projection is not a hand-listed + allow-list that a future wave can silently outgrow. `_presets_after_write` hands this dict to + `_spawn_presets` and `_spawn_presets` **never receives `defn` at all** — so it is structurally + incapable of reading a fifth key that this function does not carry. Teaching the spawn to read + something new REQUIRES adding it here, and the guard then follows for free. That is a stronger + guarantee than wave 30's, which held only as long as somebody remembered the comment. + + ⚠ CANONICAL, because the input is a rebuild: each leaf is normalised exactly the way the spawn + itself normalises it (`str(...).strip()` for a field name, `bool(...)` for a switch). Without + that, `postMetrics: false` and a missing `postMetrics` would compare unequal while the spawn + treats them identically — a guard that fires on a difference its consumer cannot see is the + same defect in a smaller font. + + ⚠ IT DELIBERATELY COVERS THE DISCOVERY BRANCH **AND** THE ENRICH BRANCH IN ONE PROJECTION, even + though today's control flow returns before the second can run for a discovery automation. That + early return is D-178 (W31-T34) and it is going away; a projection built around today's branch + would silently narrow the guard the moment it does. + """ + defn = defn or {} + cfg = defn.get("config") or {} + actions = ((defn.get("flow") or {}).get("actions") or []) + return { + "target": str(_flow_table(defn) or cfg.get("targetTable") or ""), + "label": str(cfg.get("targetLabel") or ""), + "flowId": str(defn.get("id") or ""), + "discoveryKind": DISCOVERY_TRIGGER_KIND.get( + (defn.get("trigger") or {}).get("key") or ""), + "actions": { + kind: [{k: (_norm_switch(a, k) if k != "profileField" + else str((a.get("config") or {}).get(k) or "").strip()) + for k in keys} + for a in _actions_of_kind(actions, kind)] + for kind, keys in PRESET_ACTION_INPUTS.items() + }, + } + + +def _norm_switch(action, key): + """A capture switch as the spawn reads it — `bool`, so absent and False are ONE value.""" + return bool((action.get("config") or {}).get(key)) + + +def _spawn_presets(rt, inputs, username): + """The spawn itself. Takes `preset_inputs(defn)` — never the definition — see that docstring. + + ⭐⭐ WAVE 31 · T30, THE SECOND HALF: **ONE `user_tables` read for the whole pass, not five.** + Every `ut_ensure` below used to take its own copy of the tenant document through + `ut_get` → `ut_all` → `rt.get`, which deep-copies unconditionally (ceiling 35.8 MB / ~1.4 s), and + the TikTok arm reaches FOUR of them plus the `ut_get` above — measured **7,912 ms** on + `4258a93`. Lending one snapshot is the same fix W30-T13 made on the automation READ path + (`routes_automation.py::_LentTables`); this is deliberately NOT a third copy of that class, but + the `tables=` parameter `retire_automation_stage_fields` in this very module already + established, because here every consumer is a function we own and pass to directly. + """ + target = inputs["target"] + if not target: + return + tag = inputs["flowId"] + ig_actions = inputs["actions"]["enrich_instagram"] + tt_actions = inputs["actions"]["enrich_tiktok"] + try: + # ⭐ THE ONE READ. Everything below answers out of this snapshot. + tables = ut_all(rt) + # ⭐⭐ WAVE 30 · T06 — BOTH DISCOVERY KINDS SPAWN ON SAVE, and TikTok's absence here was + # R10 simply not holding for the second platform: `TT_PROFILE_FIELDS` reached `ut_ensure` + # at exactly ONE site — inside `run_discover_tiktok` — so the database did not exist until + # money had already been spent finding out what was in it. That is the precise pre-R10 + # behaviour `verify_automation`'s NC39 forbids on the Instagram side. + # ⛔⛔ THE DISCRIMINATOR IS THE **TRIGGER**, NOT THE KIND, AND THE GATE PAID FOR THAT + # SENTENCE. Widening this to `kind in DISCOVERY_KINDS` looks equivalent — law 1 makes a + # discovery trigger choose its kind — but the implication only runs ONE WAY. A definition + # may carry `kind: "discover_instagram"` with **no trigger at all**: law 1 sets the kind + # from the trigger and never the reverse, so any direct API create can do it, and one of + # this repo's own fixtures does. MEASURED: the kind test spawned a preset database under a + # dry-run fixture whose check asserts that no table exists — a red on correct-looking code, + # caught only because that check happened to exist. + # ⇒ R10 is a rule about what somebody PICKED IN THE PICKER, so it reads the picker's own + # answer. `DISCOVERY_TRIGGER_KIND` is that map, and a gate asserts it agrees with law 1. + # ⚠ TRACKED because it is the ONE key several `ut_ensure` calls in this pass can share: the + # discovery arm and both enrich arms all ensure `target`. A later one must not answer out of + # a snapshot an earlier one invalidated — see the `tables=` arguments below. + wrote_target = False + _dkind = inputs["discoveryKind"] + if _dkind: + _, _, _spawn_label, _spawn_fields = discovery_facts(_dkind) + ut_ensure(rt, inputs["label"] or _spawn_label, _spawn_fields, username, + key=target, flow_tag=tag, lock_fields=True, tables=tables) + # ⭐⭐ WAVE 31 · T34 (D-178) — AND THE `return` THAT USED TO BE HERE IS GONE. + # + # ⛔ WHAT IT COST: a DISCOVERY automation could never spawn its child databases. The + # capture switches below are the ones that promise `ut_tt_posts` / `ut_tt_comments` / + # `ut_tt_post_snapshots` at SAVE, before any money is spent (W30-T10) — and only a + # `plain` flow ever reached them, because a discovery flow left this function three + # lines earlier. So the exact automation the owner would build for TikTok discovery — + # find profiles, then capture their posts — silently got the profile table and nothing + # else, and the toggles it showed were promises the save path never kept. + # ⚠ IT IS A TRAP, NOT TODAY'S ONLY SYMPTOM: nurilab's child tables are absent for a + # CONFIGURATION reason as well, so fixing this alone will not make them appear there. + # + # ⚠ THE SNAPSHOT IS NOW STALE FOR `target`, and the arms below resolve their profile + # binding out of that very table (`bound` reads its `fields`). One re-read, paid only on + # a save that actually reached the discovery spawn — correctness before the read count, + # and it is one read against the 41 this function used to take. `wrote_target` stays + # False precisely BECAUSE we refreshed: it tracks staleness, not "did somebody write". + tables = ut_all(rt) + enrich_actions = ig_actions + table = tables.get(target) or {} + named = next((a["profileField"] for a in enrich_actions if a["profileField"]), "") + bound = next((str(f.get("key") or "") for f in table.get("fields") or [] + if isinstance(f.get("profile"), dict)), "") or named + if enrich_actions and bound: + ut_ensure(rt, table.get("label") or target, _profile_schema_for(bound), username, + key=target, flow_tag=tag, lock_fields=True, tables=tables) + wrote_target = True + # ⭐ WAVE 30 · T08 — the TikTok half of W25/R10: the columns an `enrich_tiktok` step will + # fill appear when the automation is SAVED, not when money is first spent. Resolved + # through `profile_field_key(..., source=PROFILE_SOURCE_TT)` so it cannot adopt an + # Instagram binding, and through the SAME `_tt_profile_schema_for` the runner's own top-up + # uses — R10's rule is that the column set does not change between Save and the first run, + # and one shared resolver is what makes that structural instead of a convention. + if tt_actions: + tt_named = next((a["profileField"] for a in tt_actions if a["profileField"]), "") + tt_bound = profile_field_key(table, tt_named, source=PROFILE_SOURCE_TT) + if tt_bound: + ut_ensure(rt, table.get("label") or target, _tt_profile_schema_for(tt_bound), + username, key=target, flow_tag=tag, lock_fields=True, + tables=None if wrote_target else tables) + # ⭐ WAVE 30 · T10 — and the CHILD databases the step's own switches promise, at + # SAVE, before any money. R10's rule is "the columns a step will fill appear when + # you press Save"; a `postMetrics` toggle whose database only exists after a paid + # run is the same complaint one level up. + # ⛔ GATED ON THE SWITCHES, never spawned unconditionally: a database that exists + # because somebody looked at a toggle, and then never fills, is the "SECOND, empty + # database" the discovery default was rewritten to stop producing. + for _flag, _child in (("postMetrics", TT_POSTS_TABLE), + ("postMetrics", TT_POST_SNAPSHOTS_TABLE), + ("commentMetrics", TT_COMMENTS_TABLE)): + if any(a[_flag] for a in tt_actions): + # R9 (W31-T32): the child arrives LOCKED, from its own declaration. + ut_ensure(rt, TT_TABLE_LABELS[_child], TT_TABLE_FIELDS[_child], username, + key=_child, flow_tag=tag, lock_fields=True, tables=tables, + record_mode=tt_record_mode(_child)) + except Exception as e: # noqa: BLE001 + # ⛔ NEVER FAILS THE SAVE. The definition is already committed by the time this runs, so + # raising here would answer 500 for an automation that IS stored — the caller would retry + # and create a second one. The columns then arrive on the first run, which is precisely + # the behaviour this function is an improvement on rather than a replacement for. + print(f"[aios-auto] preset spawn deferred to the first run: {type(e).__name__}: {e}") + + +def remove(rt, auto_id): + aid = str(auto_id) + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + cur.pop(aid, None) + return cur + # ⛔ WHY THIS IS `async` AND NOT `sync` (owner report 2026-08-12, wave 30, MEASURED). + # Owner, verbatim: *"it takes forever to delete an automation"*. A delete was paying up to + # TWO BLOCKING UPLOADS inside the request — this one, plus the whole-document rewrite below, + # which fires for every DISCOVERY automation because those always tag the columns they + # spawned (deleting one marked all 30 columns of `ut_tt_profile` disabled). Measured 977 ms + # for the CHEAPEST case (a `plain` flow, no tagged columns) against a 757 ms `GET /automations` + # baseline on the same connection; the discovery case is strictly worse by a full-document + # read plus a full-document commit. + # + # `flush="async"` is not a weakening: `Store.update`'s async branch applies the mutation to + # the in-process cache, marks it owned+dirty and schedules a COALESCING worker, and the + # class's read-your-writes contract means the very next `GET /automations` already sees the + # deletion. This is the same trade `user_tables.add_row` makes for ROW creation — strictly + # more valuable data than an automation definition — so a delete is not the place to be + # stricter than a row insert. [[sync-write-eats-pending-async-write]] is the hazard in the + # OTHER direction (a sync writer discarding a pending async write) and `Store.update`'s dirty + # branch already handles it. + _store_update(rt, _up, flush="async") + # C8 — the fields this flow tagged are DISABLED with the reason on them, never orphaned + # silently: the column keeps its values and its config, and says why it stopped. Scanned + # first so a delete with no tagged fields costs no store commit. + # ⚠ HONEST RESIDUAL, stated rather than smoothed: this scan still costs ONE full read of the + # tenant's `user_tables` document on EVERY delete (`rt.get` deep-copies by design), and the + # update below reads it a second time. Removing that needs a tagged-field index, which is a + # feature and not a wave-tail edit — booked rather than improvised. What it no longer costs + # is the part the owner could feel: the blocking network commits. + # ⭐ WAVE 31 · T39(a) — D-177(a): ONE WALK, AND THE PLAN CARRIES WHAT IT FOUND. + # This used to scan the whole document to answer a boolean (`hit`), then walk the WHOLE + # document again inside `_disable` to re-derive the same matches. The scan now records the + # `(table, field)` pairs it found and the mutation applies exactly those — the shape + # `bind_unbound_fields` in this module already uses, so a delete costs one walk instead of two + # and the write touches only the fields it named. + # ⭐⭐ WAVE 33 (D-179) — THE SCAN NO LONGER READS THE ROWS, and that is where the cost was. + # + # This walk asks ONE question — "which fields did this flow tag?" — and it has never looked at + # a row to answer it. It was nevertheless taking `ut_all`, a whole deep copy of the tenant's + # `user_tables` document, of which **99.9% of the bytes are `rows` nothing here reads** + # (measured on tenant #0: 28.6 MB, 81,330 rows, 703 ms warm). A's C5 projection makes that + # ~0.1% of the bytes. + # ⛔ HANDED TO A READ AND NOTHING ELSE. `lend_defs` serves `_Projected` values, so reaching + # `defn['rows']` raises a `KeyError` NAMING the projection — by design — and `plan` carries + # only `(table_key, field_key)` STRINGS out of this scope. The mutation below takes its own + # strict document from `rt.update` and never sees a projected value, which is C5's second + # clause: a projected snapshot is never handed to a post-write read-back. + # ⚠ HONEST RESIDUAL, STATED RATHER THAN SMOOTHED (R6's second sentence): D-179's exit condition + # asks for ZERO full-document reads and this is ONE — `rt.update` below takes a strict read + # that belongs to the WRITE and cannot be removed from here. What is gone is the expensive one. + # ⚠ AND THE FALLBACK IS DELIBERATE: `all_defs` degrades to the whole read on any failure, so a + # store that cannot project still deletes correctly, only slower. + # + # ⭐⭐ WAVE 35 · T34 — THE RESIDUAL IS NOW MEASURED RATHER THAN ASSERTED, and the count is + # better than the paragraph above claims. Whole `user_tables` reads per delete, against a + # runtime that has production's `get_projection`: + # TAGGED (a discovery flow that spawned columns) → 1 whole + 1 projected + # UNTAGGED (a `plain` flow) → 0 whole + 1 projected + # The untagged case pays NONE because `if not plan: return` fires before the write. The + # tagged case's ONE is `rt.update`'s own strict read, and it is a FLOOR, not a choice: + # `core/store.py::get_projection` states that a write must always read whole, because a + # read-modify-write handed a rows-less document would upload the tenant with its rows deleted. + # ⛔ Reaching ZERO needs D-179's `{flowId: [(table_key, field_key)]}` index, written where the + # presets are spawned — a feature, not an edit to this function. Nothing here can do better. + # ⚠ THOSE NUMBERS WERE UNOBSERVABLE UNTIL W35-T34: `verify_automation`'s delete-speed double + # had no `get_projection`, so `all_defs` fell back and the gate measured 2 and 1 — the + # PRE-WAVE-33 path — for two waves. The double is producer-faithful now and asserts the counts. + try: + import core.user_tables as _ut_defs # noqa: PLC0415 + _scan = dict(_ut_defs.all_defs(rt) or {}) + except Exception: # noqa: BLE001 + _scan = ut_all(rt) + plan = [(tk, str(f.get("key") or "")) + for tk, t in _scan.items() + for f in ((t or {}).get("fields") or []) + if str((f.get("automation") or {}).get("flowId") or "") == aid] + if not plan: + return + + def _disable(cur): + cur = cur if isinstance(cur, dict) else {} + note = f"its automation was deleted {_stamp()}" + for tk, fk in plan: + for f in ((cur.get(tk) or {}).get("fields") or []): + a = f.get("automation") + if f.get("key") == fk and isinstance(a, dict): + a["disabled"] = True + a["statusNote"] = note + return cur + # The expensive half of the owner's complaint: a whole-document rewrite, committed while the + # person waits. Async for the reason given above — the disable is visible to the next read + # immediately; only the upload is deferred, and it coalesces with any other pending write to + # the same key instead of racing it. + rt.update(UT_STORE_KEY, _disable, flush="async") + + +# --------------------------------------------------------------------------------------------- +# RUN STATE — process memory only (see the module header for why this may never be persisted) +# --------------------------------------------------------------------------------------------- + +_RUN_LOCK = threading.RLock() +_RUNNING = {} # (tenant, id) -> {'startedAt', 'step'} + + +def running(tenant, auto_id): + with _RUN_LOCK: + return dict(_RUNNING.get((tenant, str(auto_id))) or {}) or None + + +def _claim(tenant, auto_id): + with _RUN_LOCK: + if (tenant, str(auto_id)) in _RUNNING: + return False + _RUNNING[(tenant, str(auto_id))] = {"startedAt": _iso(), "step": "starting"} + return True + + +def _step(tenant, auto_id, text): + with _RUN_LOCK: + cur = _RUNNING.get((tenant, str(auto_id))) + if cur is not None: + cur["step"] = text + + +def _no_step(_text): + """The default `step` sink for a runner nobody gave a live slot to (a gate driving a runner + directly). A runner must never REQUIRE the slot — the slot is process state, and the runner + is the part a test is allowed to call on its own.""" + + +def _release(tenant, auto_id): + with _RUN_LOCK: + _RUNNING.pop((tenant, str(auto_id)), None) + # A completed/manual/event run may have changed rows that feed a metric, link or source + # rollup. Mark the derived lane dirty; the next durable tick will refresh it once. Merely + # waking the scheduler does not mark anything dirty and therefore does not poll Neon. + _mark_derived_dirty(tenant) + + +def _commit_run(rt, auto_id, state, summary, counts, ok, affected=None, steps=None, notes=None): + """THE ONE store write a run performs on the `automations` bucket. + + `steps` is the per-NODE outcome map the canvas paints its dots from. It is recorded here + because it is a MEASUREMENT the runner took as it walked — see `graph`, which refuses to + invent a node status when this map is absent (every run stored before W19-C has no `steps`, + and painting those nodes green from the run's overall state would be a fabrication). + + ⭐⭐ `notes` CLOSES DEBT D-103 (2026-08-09). A run's `counts` are integers and the + comprehension below drops everything else — so the per-record sentence a vendor gave us had + literally nowhere to live, and *"1 profile read(s) were blocked"* was the whole of what the + product could say. MEASURED on nurilab: the same record blocked on three separate runs and + the reason was unrecoverable from the store afterwards, so the diagnosis had to be rebuilt by + calling the vendors by hand. The note is the most valuable thing a run produces — a dead + handle, a vendor hiccup and an exhausted key are three different actions — and it was the one + thing thrown away. + ⚠ ON THE RUN, NOT ON THE RECORD. D-103's own prescription: no tenant table gains a column it + did not ask for, and W25/R4 already retired writing status STRINGS into people's grids. + """ + entry = {"ts": _iso(), "ok": bool(ok), "summary": _s(summary, 400), + "counts": {k: int(v) for k, v in (counts or {}).items() + if isinstance(v, (int, float))}, + "steps": {str(k): str(v) for k, v in (steps or {}).items()}, + # Bounded on both axes: a 100-profile run must not put 100 sentences into a store + # entry that is kept 20 deep per automation. + "notes": [_s(n, 300) for n in (notes or []) if str(n or "").strip()][:25], + "affected": list(affected or [])[:200]} + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + d = cur.get(str(auto_id)) + if d is None: + return cur + d["status"] = {"state": state, "lastRunAt": entry["ts"], + "lastSummary": entry["summary"]} + d["runs"] = ([entry] + list(d.get("runs") or []))[:MAX_RUNS] + # Wave 22 (airtable-brief rec 6): K consecutive FAILURES auto-pause the automation with + # the reason on it — a dead credential must not burn quota (or a paid vendor's records) + # 96 times a day while a green toggle sits over a buried error log. Errors only: + # `partial` is the honest-progress state and pausing on it would punish honesty. + runs = d.get("runs") or [] + if state == "error" and len(runs) >= CONSECUTIVE_FAILURE_PAUSE and all( + not r.get("ok") for r in runs[:CONSECUTIVE_FAILURE_PAUSE]): + sch = d.get("schedule") + if isinstance(sch, dict) and sch.get("enabled"): + sch["enabled"] = False + trg = d.get("trigger") + if isinstance(trg, dict): + trg["paused"] = True + d["statusNote"] = (f"auto-paused after {CONSECUTIVE_FAILURE_PAUSE} consecutive " + f"failures. Fix the cause, then re-enable: " + f"{entry['summary'][:120]}") + return cur + + _store_update(rt, _up, flush="sync") + _notify_outcome(rt, auto_id, state, entry) + return entry + + +def _notify_outcome(rt, auto_id, state, entry): + """⭐⭐ WAVE 27 ITEM 31 / CONTRACT C5 — the bell rings on EVERY outcome, good and bad (I7). + + Owner: notify on both success and error. `_commit_run` is the ONE place a run's outcome is + written, so it is the only place this can go without a second definition of "what happened". + + ⛔ AMENDMENT A1 MOVED TWO REQUIREMENTS HERE AND THE FIRST IS A CROSS-TENANT DEFECT IF MISSED. + 1. **`st=` IS PASSED EXPLICITLY.** `core.alerts.notify`'s `st=None` default resolves to the + MODULE-level store, which is tenant #0's (Royal Imports). The automation tick runs on a + background thread with no session, so an omitted `st` files nurilab's automation failures + in Royal Imports' inbox — D-16's class, silently, forever. + 2. **THE OWNER IS THE AUTOMATION'S `createdBy`, NEVER THE RUNNER'S IDENTITY.** The scheduler + has no session at all, so the alternative is not "the wrong person" but "nobody", and + `notify` returns None on a blank owner — the notification would simply never exist, on + exactly the runs nobody was watching. + ⚠ AND NO SERVER ADMISSION WAS NEEDED. A1 measured that `routes_alerts._TOPICS` gates only + view-alert CREATION; `notify()` writes straight into the notifications bucket and + `GET /notifications` filters by nothing, so an `automation` topic already reaches the bell. + ⚠ FAILURE HERE IS SWALLOWED. A notification that cannot be filed must not turn a run that + SUCCEEDED into one that raised — the run entry is already committed one line above, and the + bell is a courtesy on top of it. + """ + try: + defn = (rt.get(STORE_KEY) or {}).get(str(auto_id)) or {} + owner = str(defn.get("createdBy") or "").strip() + if not owner: + return + from core import alerts as _alerts + _alerts.notify( + owner=owner, + # ⭐ WAVE 34 · R12 — the fallback name a notification wears when the automation has + # none. "Agent", singular: this names ONE of them, not the module. + label=_s(defn.get("name") or "Agent", 80), + topic="automation", + key=str(auto_id), + detail=f"{state}: {entry.get('summary') or ''}"[:300], + st=rt) + except Exception: # noqa: BLE001 + pass + + +# --------------------------------------------------------------------------------------------- +# INSTAGRAM (R7, extended by R1) — PUBLIC DATA ONLY, NEVER AN INSTAGRAM LOGIN +# --------------------------------------------------------------------------------------------- +# ⚠ AMENDED W19-C, vendor swapped W20. This section read "ANONYMOUS PUBLIC ENDPOINTS ONLY" and +# that is no longer the whole truth: R1 added a paid VENDOR rung (Bright Data, further down) +# which does authenticate — to the vendor. The rail that matters is unchanged and is sharper: +# +# **NOTHING HERE EVER AUTHENTICATES TO INSTAGRAM.** No login, no password, no session cookie, +# no account to get banned, public data only. The vendor key is a key to a SUPPLIER, and the +# supplier takes the scraping risk; it is never an Instagram credential and this module must +# never be given one. +# +# The rungs below this line are the ANONYMOUS ladder ($0, no key of any kind). +# +# ⚠ HONEST STATUSES ARE THE FEATURE. Instagram rate-limits and outright blocks datacenter egress +# (the HF Space and any AWS Lambda are both in that class), and its anonymous surface has been +# progressively closed for years. So this returns a STATE — ok / partial / blocked / error — and +# the cell says which. A pull that quietly wrote zero posts and reported success would be worse +# than no automation at all: the table would look maintained and be empty. +# +# THE LADDER, tried in order with pacing between rungs. Each rung records how it did in `via`, +# so a run history answers "what still works anonymously?" from data rather than from memory. + +PACE_SECONDS = float(os.environ.get("AIOS_IG_PACE_SECONDS") or 2.5) + +#: ⭐ MAP THE SCHEMA, NOT JUST WHAT WAS POPULATED ON THE PROBE — and the reason is this page's +#: own headline. **HISTORY IS UNBUYABLE.** No vendor sells "followers on 1 January"; every number +#: is a NOW value, so the series starts the day capture starts and a field we do not capture +#: TODAY is permanently lost for today. Dropping a column because it was null on two probe rows +#: has exactly the same cost as delaying capture — for that column — and is justified by exactly +#: the evidence (n=2) that was judged too thin to close D-25. Adding a column that turns out to +#: be usually-empty costs a blank cell; omitting one costs the months before somebody notices. +#: Every returned provider field is retained in Source data. Promoted columns make commonly used +#: values filterable; the raw source document prevents a new or uncommon field from being lost. +#: ⚠ A blank in any of these means NOT READ — never "they have none". The anonymous rungs expose +#: almost none of them, which is what the `source` column is for. +SNAPSHOT_FIELDS = [ + field_def("snapshot_key", "Snapshot"), field_def("influencer_key", "Influencer"), + field_def("pulled_at", "Pulled at", "date"), field_def("followers", "Followers", "int"), + field_def("following", "Following", "int"), + field_def("posts_count", "Post count", "int"), + field_def("full_name", "Name"), field_def("bio", "Bio"), + field_def("verified", "Verified", "checkbox"), field_def("source", "Read via"), + field_def("approx", "Counts are approximate", "checkbox"), + # The link in bio — the paid rung's field, and commercially the most useful single string an + # influencer row can carry (it is the shop/affiliate destination). + field_def("external_url", "Link in bio", "url"), + # --- the rest of the vendor's Profiles schema. + field_def("ig_id", "Instagram id"), + field_def("profile_url", "Profile", "url"), + field_def("avg_engagement", "Avg engagement", "pct"), + field_def("category", "Category"), + field_def("business_category", "Business category"), + field_def("is_business", "Business account", "checkbox"), + field_def("is_professional", "Professional account", "checkbox"), + field_def("is_private", "Private", "checkbox"), + field_def("highlights_count", "Story highlight count", "int"), + field_def("bio_hashtags", "Bio hashtags"), + field_def("pronouns", "Pronouns"), + # ⭐ 2026-08-07 — the rest of the vendor's Profiles schema (see `_bd_profile`). The snapshot + # row maps the WHOLE schema by design, so these belong here the moment the map reads them; + # leaving them out would be the "captured but unrecorded" half of the same loss the note at + # the top of this list is about. + field_def("profile_name", "Profile name"), + field_def("is_joined_recently", "Joined recently", "checkbox"), + field_def("has_channel", "Has channel", "checkbox"), + field_def("partner_id", "Partner id"), + field_def("external_url_title", "Link title"), + field_def("fbid", "Facebook id"), + field_def("related_accounts", "Related accounts"), + field_def("country_code", "Country"), + field_def("source_payload", "Source data", "json"), +] +POST_FIELDS = [ + field_def("shortcode", "Shortcode"), field_def("influencer_key", "Influencer"), + field_def("posted_at", "Posted at", "date"), + field_def("type", "Type", "select", options=["image", "video", "carousel"]), + field_def("caption", "Caption"), field_def("url", "URL", "url"), + # A tagged place is useful content context and can be filtered as ordinary text. It is never + # presented as the creator's location: a creator can tag a holiday, venue or brand location. + field_def("tagged_location", "Tagged location"), + # The per-field map below gives operational fields first; this locked JSON document retains + # every other value Bright Data supplied, so a vendor schema addition is preserved + # instead of being silently discarded while the canonical field model catches up. + field_def("source_payload", "Source data", "json"), + # Sponsored-post detection — §2c called it one of the fields worth having that the anonymous + # ladder cannot reach, and it came back MEASURED-populated (`True`, with the brand attached). + field_def("paid_partnership", "Paid partnership", "checkbox"), + field_def("partner", "Partner brand"), + field_def("hashtags", "Hashtags"), + field_def("alt_text", "Alt text"), + # ⭐⭐ 2026-08-07 — THE LATEST-VALUE ENGAGEMENT COLUMNS, AND THEY EXIST TO KEEP A ROLLUP AT + # ONE HOP. + # + # "Average views over the last N posts" is naturally TWO hops: profile → posts → each post's + # most recent snapshot → `views`. Airtable's rollup is one hop, and growing a second one is a + # much bigger build with a much worse failure mode (a rollup over a rollup, invalidated + # transitively). So the post row carries its own LATEST value and the rollup reads it + # directly. + # + # ⛔ R3's "ONE STORE FOR ONE SERIES" IS UNTOUCHED, and this is exactly the pattern the profile + # row already uses one level up: LATEST on the row (+ `enriched_at` to date it), the SERIES in + # `ut_ig_post_snapshots`. These three cells are rewritten on every pull from the snapshot that + # was just appended — they are a projection of that store, never a second copy of it, and + # deleting them would cost a convenience rather than a measurement. + # ⚠ BLANK, NEVER ZERO — and `_ig_zero_is_blank` is what finally ENFORCES it. `postMetrics` is + # off by default (it buys one vendor record per post), so on most pulls this stays empty, and + # empty means "not read", which is what makes an honest average possible at all. A 0 here + # would claim a post nobody watched. ⚠ The rule is scoped to this paid rung: a zero LIKE or + # COMMENT count is a real measurement and must survive. + # + # ⛔⛔ THERE IS NO `views` COLUMN HERE, AND THAT IS A RULING, NOT AN OVERSIGHT (2026-08-08, + # instagram-capture.md §4e/§4f — owner-approved after the evidence was bought). + # Bright Data's `views` is delivered at ACCOUNT grain: one value across up to 12 distinct + # reels, on 18 of 19 creators, through BOTH collection modes, while `likes` varies richly in + # the very same rows. Mapping it here is what put one number on many unrelated posts. We + # cannot say what it measures at ANY grain (`sriyynntt` returns 0 at 11,102 followers; + # `reviewby_ayyaa` returns null at 20k–328k likes), so it is not promoted anywhere — it stays + # in Source data, unlabelled, making no claim. Nothing is lost: the payloads are retained + # whole, so if the vendor ever populates it per-reel the history is re-derivable. + # ⚠ ENUMERATED, so nobody re-opens this hoping for a differently-named field: across all 30 + # fields of a Reels row, `views` and `video_play_count` are the ONLY view-shaped keys, and the + # Posts dataset carries none at all. `plays` below IS the per-reel equivalent, correctly named + # and correctly mapped. It is blank because Meta retired the Plays metric on 2025-04-10 + # (folded into a single "Views"), not because we are reading the wrong key. + # ⭐⭐ VIEWS IS BACK (2026-08-08), AND FROM A DIFFERENT SOURCE THAN THE ONE THAT WAS RETIRED. + # The retired column was fed by Bright Data's account-grain `views`. This one is fed by the + # `ig_post_views` CAPABILITY (providers.py), which resolves to Apify's `videoPlayCount` — + # measured against a browser-read ground truth to the digit. Same column name, same type, same + # rollup; the engine underneath is swappable and the preset database never moved. That is the + # owner's schema ruling working exactly as intended. + # ⚠ STILL BLANK, NEVER ZERO, and still only on the paid rung. + field_def("views", "Views", "int"), + field_def("plays", "Plays", "int"), + field_def("likes", "Likes", "int"), + field_def("comments", "Comments", "int"), + # ⭐⭐ 2026-08-09 (owner: *"whatever APIfy has more than BD pls use it and fix the post data + # further"*). Two fields the primary source does not return AT ALL, already paid for inside + # the same engagement response — so capturing them costs nothing extra. + # ⛔ A COLUMN MUST EXIST BEFORE A CELL CAN BE WRITTEN. `user_tables` filters an unknown key on + # every write door, so a normaliser that emits `video_duration` without this line writes + # nothing and reports success — the wave-28 defect that swallowed nine preset cells. + field_def("video_duration", "Video length (s)", "int"), + field_def("comments_disabled", "Comments off", "checkbox"), + field_def("measured_at", "Engagement read at", "date"), + # ⭐ 2026-08-07 — the second half of "spawn relevant Post/Comment database that is LINKED": + # a post reaches its own comment rows the same way a profile reaches its posts. Derived from + # `shortcode`, so it is correct the moment a comment row exists and needs no maintenance. + # ⚠ It resolves to nothing until comment capture is switched on, which is the honest state + # for a relation whose far side is empty — the same standing `ut_ig_post_snapshots` has when + # `postMetrics` is off. + field_def("comments_link", "Comment rows", "link", + description="Comment records linked to this post.", + link={"table": IG_COMMENTS_TABLE, "on": "shortcode", "from": "shortcode"}), + field_def("post_snapshots_link", "Measurement rows", "link", + description="Engagement measurements linked to this post.", + link={"table": IG_POST_SNAPSHOTS_TABLE, "on": "shortcode", "from": "shortcode"}), + field_def("measurements_captured", "Measurements captured", "rollup", + rollup={"link": "post_snapshots_link", "fn": "countall"}), +] + +#: ⭐⭐ 2026-08-07 (owner instruction) — THE COMMENT DATABASE. +#: +#: Owner: *"an enrichment automation should spawn relevant Post/Comment database that is linked to +#: the profile automatically."* So the schema and the LINK ship; what does NOT ship on by default +#: is the capture. +#: +#: ⛔ THIS REVERSES A STANDING RULING (D-22 / R11) AND THE REVERSAL IS DELIBERATE AND BOUNDED. +#: Comment capture was refused on two grounds, and BOTH ARE STILL TRUE: +#: 1. COST — comments are ~98% of a full-history bill (~$56,000 at 37.5M comments, §2a of +#: `instagram-capture.md`). They are the single most expensive thing this product can buy. +#: 2. PRIVACY — Bright Data flags `comment_user` and `post_user` as PII, and the append law has +#: no erasure path for third parties who never appeared in the tracked set (D-24). A comment +#: thread ingests identifiable people who never entered anybody's influencer list. +#: ⛔ SO WHAT SHIPS HERE IS THE SCHEMA AND THE LINK. **NOTHING CAPTURES COMMENTS TODAY** — no code +#: calls Bright Data's Comments dataset (`gd_ltppn085pokosxh13`), there is no `config.comments` +#: switch, and this table stays EMPTY until somebody builds one. Stated in the present tense on +#: purpose: an earlier draft of this note described the opt-in switch as if it existed, which is +#: exactly the doc that "reads as authority and silently ages" that §0 of `instagram-capture.md` +#: is written against. +#: ⚠ WHEN IT IS BUILT it should be an OPT-IN defaulted OFF, the same posture as +#: `config.postMetrics` and the Bright Data money switch W26/R15 kept — the relation costs +#: nothing; the spend and the third-party ingest are a click somebody makes knowingly. +#: Embedded comment payloads delivered with a Profile/Post/Reel response are retained because +#: they are part of a record already paid for. The separate full Comments dataset remains OFF +#: unless `commentMetrics` is explicitly enabled on the enrichment action. +COMMENT_FIELDS = [ + field_def("comment_key", "Comment"), field_def("shortcode", "Shortcode"), + field_def("influencer_key", "Influencer"), + # ⭐⭐ OWNER RULING 2026-08-12 — the same one that put `text` on the TikTok comment schema, and + # it lands on BOTH networks in the same change on purpose: the two comment tables are read side + # by side, and one carrying the content while the other does not is the divergence this repo + # keeps paying for. ⛔ TEXT ONLY — `comment_user` / `comment_user_url` are vendor-FLAGGED PII + # and stay in `source_payload`, uncolumned. + field_def("text", "Comment"), + field_def("commented_at", "Commented at", "date"), + field_def("likes", "Likes", "int"), + field_def("replies", "Replies", "int"), + field_def("source_payload", "Source data", "json"), + field_def("post_link", "Post", "link", + description="Post record linked to this comment.", + link={"table": IG_POSTS_TABLE, "on": "shortcode", "from": "shortcode", + "single": True}), + # Author/text and every provider-specific value stay intact in Source data. The promoted + # columns intentionally keep the grid concise while Likes and Replies make comment engagement + # directly filterable and rollup-ready. +] +POST_SNAPSHOT_FIELDS = [ + field_def("post_snapshot_key", "Snapshot"), field_def("shortcode", "Shortcode"), + field_def("influencer_key", "Influencer"), + field_def("pulled_at", "Pulled at", "date"), field_def("likes", "Likes", "int"), + field_def("comments", "Comments", "int"), + # ⭐⭐ 2026-08-10 — TWO DENORMALISED POST FACTS, and they are worth having on their own merits + # before any rollup argument is made. + # + # `pulled_at` says when we LOOKED; `posted_at` says when the creator POSTED. A fact table with + # only the first can describe a measurement but not its AGE, so the single most useful thing + # this series can compute — views at N days old, i.e. velocity — is not expressible over it at + # all. `type` is the same shape one column over: "reels only" is the default lens on this data + # and without it every question has to go back through `ut_ig_posts` to ask what kind of post + # this was. + # + # ⚠ THE WRITER HAS BOTH IN HAND (`capture_rows` builds the identity row and the measurement row + # from the same vendor record), so this costs one dict key each and no extra call. + # + # ⛔ `options` IS NOT OPTIONAL ON A `select`. A select declaring none is the wave-26 item-24 + # shape: the filter panel receives an empty list as an ANSWER, `if ([])` is truthy, and the + # control renders dead with nothing to say. Same three values `POST_FIELDS` declares — copied + # from it deliberately rather than shared, because these are two different tables' columns that + # happen to agree today, and `verify_automation` asserts the agreement. + # ⛔⛔ AND THEY DO NOT MAKE "THE LAST 10 REELS" EXPRESSIBLE OVER THIS TABLE. That takes TWO + # orderings (newest measurement per post, then newest posts), the rollup bag has one `sortBy`, + # and `ut_ig_posts` already performs the first of them. See the verdict in + # `.claude/wiki/research/entity-vs-series.md` — these columns are for AGE and KIND, not for + # moving the window here. + field_def("posted_at", "Posted at", "date"), + field_def("type", "Type", "select", options=["image", "video", "carousel"]), + # Plays move independently of likes on video, so it is its own series rather than a thing to + # derive. Paid rung only; blank means not read. + # The series regains Views alongside the latest-value projection on the Post row — one store + # for one series (R3) is untouched; this IS that store. + field_def("views", "Views", "int"), + field_def("plays", "Plays", "int"), + field_def("source_payload", "Source data", "json"), + field_def("post_link", "Post", "link", + description="Post record linked to this measurement.", + link={"table": IG_POSTS_TABLE, "on": "shortcode", "from": "shortcode", + "single": True}), +] + +#: ⭐⭐ WAVE 32 · T41 — THE INSTAGRAM TWIN OF `TT_TABLE_FIELDS`, WHICH DID NOT EXIST. +#: +#: `TT_TABLE_FIELDS`' own note said so and named the consequence: *"The Instagram side has no +#: equivalent map, which is exactly why its table names are scattered across the module."* It is a +#: map now, for a concrete reason rather than symmetry: A's boot-time delivery sweep (`W32-T07`) has +#: to call `ut_ensure` for every locked child of BOTH platforms, and a sweep that can ask TikTok for +#: its table set and must hand-type Instagram's is one platform away from the drift this pair of +#: maps exists to stop. +#: +#: ⛔ THESE FOUR LISTS ARE THE *CANONICAL* DECLARATION — the per-tenant BACKLINK fields +#: `ensure_ig_graph` appends are NOT here, and must not be. A backlink is derived per profile +#: database (`_profile_backlink_field`), so folding it into a module constant would make one +#: tenant's relation a global fact. `ensure_ig_graph` reads this map and adds its own backlinks on +#: top, which is why that function still owns the graph write and this only owns the schema. +IG_TABLE_FIELDS = { + IG_SNAPSHOTS_TABLE: SNAPSHOT_FIELDS, + IG_POSTS_TABLE: POST_FIELDS, + IG_POST_SNAPSHOTS_TABLE: POST_SNAPSHOT_FIELDS, + IG_COMMENTS_TABLE: COMMENT_FIELDS, +} +#: The labels those four wear in the database list. ⚠ LIFTED VERBATIM from `ensure_ig_graph`'s own +#: `graph` literal, which is now derived from this map — a renamed label here renames the table +#: everywhere rather than leaving two spellings of one database. +IG_TABLE_LABELS = { + IG_SNAPSHOTS_TABLE: "IG snapshots", + IG_POSTS_TABLE: "IG posts", + IG_POST_SNAPSHOTS_TABLE: "IG post snapshots", + IG_COMMENTS_TABLE: "IG comments", +} + + +# --------------------------------------------------------------------------------------------- +# ⭐⭐ TIKTOK — THE FIVE `ut_tt_*` SCHEMAS (wave 29 · item 7 · D-9 · rulings R1 + R2) +# --------------------------------------------------------------------------------------------- +# ⛔ EVERY VENDOR FIELD NAME BELOW IS FROM ONE PROBED SOURCE — `waves/wave29/proto/tiktok-schema.md`, +# 40 profile / 43 post / 17 comment fields read live from `GET /datasets/{id}/metadata` for $0.00, +# each carrying the vendor's own type, description and `pii` flag. NOTHING HERE IS GUESSED, and the +# names live in `connectors_tt.py` (the map), not here (the schema). This file declares what a +# COLUMN is; the connector declares what the wire calls it. +# +# ⭐ THE SAME LAW AS THE INSTAGRAM LISTS ABOVE: map the schema, not just what was populated on the +# probe. HISTORY IS UNBUYABLE — no vendor sells "followers on 1 January" — so a field we do not +# capture today is permanently lost for today, and an occasionally-empty column costs a blank cell +# while a missing one costs the months before somebody notices. +# +# ⚠ TWO DELIBERATE DIVERGENCES FROM THE INSTAGRAM SCHEMA, both licensed by R2 ("the two schemas may +# diverge where the vendors do") and both recorded so neither reads as an omission: +# +# 1. NO `plays` COLUMN. TikTok returns ONE number, `play_count`, and it is the count TikTok +# itself displays under a video — i.e. our `views`. Instagram has two columns because Meta +# once had two metrics (`plays` retired 2025-04-10). Writing one vendor number into two of our +# columns would manufacture a second measurement that a rollup could average or double-count, +# and blank-vs-zero discipline says an unmeasured column must be blank, not a copy. +# ⇒ `play_count` -> `views`, and `plays` does not exist on this family. +# 2. `tt_id`, NOT `ig_id`. The probe doc flags this: our profile key is literally NAMED `ig_id`. +# A TikTok row carrying a column labelled "Instagram id" is a header that lies, and this is a +# NEW table family with no stored rows to migrate — so the honest name costs nothing. +# +# ⚠ THE COLUMNS THAT STAY BLANK ARE NAMED RATHER THAN DROPPED SILENTLY. TikTok has no equivalent of +# `category`, `business_category`, `is_professional`, `highlights_count`, `bio_hashtags`, +# `pronouns`, `profile_name`, `is_joined_recently`, `has_channel`, `partner_id`, +# `external_url_title`, `fbid` or `related_accounts` (profile), nor of `alt_text`, +# `comments_disabled`, `paid_partnership` or `partner` (post) — `commerce_info` is a LOCATION, not +# a paid-partnership flag, and TikTok's `comment_setting` is profile-level rather than per-post. +# Those columns are ABSENT here rather than declared and permanently blank — a column nothing can +# ever write is a promise the grid keeps making that the vendor cannot keep. + +#: ⭐ WAVE 29 (contract C2) — the value the `profile` FLAG carries on a TikTok handle column: the +#: twin of `PROFILE_SOURCE_IG`, which is declared further down beside the `PLATFORM_*` vocabulary +#: with the note on why those two families of string are NOT the same thing. It sits here rather +#: than there because the lists below are evaluated at IMPORT and would raise a NameError otherwise. +#: ⛔ THIS CONSTANT IS ONLY HALF THE CONTRACT, AND THE OTHER HALF HAS LANDED (verified 2026-08-12, +#: W30-T08): `platform/core/user_tables.py:PROFILE_SOURCES` now reads `('instagram', 'tiktok')`. +#: Before it did, `_clean_profile` REFUSED this flag and the handle column of a `ut_tt_profile` +#: spawn was dropped — fail-closed and loud by design, because the alternative is a profile +#: database whose profile column silently is not one. Left written down rather than deleted: the +#: refusal is what a TikTok binding looks like on any deployment where that line is missing. +PROFILE_SOURCE_TT = "tiktok" + +#: The profile row: LATEST values plus `enriched_at`. The SERIES lives in `ut_tt_snapshots` — +#: R3's "one store for one series" carries over unchanged, because the reason for it does (no +#: vendor sells history, so the append table is the only history there will ever be). +TT_PROFILE_FIELDS = [ + # `platform` FIRST, exactly as the Instagram preset set has it — and on all five tables here, + # not just this one. The IG family carries it on the profile table alone, which is why a TikTok + # POST could never have lived beside an Instagram post; declaring it everywhere is what makes a + # future cross-platform union view a UNION rather than a guess. + tt_field_def("platform", "Platform"), + tt_field_def("handle", "Handle", pinned=True, + profile={"source": PROFILE_SOURCE_TT}), + tt_field_def("full_name", "Name"), + tt_field_def("tt_id", "TikTok id"), + tt_field_def("profile_url", "Profile", "url"), + tt_field_def("bio", "Bio"), + tt_field_def("external_url", "Link in bio", "url"), + tt_field_def("verified", "Verified", "checkbox"), + tt_field_def("is_private", "Private", "checkbox"), + tt_field_def("is_business", "Business account", "checkbox"), + tt_field_def("followers", "Followers", "int"), + tt_field_def("following", "Following", "int"), + tt_field_def("posts_count", "Video count", "int"), + # TikTok gives THREE engagement rates where Instagram gives one. All three are stored ×100 for + # the same reason `avg_engagement` is on the IG side (C1-a): the vendor sends a 0–1 fraction and + # our `pct` renderer appends the sign to the stored number, so the raw fraction would print a + # 6.6% creator as `0.0%`. + tt_field_def("avg_engagement", "Avg engagement", "pct"), + tt_field_def("like_engagement", "Like engagement", "pct"), + tt_field_def("comment_engagement", "Comment engagement", "pct"), + # Total likes RECEIVED across the account's videos — a TikTok-only number with no Instagram + # equivalent, and one of the few profile-level engagement facts a vendor gives away. + tt_field_def("likes_received", "Likes received", "int"), + tt_field_def("country_code", "Country"), + tt_field_def("region", "Region"), + tt_field_def("predicted_lang", "Language"), + # ⚠ ACCOUNT AGE, NOT A MEASUREMENT STAMP. The vendor's `create_time` on a profile is when the + # ACCOUNT was created; it is emphatically not "when `followers` was true". TikTok carries no + # measurement timestamp at all, exactly like Instagram — which is why the append law below + # (`@`) is the only thing that can date a number. + tt_field_def("account_created_at", "Account created", "date"), + tt_field_def("first_found", "First found", "date"), + tt_field_def("last_found", "Last found", "date"), + tt_field_def("found_count", "Times found", "int"), + tt_field_def("created_by", "Found by"), + tt_field_def("enriched_at", "Enriched at", "date"), + tt_field_def("source", "Read via"), + tt_field_def("source_payload", "Source data", "json"), + tt_field_def("profile_snapshots_link", "Measurement rows", "link", + description="Profile measurements linked to this account.", + link={"table": TT_SNAPSHOTS_TABLE, "on": "influencer_key", "from": "handle"}), + tt_field_def("posts_link", "Post rows", "link", + description="Post records linked to this account.", + link={"table": TT_POSTS_TABLE, "on": "influencer_key", "from": "handle"}), +] + +#: The profile SERIES. One row per profile per pull, keyed `@` — the append law +#: from `instagram-capture.md` §3, unchanged, because its cause is unchanged: the vendor stamps +#: nothing, so the only honest date a number can carry is the moment WE read it. +TT_SNAPSHOT_FIELDS = [ + tt_field_def("platform", "Platform"), + tt_field_def("snapshot_key", "Snapshot"), + tt_field_def("influencer_key", "Influencer"), + tt_field_def("pulled_at", "Pulled at", "date"), + tt_field_def("followers", "Followers", "int"), + tt_field_def("following", "Following", "int"), + tt_field_def("posts_count", "Video count", "int"), + tt_field_def("likes_received", "Likes received", "int"), + tt_field_def("full_name", "Name"), + tt_field_def("bio", "Bio"), + tt_field_def("verified", "Verified", "checkbox"), + tt_field_def("is_private", "Private", "checkbox"), + tt_field_def("is_business", "Business account", "checkbox"), + tt_field_def("avg_engagement", "Avg engagement", "pct"), + tt_field_def("like_engagement", "Like engagement", "pct"), + tt_field_def("comment_engagement", "Comment engagement", "pct"), + tt_field_def("external_url", "Link in bio", "url"), + tt_field_def("tt_id", "TikTok id"), + tt_field_def("profile_url", "Profile", "url"), + tt_field_def("country_code", "Country"), + tt_field_def("region", "Region"), + tt_field_def("predicted_lang", "Language"), + tt_field_def("account_created_at", "Account created", "date"), + tt_field_def("source", "Read via"), + tt_field_def("approx", "Counts are approximate", "checkbox", + description="Checked when the counts on this row are rounded, not exact."), + tt_field_def("source_payload", "Source data", "json"), +] + +#: One row per TikTok post. Identity is `shortcode`, and on TikTok that is a 19-digit numeric id — +#: ✅ the SAME shape as the Comments dataset's `post_id`, so the comments→posts link joins on +#: equality with NO normaliser (measured in the probe doc; Instagram needed one). +TT_POST_FIELDS = [ + tt_field_def("platform", "Platform"), + tt_field_def("shortcode", "Post id"), + tt_field_def("influencer_key", "Influencer"), + tt_field_def("posted_at", "Posted at", "date"), + # ⛔ THE VENDOR'S VOCABULARY IS `"video"` / `"content"` AND OURS HAS NO `"content"`. Declaring + # the vendor's word would put an untranslated API token in front of a user; declaring an option + # the mapper can emit but the select does not list would fail `_clean_field`. So the OPTIONS + # stay this product's words and `connectors_tt.normalize_post` does the translation — the same + # posture every other vendor value here takes. `carousel` is listed because a TikTok photo post + # carrying more than one image IS one, and the mapper decides from `carousel_images` rather + # than from the type token, which cannot express it. + tt_field_def("type", "Type", "select", options=["image", "video", "carousel"]), + tt_field_def("caption", "Caption"), + tt_field_def("url", "URL", "url"), + tt_field_def("hashtags", "Hashtags"), + tt_field_def("tagged_location", "Commerce location"), + # ⛔ `views` AND NO `plays` — see the divergence note at the top of this section. One vendor + # number, one column. + tt_field_def("views", "Views", "int"), + tt_field_def("likes", "Likes", "int"), + tt_field_def("comments", "Comments", "int"), + tt_field_def("shares", "Shares", "int"), + tt_field_def("saves", "Saves", "int"), + tt_field_def("video_duration", "Video length (s)", "int"), + tt_field_def("measured_at", "Engagement read at", "date", + description="When the engagement numbers on this row were read."), + tt_field_def("source_payload", "Source data", "json"), + tt_field_def("comments_link", "Comment rows", "link", + description="Comment records linked to this post.", + link={"table": TT_COMMENTS_TABLE, "on": "shortcode", "from": "shortcode"}), + tt_field_def("post_snapshots_link", "Measurement rows", "link", + description="Engagement measurements linked to this post.", + link={"table": TT_POST_SNAPSHOTS_TABLE, "on": "shortcode", "from": "shortcode"}), + tt_field_def("measurements_captured", "Measurements captured", "rollup", + description="How many measurements this post has.", + rollup={"link": "post_snapshots_link", "fn": "countall"}), +] + +#: The POST series. ⚠ D-117 is live on the Instagram twin — `posted_at`/`type` are denormalised onto +#: snapshot rows there and never reconcile with the post row afterwards. That pattern is NOT +#: reproduced: this table carries the measurement and its identity, and asks `ut_tt_posts` for what +#: kind of post it was. A fact that can disagree with its own dimension table is a fact nobody can +#: trust, and the convenience it buys (one fewer hop in a rollup) is not worth a column that can be +#: wrong. +TT_POST_SNAPSHOT_FIELDS = [ + tt_field_def("platform", "Platform"), + tt_field_def("post_snapshot_key", "Snapshot"), + tt_field_def("shortcode", "Post id"), + tt_field_def("influencer_key", "Influencer"), + tt_field_def("pulled_at", "Pulled at", "date"), + tt_field_def("views", "Views", "int"), + tt_field_def("likes", "Likes", "int"), + tt_field_def("comments", "Comments", "int"), + tt_field_def("shares", "Shares", "int"), + tt_field_def("saves", "Saves", "int"), + tt_field_def("source_payload", "Source data", "json"), + tt_field_def("post_link", "Post", "link", + description="Post record linked to this measurement.", + link={"table": TT_POSTS_TABLE, "on": "shortcode", "from": "shortcode", + "single": True}), +] + +#: Comments. Same posture as Instagram's: the SCHEMA and the LINK ship, the CAPTURE is opt-in and +#: defaults OFF (`commentMetrics`), because comments are the single most expensive thing this +#: product can buy and they ingest identifiable third parties who never entered anybody's list +#: (D-24). ⚠ TikTok's `replies` is an ARRAY where ours is an INT count — a name collision, resolved +#: in the mapper by storing `num_replies` and leaving the array in Source data. +TT_COMMENT_FIELDS = [ + tt_field_def("platform", "Platform"), + tt_field_def("comment_key", "Comment"), + tt_field_def("shortcode", "Post id"), + tt_field_def("influencer_key", "Influencer"), + # ⭐⭐ OWNER RULING 2026-08-12, verbatim: *"why isn't any of the comments column actually HAS + # the comments content, fix it."* The text was bought, stored and INVISIBLE — retained whole in + # `source_payload` under the 2026-08-07 superseding ruling (`instagram-capture.md` §4b), but + # promoted to no column, so a person paying per comment record saw only Likes and Replies. + # ⛔ THE COMMENT TEXT ONLY — the commenter's IDENTITY (`commenter_user_name`, `commenter_id`, + # `commenter_url`; the vendor FLAGS the first as PII) stays in `source_payload` and gets no + # column. The ruling names the comments' CONTENT, and promoting a third party's name and + # profile URL into a filterable, exportable column is a different decision that nobody made. + tt_field_def("text", "Comment"), + tt_field_def("commented_at", "Commented at", "date"), + tt_field_def("likes", "Likes", "int"), + tt_field_def("replies", "Replies", "int"), + tt_field_def("source_payload", "Source data", "json"), + tt_field_def("post_link", "Post", "link", + description="Post record linked to this comment.", + link={"table": TT_POSTS_TABLE, "on": "shortcode", "from": "shortcode", + "single": True}), +] + +#: table key -> the field list that defines it. ⭐ DERIVED CONSUMPTION IS THE POINT: `ut_ensure`, +#: the gates and every future runner read THIS rather than naming five constants, so adding a sixth +#: `ut_tt_*` table is one entry instead of a sweep. The Instagram side has no equivalent map, which +#: is exactly why its table names are scattered across the module. +TT_TABLE_FIELDS = { + TT_PROFILE_TABLE: TT_PROFILE_FIELDS, + TT_SNAPSHOTS_TABLE: TT_SNAPSHOT_FIELDS, + TT_POSTS_TABLE: TT_POST_FIELDS, + TT_POST_SNAPSHOTS_TABLE: TT_POST_SNAPSHOT_FIELDS, + TT_COMMENTS_TABLE: TT_COMMENT_FIELDS, +} +#: The human labels a spawned `ut_tt_*` table wears in the database list. +TT_TABLE_LABELS = { + TT_PROFILE_TABLE: "TikTok profiles", + TT_SNAPSHOTS_TABLE: "TikTok profile measurements", + TT_POSTS_TABLE: "TikTok posts", + TT_POST_SNAPSHOTS_TABLE: "TikTok post measurements", + TT_COMMENTS_TABLE: "TikTok comments", +} +#: ⭐⭐ WAVE 31 · OWNER RULING R9 — the `ut_tt_*` children a person may not type records into. +#: Owner, verbatim: *"Tiktok database for post and comments and their snapshots should also be a +#: locked database with the lock icon, exactly like how instagram"*. +#: +#: ⛔ DECLARED ONCE BECAUSE THE DEFECT WAS TWO CALL SITES DISAGREEING WITH A THIRD. `ensure_ig_graph` +#: passes `record_mode=AUTOMATION_RECORD_MODE` for all four Instagram children; TikTok's two spawn +#: sites — `_spawn_presets`' child loop (SAVE) and `_tt_write_tables` (RUN) — each passed +#: `lock_fields=True` and no `record_mode` at all. That is the whole bug: not a missing feature, one +#: argument missing at two places, which is exactly the shape `_tt_write_tables`' own docstring warns +#: about (*"one function, two callers, so the inline and deferred paths cannot answer differently"*). +#: A SET plus a resolver means a sixth `ut_tt_*` table joins by declaration, not by remembering. +#: +#: ⚠ FOUR, NOT THE THREE R9 ENUMERATES, and the fourth is argued rather than assumed. +#: `ut_tt_snapshots` is the profile measurement series — the exact twin of `ut_ig_snapshots`, which +#: IS locked. R9's own comparison is *"exactly like how instagram"*, and the alternative is to leave +#: a machine-append time series that a person can type into, one table away from three that they +#: cannot. That is the fix-applied-to-one-platform-and-not-its-twin shape this very wave is closing +#: elsewhere (D-177(c)). `ut_tt_profile` is deliberately ABSENT: it is the database a human adds +#: handles to, and its Instagram counterpart is not locked either. +#: +#: ⚠ LOCKED DATABASE, NOT READ-ONLY (DESIGN.md §4): `records_mutable` goes False, so records cannot +#: be added/edited/deleted — **adding a FIELD must still work**, and a UI hiding add-field here is a +#: defect, not the intent. +TT_LOCKED_TABLES = frozenset({TT_SNAPSHOTS_TABLE, TT_POSTS_TABLE, TT_POST_SNAPSHOTS_TABLE, + TT_COMMENTS_TABLE}) +#: ⭐ THE INSTAGRAM HALF, NAMED. It was only ever the four keys `ensure_ig_graph`'s loop happens to +#: iterate, which is a set that exists but cannot be ASKED — so nothing could assert that IG and +#: TikTok lock the same shape, and the QA sweep below could not be written at all. +IG_LOCKED_TABLES = frozenset({IG_SNAPSHOTS_TABLE, IG_POSTS_TABLE, IG_POST_SNAPSHOTS_TABLE, + IG_COMMENTS_TABLE}) +#: ⭐⭐ W31 QA — OWNER, VERBATIM (2026-08-13): *"just like Instagram Post database (which is +#: locked), only the IG Profile and TT Profile should be editable."* That is this set plus its +#: complement: every `ut_ig_*`/`ut_tt_*` child is locked, and `ut_ig_profile` / `ut_tt_profile` are +#: the two a person types handles into. Registered into `core.user_tables` at import so the lock is +#: a DECLARATION rather than a flag some past automation run happened to stamp — see that module's +#: `_LOCKED_RECORD_KEYS` for why the stored flag alone left production unlocked. +#: ⛔ THE REGISTRATION ITSELF IS IN `main.py`, NOT HERE, and that is this module's own rule rather +#: than a preference: it imports NOTHING from `core` at module level (see `MAX_UT_ROWS` and +#: `MACHINE_OWNERS`, both local literals "so this module stays dependency-light for the API's boot +#: path"). Adding `import core.user_tables` at line 3570 would put `core` on the import path of +#: every process that touches the engine, to run one line. The composition root wires it, and +#: `verify_automation` asserts that main.py CARRIES that call — a declaration whose registrar is +#: missing is [[flag-shipped-without-its-writer]], which is the defect class this whole fix is in. +LOCKED_CHILD_TABLES = IG_LOCKED_TABLES | TT_LOCKED_TABLES + + +def tt_record_mode(table_key): + """R9's answer for one `ut_tt_*` key — the `record_mode` its `ut_ensure` must carry. + + `""` for anything outside the locked set, which is what `ut_ensure` already treats as "say + nothing about record mode", so an unlocked table is untouched rather than explicitly opened. + """ + return AUTOMATION_RECORD_MODE if str(table_key) in TT_LOCKED_TABLES else "" + + +# --------------------------------------------------------------------------------------------- +# THE INSTAGRAM CONNECTOR — moved out (wave 27 item 23). See `connectors_ig.py`. +# --------------------------------------------------------------------------------------------- +# Bright Data, Apify, the corpus routes and the anonymous ladder used to be ~1,900 lines HERE. +# They are a CONNECTOR: they know what a vendor's rows look like. Nothing in the automation +# runtime needs that, and the runtime is what this file is for. +# +# ⭐ THE LIST BELOW IS A DEPENDENCY STATEMENT, NOT A CONVENIENCE. It is every name the automation +# runtime still reaches for after the split — twenty, down from the seventy-one that used to be +# defined here — so `git diff` on this block is how anyone sees the engine growing a new vendor +# dependency. Re-exporting them is also REQUIRED rather than tidy: `routes_automation` and +# `routes_connectors` call `engine.bd_ready()` on the module object, and this is what keeps that +# true without those files having to learn where the wire went. +# ⚠ SO THE RE-EXPORT ALSO MEANS `engine.bd_call` STILL RESOLVES, and a gate that only checked +# that would be green whether or not anything moved. `verify_automation`'s `section_split` +# therefore asserts `__module__` on the whole moved set — it derives the split from the RUNNING +# system instead of trusting this import line ([[gate-answers-the-wrong-question]]). +# +# ⚠ AND IT IS DELIBERATELY HERE, mid-file, rather than at the top. `connectors_ig` reaches back +# for the SSRF rail (`fetch`/`fetch_json`/`Refused`) and for `PACE_SECONDS`; it does so lazily, +# inside its functions, precisely so the two modules cannot cycle. This import sits BELOW the rail +# and below `PACE_SECONDS`, so even if somebody later makes one of those reach-backs a +# module-level import, the names it wants already exist and the cycle still resolves. +# ⭐⭐ WAVE 30 · T09 (D-128) — TWO IMPORT BLOCKS NOW, AND THE SPLIT BETWEEN THEM IS THE POINT. +# `connectors_bd` is the SUPPLIER's wire — it serves both platforms and knows neither. `connectors_ig` +# is INSTAGRAM's vocabulary: its dataset ids, its row mappers, its free rungs. Anything that would +# have to change to serve a third platform belongs in the second block, not the first. +# ⚠ `bd_ready` is re-exported through this module ON PURPOSE — `routes_automation` and +# `routes_connectors` both reach `engine.bd_ready()` on the module object, and those files belong to +# other sessions. Dropping it here would 500 two surfaces that never mention a vendor. +from connectors_bd import ( # noqa: E402 + BD_EXCLUDE_MAX, BD_PATH_SNAPSHOT, BD_RECORD_PRICE_SPEC, + _bd_deferral, _bd_first_url, _bd_rows, + _first, _ig_int, + bd_call, bd_filter_rows, bd_filter_start, bd_filter_status, bd_ready, + bd_snapshot_progress, depth_refusal, +) +from connectors_ig import ( # noqa: E402 + BD_DS_COMMENTS, BD_DS_POSTS, BD_DS_PROFILES, BD_DS_REELS, + _bd_comment, _bd_post_metrics, _bd_profile, + _bd_tagged_location, + DEFERRED_MARK, + apify_profile, + bd_profiles_batch, + ig_handle, public_source, pull_profile, select_post_groups, top_up_views, +) + + +# --------------------------------------------------------------------------------------------- +# DISCOVERY — sourcing handles we do NOT already know (owner ruling R7, DEBT D-23) +# --------------------------------------------------------------------------------------------- +# The engine until now only ENRICHED a profile set somebody typed in. This queries the vendor's +# **620,000,000-record pre-collected Profiles corpus** with a real query language and returns +# handles nobody here has ever seen. MEASURED end-to-end 2026-08-04: `followers 10k–200k AND +# biography includes "floral"` returned five real profiles, one of them a verified 20.8k-follower +# Georgia/Florida floral-design company — Fisch Floral's actual market. +# +# ⛔ FIVE PROPERTIES, EACH OF THEM A MEASURED FAILURE MODE RATHER THAN A PREFERENCE: +# +# 1. **A DIFFERENT ROUTE AND A DIFFERENT NAMESPACE.** `POST /datasets/filter` — ⚠ with NO `/v3/` +# segment; `/datasets/v3/filter` is a 404 that once got written down as "the corpus is +# unreachable". Its snapshots are `snap_…` and are read at `/datasets/snapshot/…`; the +# scraper's are `sd_…` at `/datasets/v3/snapshot/…`, and the two 404 each other. +# 2. **`records_limit` IS REQUIRED.** Unbounded queries die `NOT_ENOUGH_FUNDS` (code 104, +# `per_set` billing). ⚠ And bounding is NOT sufficient: **scan time tracks PREDICATE BREADTH, +# not row count** — a `records_limit:10` query on a bare `followers > 10000` was still +# `building` 50 minutes later. So the predicate is capped too, and the runner tolerates a +# snapshot that is not ready when it looks. +# 3. **IT IS SLOW, AND THE RUN HANDS OFF RATHER THAN HOLDING A THREAD.** MEASURED delivery +# latency on the narrow query: **19.6 minutes** (`created` 12:38:00 → `delivery_time` +# 12:57:42). Blocking a worker thread for twenty minutes on a free-tier container to poll is +# the wrong shape, so a run polls for a short budget and then PERSISTS the snapshot id; the +# next run collects it. A `partial` that says "still building, the next run collects it" is +# the honest report of exactly what happened. +# 4. **PROMOTION IS MANUAL, ALWAYS (R7).** The measured result set was ~40% on-target and +# included a hashtag-aggregator account that is not a person. Auto-promoting a candidate into +# a wider profile set would silently multiply the enrichment bill AND pollute the +# snapshot series with rows nobody chose. This path never decides WHICH rows to enrich. +# 5. **SERIAL.** `429 too_many_parallel_jobs` is real and the original probe's most important +# test died on it. +# +# ⚠ THE PRICE IS NOT MEASURED AND MAY NOT BE PRESENTED AS IF IT WERE. The funds gate fires BEFORE +# the API returns a number, `price: 0` means "not priced" rather than "free", and +# `/customer/balance` answers 403 for our token — so there is no balance to read and no spend to +# report. Every estimate below is derived from the PRICING PAGE and says so. + +#: The table a discovery run writes when the tenant has no Instagram profile database yet — the +#: FIRST one, never a second (`discover_default_table` elects an existing one before this is +#: reached). Upsert by handle, so re-finding a profile is free. +#: +#: ⭐ 2026-08-10 — RENAMED FROM `ut_ig_candidates` / "IG candidates" (owner: *"We only need ONE IG +#: profile so it's not confusing"*). The old pair was two kinds of wrong at once. The KEY said +#: `candidates` while every other table in the graph says what it holds (`ut_ig_posts`, +#: `ut_ig_comments`, `ut_ig_snapshots`, `ut_ig_post_snapshots`) — and "candidate" was a concept +#: W26/R6 retired when it deleted `tracked`: a row here is a PROFILE, and whether anyone wants it +#: is a stage column's business, not the table's name. The LABEL then disagreed with the key on +#: any tenant who renamed the table, which is exactly the mismatch the owner hit. +#: +#: ⚠ SAFE TO MOVE ONLY BECAUSE NOTHING HOLDS THE OLD KEY. Census 2026-08-10 across all three +#: tenant stores: royal-imports ABSENT, gtmlab has no tables at all, and nurilab's was deleted the +#: same day (0 rows, 0 views, 0 dependent automations). An automation that stored the old key +#: explicitly still resolves to it — an explicit target is never rewritten — so a legacy tenant +#: would keep working; there simply is not one. +#: ⛔ A future rename is NOT this cheap. Once rows exist under a key, moving it is a migration +#: touching rows, `link.table`, every stored `targetTable` and the `ut__*` bucket family. +DISCOVER_TABLE = "ut_ig_profile" +#: The label that key is CREATED with. `ut_ensure` sets a label only on create and never relabels, +#: so changing this can rename nothing that already exists. One constant because it was three +#: copies of the same string literal, and a default spelled three times is one edit from drifting. +DISCOVER_LABEL = "IG profile" +#: A hard ceiling on one run's ask — and it is DERIVED, not chosen. A corpus row MEASURED at +#: ~35 KB (`file_size` 175,617 for 5 rows), so 1,000 records ≈ 33 MB would have crossed +#: `BD_MAX_KB` and come back as a truncated document **after being billed**. 500 leaves real +#: headroom. ⚠ If the row size grows, this number is the thing to re-derive. +BD_MAX_RECORDS = 500 +#: How long ONE run waits on a building snapshot before handing off to the next run (see 3 above). +BD_FILTER_WAIT = float(os.environ.get("AIOS_BD_FILTER_WAIT") or 120) +BD_FILTER_POLL = float(os.environ.get("AIOS_BD_FILTER_POLL") or 15) + +#: The operator set, enumerated BY REJECTION — send a bogus one and the API's own validation error +#: lists the legal set. The cheapest kind of measurement, and it makes this list a fact. +BD_FILTER_OPS = ("=", "!=", "<", "<=", ">", ">=", "in", "not_in", "includes", "not_includes", + "array_includes", "not_array_includes", "is_null", "is_not_null") +#: Operators that take NO value (everything else requires one). +BD_NULLARY_OPS = ("is_null", "is_not_null") + +#: Fields a predicate may name. MEASURED-accepted (24 of 25 probed; **`country_code` is REJECTED** +#: by the API and is therefore absent rather than offered-and-broken). +#: ⛔ `email_address`, `phone_number` and `business_email` ARE filterable and are DELIBERATELY +#: MISSING. Selecting people BY CONTACT DETAIL across 620M records is the materially heavier +#: privacy posture D-24 flags — a different licence question from per-URL enrichment, and one the +#: owner has not been asked. A vocabulary that cannot express it cannot be asked for it by +#: accident; adding them back is a decision, not a typo. +BD_FILTER_FIELDS = ("followers", "following", "posts_count", "avg_engagement", "biography", + "category_name", "business_category_name", "is_business_account", + "is_professional_account", "is_verified", "account", "full_name", + "external_url", "bio_hashtags", "post_hashtags", "profile_url", + "related_accounts", "profile_name", "id", "fbid", "highlights_count") +#: ⭐ The three the FIND SURFACE leads with, and the reason is measurement rather than taste: +#: these are the fields seen carrying values on real CORPUS rows. `category_name` and +#: `related_accounts` are filterable and were null/empty on every row we have looked at — a filter +#: on an unpopulated field returns nothing and looks exactly like "no such influencers exist". +BD_FILTER_LEAD = ("followers", "biography", "avg_engagement") + +#: ⭐⭐ WAVE 32 · T46 / DEBT D-167 — THE FILTER VOCABULARY HAS A PLATFORM NOW, AND IT COST MONEY +#: NOT TO. `BD_FILTER_FIELDS` above was measured against INSTAGRAM in wave 22; waves 29 and 30 hung +#: a second platform on the same tuple, so a `discover_tiktok` could be built on any of the **16 +#: fields Bright Data's TikTok Profiles dataset does not have** — and `BD_FILTER_LEAD`'s third +#: field, `avg_engagement`, is one of the three the Find panel LEADS with and is absent there. +#: A search on a field the corpus does not carry does not error: it returns nothing, and reads +#: exactly like *"no such creators exist"* after the money has been spent. +#: +#: ⛔ MEASURED, NOT REASONED — W30-B20, against TikTok's 40 declared dataset fields: **5 of the 21 +#: names survive.** They are these. Anything else is refused at the door with the field named, +#: which is `clean_predicates`' existing posture (*"refuses rather than coerces … the alternative +#: turns 'find me verified accounts in Georgia' into 'find me any account' and bills for the +#: difference"*) — now applied per platform rather than per product. +#: ⚠ NOT DERIVED FROM `connectors_tt.normalize_profile`, TEMPTING AS THAT IS. That mapper declares +#: the vendor's READ names on a returned row; these are the names its FILTER validator accepts, and +#: the two vocabularies are not the same thing on either platform (`awg_engagement_rate` reads, +#: `avg_engagement` filters). Deriving one from the other would look rigorous and be wrong. +TT_FILTER_FIELDS = ("followers", "following", "biography", "is_verified", "id") +#: The two of `BD_FILTER_LEAD`'s three that TikTok actually carries. `avg_engagement` is dropped +#: for the reason above — leading with a field the corpus cannot answer is worse than leading with +#: two. +TT_FILTER_LEAD = ("followers", "biography") + + +def filter_fields(kind=""): + """The searchable field names for one discovery kind — `(fields, lead)`. + + ⛔ ONE ACCESSOR, so the ROUTE that publishes the vocabulary and the VALIDATOR that enforces it + cannot come to disagree about it. That divergence is D-167 itself: the route served 21 names + with no platform dimension and `clean_predicates` validated against the same tuple with no kind + parameter, so both halves were consistently wrong together and nothing could notice. + """ + if str(kind or "") == "discover_tiktok": + return TT_FILTER_FIELDS, TT_FILTER_LEAD + return BD_FILTER_FIELDS, BD_FILTER_LEAD + +#: ⛔ C4 (wave 22) — THE TOO-BIG GUARD, and its unit is the PREDICATE, not the row count. +#: MEASURED (2026-08-04): `records_limit: 10` over a bare `followers > 10000` was still +#: `building` 50 minutes later — scan time tracks PREDICATE BREADTH; bounding the rows does not +#: bound the scan. What actually narrows a 620M-row corpus is CONTENT: a text/array match on a +#: field that discriminates. Numeric ranges and booleans partition the corpus into slabs the +#: scanner still has to walk, so they refine a search and cannot BE one. +BD_NARROWING_FIELDS = frozenset({ + "biography", "account", "full_name", "profile_name", "category_name", + "business_category_name", "external_url", "bio_hashtags", "post_hashtags", + "related_accounts", "id", "fbid", "profile_url"}) +#: The ops that actually pin content — `!=`/`not_includes` on a text field matches nearly the +#: whole corpus, which is breadth wearing a condition's clothes. +BD_NARROWING_OPS = frozenset({"=", "in", "includes", "array_includes"}) +#: Fields MEASURED carrying values on real corpus rows — D-25's fill-rate table, 50 mixed +#: profiles (snapshot `snap_msfrlbmt7qvej8uby`, collected 2026-08-05). The bar is ≥60% +#: populated: below it a filter misses more corpus than it matches, and the surface should +#: say so. MEASURED AND EXCLUDED: category_name 48%, business_category_name 34%, +#: related_accounts 22%, post_hashtags 18%, bio_hashtags 10% — and the PII trio the map never +#: carries anyway read 2%/0%/0%, so even the thing R3 forbids would barely have worked. +#: ⚠ posts_count was 100% populated on CORPUS rows — the fabricated-zero finding +#: (`_bd_posts_count`) is a SCRAPE-path fact and both stay true. +BD_POPULATED_FIELDS = frozenset({ + "account", "followers", "following", "posts_count", "avg_engagement", "biography", + "external_url", "is_business_account", "is_professional_account", "is_verified", + "full_name", "profile_name", "highlights_count", "id", "fbid", "profile_url"}) +BD_MIN_NARROWING = 1 + +# ── THE FIELD SCHEMA — what a person sees, and what they are allowed to ask. ────────────────── +# ⛔ THE SURFACE USED TO SHOW THE VENDOR'S OWN COLUMN NAMES AND ALL FOURTEEN OPERATORS ON EVERY +# ROW. So `is_business_account` offered `>=`, `fbid` sat in the list with no explanation of what +# it is, and `not_array_includes` was a thing a customer was expected to reason about. Owner: +# *"look at each damn schema and only provide toggles operator that make sense"*. +# +# ⚠ THE LABEL IS NOW LOAD-BEARING IN BOTH DIRECTIONS. The old comment in `AutomationFind.tsx` +# defended raw names on the grounds that a refusal names the field exactly as the row does — and +# it was right, which is why `field_label()` is used by the REFUSALS too. Prettifying only the UI +# is how you get an error message about `bio_hashtags` on a row labelled "Hashtags in bio". +BD_FIELD_LABELS = { + "followers": "Followers", "following": "Following", "posts_count": "Post count", + "avg_engagement": "Engagement rate", "biography": "Bio", "category_name": "Category", + "business_category_name": "Business category", "is_business_account": "Business account", + "is_professional_account": "Professional account", "is_verified": "Verified", + "account": "Handle", "full_name": "Name", "profile_name": "Profile name", + "external_url": "Link in bio", "profile_url": "Profile link", + "bio_hashtags": "Hashtags in bio", "post_hashtags": "Hashtags in posts", + "related_accounts": "Related accounts", "id": "Instagram ID", "fbid": "Facebook ID", + "highlights_count": "Story highlight count", +} +#: A one-line "what IS this" for the rows nobody can be expected to guess. Absent = self-evident; +#: a hint on all 21 rows is a hint on none of them (the wave-24 chip lesson). +BD_FIELD_HINTS = { + "account": "The @username", + "avg_engagement": "Likes and comments as a share of followers", + "id": "Instagram's internal number for the account. For matching a list you already have", + "fbid": "The linked Facebook id. For matching a list you already have", + "profile_name": "The name shown above the bio, when it differs from the account name", + "related_accounts": "Accounts Instagram suggests alongside this one", +} +#: The KIND decides the comparisons offered and the control drawn for the value. +BD_FIELD_KINDS = { + **{n: "number" for n in ("followers", "following", "posts_count", "avg_engagement", + "highlights_count")}, + **{n: "boolean" for n in ("is_business_account", "is_professional_account", "is_verified")}, + **{n: "tags" for n in ("bio_hashtags", "post_hashtags", "related_accounts")}, + **{n: "choice" for n in ("category_name", "business_category_name")}, + **{n: "text" for n in ("biography", "account", "full_name", "profile_name", "external_url", + "profile_url", "id", "fbid")}, +} +#: ⛔ ONLY WHAT MAKES SENSE, and the ORDER is the order they are offered in — the first entry of +#: each list is what `default_operator` must agree with. +BD_OPS_BY_KIND = { + # ⚠ `in`/`not_in` are DELIBERATELY ABSENT. "is any of" is what `=` becomes the moment a second + # value is typed (see `BD_MULTI_VALUE_OPS`), so offering both would be two controls for one + # idea — and the second one is the one written in vendor grammar. + "text": ("includes", "not_includes", "=", "is_not_null", "is_null"), + "choice": ("=", "!=", "is_not_null", "is_null"), + "number": (">=", "<=", "=", ">", "<"), + # A yes/no field has exactly one sensible comparison and a two-option value. `!=` on a boolean + # is `=` with the other value, written the confusing way. + "boolean": ("=",), + "tags": ("array_includes", "not_array_includes", "is_not_null", "is_null"), +} +#: Plain English for every comparison the surface can offer. The vendor's token stays on the wire +#: (it is what the API accepts); nobody has to read it. +BD_OP_LABELS = { + "includes": "contains", "not_includes": "does not contain", + "=": "is", "!=": "is not", "in": "is any of", "not_in": "is none of", + ">=": "at least", "<=": "at most", ">": "more than", "<": "less than", + "array_includes": "has any of", "not_array_includes": "does not have", + "is_null": "is empty", "is_not_null": "is not empty", +} +#: Yes/No, so a boolean is a two-item dropdown instead of a box you type `true` into. +BD_BOOLEAN_OPTIONS = [{"value": "true", "label": "Yes"}, {"value": "false", "label": "No"}] + +#: ⭐ THE OPERATORS THAT MAY CARRY SEVERAL VALUES — owner item 3, "if i want to include many +#: keywords like floral, flower, beauty". One condition row holds the list; `bd_filter_start` +#: expands it into a NESTED OR group, so it composes with the other conditions instead of forcing +#: the whole search to "match any" (which would also drag Followers into the union — the hole the +#: OR guard closed the same day). +#: +#: ⛔ POSITIVE OPERATORS ONLY, and this is a correctness line rather than a scope line. "does not +#: contain floral OR does not contain flower" matches nearly every account alive — a negative over +#: a list is an AND (De Morgan), and quietly OR-ing it would build a filter that reads like a +#: narrowing and behaves like the whole corpus. Negatives stay single-valued until someone needs +#: them enough to write the AND branch. +BD_MULTI_VALUE_OPS = frozenset({"includes", "=", "array_includes"}) +MAX_PREDICATE_VALUES = 12 + + +def field_label(name): + """The human name for a searchable field — used by the SURFACE and by every REFUSAL.""" + return BD_FIELD_LABELS.get(name) or str(name or "that field") + + +def field_kind(name): + return BD_FIELD_KINDS.get(name, "text") + + +def ops_for(name): + """The comparisons this field may be asked. A tuple, in offer order.""" + return BD_OPS_BY_KIND.get(field_kind(name), BD_OPS_BY_KIND["text"]) + +#: ⭐ WHAT A **FRESH** CONDITION ON A FIELD STARTS AS — and it lives HERE, beside the narrowing +#: law, because the two drifted apart and the drift cost a user their first automation. +#: +#: ⛔ THE SCAR, IN THREE WAVES. Wave 21 fixed a seeded condition the server refused for having no +#: value by making a new condition NULLARY (`is_not_null` — "this field has any value"), and its +#: comment called that "narrowing-in-the-right-direction". Wave 22 then wrote `BD_NARROWING_OPS` +#: and `is_not_null` is NOT IN IT, which silently made that default a condition the save door +#: always refuses. Wave 24 deleted the create wizard, so the Find panel became the ONLY way in — +#: and the refusal moved from a corner case to the first thing a new user meets. Three waves, one +#: sentence of drift, and nothing red anywhere at any point. +#: +#: So the rule is now DERIVED and ASSERTED (`verify_automation.py`): a fresh condition on a field +#: that CAN narrow must narrow. The client asks for `defaultOperator` per field and never picks +#: one itself — a client-side default is a second copy of this table, free to disagree with the +#: guard that judges it, which is exactly what happened. +BD_ARRAY_FIELDS = frozenset({"bio_hashtags", "post_hashtags", "related_accounts"}) +BD_BOOLEAN_FIELDS = frozenset({"is_business_account", "is_professional_account", "is_verified"}) +#: Narrowing fields whose values are IDENTIFIERS — a substring match on a handle or an id is a +#: worse question than an equality, and both narrow. +BD_EXACT_FIELDS = frozenset({"account", "id", "fbid", "profile_url"}) + + +def default_operator(name): + """The operator a NEW condition on `name` starts with. + + Narrowing wherever the field can narrow, so a fresh condition is ONE step (type a value) from + saveable rather than two (change the comparison, then type a value) — with the second step + being one the surface never told anyone to take. + """ + kind = field_kind(name) + if kind == "tags": + return "array_includes" + if kind in ("boolean", "choice"): + # ⚠ `choice` LANDS HERE AND NOT ON `includes`, which is the arm it used to fall through + # to (Category is a narrowing field). `includes` is not in `BD_OPS_BY_KIND["choice"]`, so + # the default would have been an operator the same module refuses to offer — the exact + # default-versus-guard split this function was written to close, reintroduced one commit + # later by adding a kind. Asserted per field in `verify_automation.py`. + return "=" + if kind == "number": + # `>=` cannot narrow the scan and is not pretending to — it is the comparison a person + # reaching for Followers means, and the guard's sentence is the honest answer when it is + # the ONLY condition present. + return ">=" + return "=" if name in BD_EXACT_FIELDS else "includes" + + +def predicate_narrows(p): + """Does ONE predicate narrow the corpus scan? Content field + content op, nothing else.""" + return (str((p or {}).get("name") or "") in BD_NARROWING_FIELDS + and str((p or {}).get("operator") or "") in BD_NARROWING_OPS) + + +def narrowing_refusal(preds, operator="and", kind=""): + """C4's server law: the sentence when a predicate set does not narrow, else ''. One function + so create/patch (via `clean_predicates`) and RUN (legacy stored configs predate the law) + refuse in the same words. + + ⭐ THE JOIN IS PART OF THE LAW, and it was missed for two waves. The guard asked "does ANY + predicate narrow?" — a question that is only correct under AND. Under **OR** the result is a + UNION, so the query is exactly as broad as its WIDEST branch: `biography includes "florist" OR + followers >= 10000` saved clean and asked the vendor for every account over ten thousand + followers, which is the measured 50-minute / NOT_ENOUGH_FUNDS shape the guard exists to stop. + A money wall that a dropdown three rows above it can walk through is not a wall. + + So: under OR every branch must narrow; under AND one is enough. An EMPTY set never narrows + either way — `all([])` is True, which would have made "no conditions at all" the widest legal + query of the lot. + """ + preds = list(preds or []) + if str(operator or "").lower() == "or": + if preds and all(predicate_narrows(p) for p in preds): + return "" + return ("with Match set to any, every condition has to describe the account itself " + "(Bio, Handle, Name, a hashtag). Matching any means the results are added " + "together, so one follower or yes/no condition widens the whole search. Narrow " + "every condition, or set Match to all") + if any(predicate_narrows(p) for p in preds): + return "" + # ⭐ WAVE 32 · T46 (D-167's neighbour) — THE NETWORK'S OWN NAME. This sentence said + # "Instagram" to somebody building a TIKTOK search, on the refusal they are most likely to + # read, because the string predates the second platform. `discovery_facts` already carries the + # network's name for exactly this class of string, so there is no second literal. + _net = discovery_facts(kind)[0] if kind else PLATFORM_INSTAGRAM + return ("add a condition that describes the account itself. Bio contains, Handle is, a " + f"hashtag. Follower counts and yes/no conditions alone match too much of {_net} " + "to search") + + +def filter_meta(): + """C4's per-field flags for the Find surface: `[{name, populated, narrowing, defaultOperator}]` + over the same 21 names `BD_FILTER_FIELDS` offers (the 3 PII fields stay structurally absent, + R3). + + `defaultOperator` rides here rather than being a client constant for the reason written over + `default_operator`: the client's own default was `is_not_null` for every field, which the + narrowing guard refuses on every field. + """ + return [{"name": n, + "label": field_label(n), + "hint": BD_FIELD_HINTS.get(n, ""), + "kind": field_kind(n), + "populated": n in BD_POPULATED_FIELDS, + "narrowing": n in BD_NARROWING_FIELDS, + "defaultOperator": default_operator(n), + # The comparisons THIS field may be asked, already in plain English and already in + # offer order. A client that filtered a global list by field kind would be a second + # copy of `BD_OPS_BY_KIND`, and the copy is what goes stale. + "operators": [{"value": o, "label": BD_OP_LABELS.get(o, o), + "nullary": o in BD_NULLARY_OPS, + # `in`/`is any of` and the tag operators take a LIST — the control + # has to know that, and deriving it from the token in the client is + # the same second-copy mistake one level down. + "multi": o in BD_MULTI_VALUE_OPS} + for o in ops_for(n)], + "options": BD_BOOLEAN_OPTIONS if field_kind(n) == "boolean" else []} + for n in BD_FILTER_FIELDS] + +#: ⭐ WAVE 25 · CONTRACT C1 — THE UNIFIED, CROSS-TENANT INSTAGRAM PRESET SET. +#: +#: ONE server-owned list. Every tenant gets the same keys and the same labels, which is what +#: "unified across tenant" means and what makes the pooled master series joinable at all — two +#: tenants calling the same number `followers` and `follower_count` is a pool you cannot query. +#: ⛔ THERE IS NO CLIENT COPY OF THIS LIST. C and D read it off the wire (`GET +#: /automations/presets`); a client-side copy is the wave-9 silent-drop failure in a new hat. +#: +#: ⭐ R3 — THESE HOLD **LATEST**, AND NOTHING ELSE. The full time series stays as append rows in +#: `ut_ig_snapshots`. ONE STORE FOR ONE SERIES: no per-record `json` history column, no second copy +#: of a number that already has a home. +#: ⛔ W29-T34 — THIS LINE USED TO ADD *"which the metric fields already read"*, AND THAT WAS WRONG. +#: A `metric` field reads the PLATFORM-WIDE MASTER, not this tenant's snapshot table: +#: `compute_metric_cells` imports `ig_master` and calls `ig_master.series_for(...)` — a +#: cross-tenant pooled repo in a different HF dataset, which is the whole point of C6's pooling and +#: which no rollup kind can reach. The metric header below says exactly that, so the module was +#: contradicting itself on which store answers the question. +#: ⚠ And the practical finding behind the correction, recorded because it is surprising: a live +#: census of all four tenants (13 tables / 228 fields / 64 store files) found **ZERO** metric fields +#: in existence, while 47 rollups are live. The vocabulary stays by owner ruling; nothing uses it. +#: +#: ⭐⭐ 2026-08-07 — THIS IS NOW *EVERY* PROFILE FIELD THE VENDOR RETURNS, BY OWNER INSTRUCTION, +#: AND THAT REVERSES D-25's FILL-RATE RULE. Owner, verbatim: *"make sure to have ALL Fields +#: available to us from Bright Data to be pre-set Fields for us and populated."* +#: +#: ⛔ THE RULE IT REPLACES IS WRITTEN DOWN HERE RATHER THAN DELETED, because it was a good rule +#: and the next reader will otherwise re-derive it and prune these columns back. It said: a +#: LATEST-value column costs something a snapshot row does not — it is a column every user sees on +#: their grid forever — so the split followed D-25's MEASURED fill-rates (50 mixed profiles, +#: snapshot `snap_msfrlbmt7qvej8uby`) and a field earned a column only at ≥60% populated. That is +#: why `business_category` (34%), `bio_hashtags` (10%), `post_hashtags` (18%) and `pronouns` were +#: captured into the snapshots and were NOT preset columns. +#: +#: ⚠ THE MEASUREMENT IS UNCHANGED AND STILL WORTH KNOWING — several of these columns WILL be +#: mostly blank on the scrape path, and D-25's central finding is why: **the scrape path and the +#: corpus path are different data.** `avg_engagement` is 0% populated on scrape and 88% on corpus; +#: `category` is 0% on scrape and 70% on corpus. So a blank here is very often "this ROUTE does +#: not carry it", not "this account does not have it" — which is exactly what `enriched_at` and +#: the blank-never-zero law exist to keep honest. What changed is the ANSWER to "does a +#: sometimes-blank column earn a place on the grid", and that was always the owner's call to make. +#: Source data retains every field the provider returns. The only separate decision is whether to +#: request the paid full Comments dataset (`commentMetrics`, default OFF). +#: +#: ⭐ WAVE 26 · R5 — THE NETWORK VOCABULARY. A handle is only unique WITHIN a network, so this is +#: half of the candidate identity (contract C3: the upsert key is `(platform, handle)`). +#: +#: ⚠ THE VALUES ARE DISPLAY STRINGS ON PURPOSE. They land in an ordinary `text` cell that a person +#: reads, filters and groups by, so `"Instagram"` beats `"ig"` — and the set is small, closed and +#: written down here rather than inferred from a runner's module name, because the day a second +#: runner spells it `"instagram"` is the day the dedup key stops working and nothing errors: the +#: profile simply appears twice. +#: ⭐ 2026-08-07 — WHERE A PROFILE HANDLE POINTS, as `core.user_tables.PROFILE_SOURCES` spells it. +#: A local literal for the same boot-path reason `MACHINE_OWNERS` and `UT_FIELD_TYPES` are locals +#: here, and held in step by a gate rather than an import. ⛔ NOT the same vocabulary as +#: `PLATFORM_*` below: this names the FLAG's source (which validator reads the cell), that names +#: the NETWORK a row belongs to (half of the identity). They read alike and mean different things, +#: which is exactly why both are written down instead of inferred. +PROFILE_SOURCE_IG = "instagram" +#: ⚠ Its TikTok twin, `PROFILE_SOURCE_TT`, is declared UP with the `ut_tt_*` schemas (wave 29): the +#: field lists there are evaluated at import and would not see a constant defined here. + +PLATFORM_INSTAGRAM = "Instagram" +#: ⭐ WAVE 29 — the `ut_tt_*` family stamps this on every row of all five of its tables. (It really +#: was written by nothing until D-9 landed; the note that said so is kept in the log, not here.) +PLATFORM_TIKTOK = "TikTok" +PLATFORM_FACEBOOK = "Facebook" +PLATFORMS = (PLATFORM_INSTAGRAM, PLATFORM_TIKTOK, PLATFORM_FACEBOOK) + + +def discovery_facts(kind): + """⭐⭐ WAVE 30 · T05 — THE THREE THINGS THAT GENUINELY DIFFER BETWEEN THE DISCOVERY KINDS: + the network's NAME, the database a run writes to when the config names none, and that + database's label. `(platform, table, label)`. + + ⛔ WHY A FUNCTION AND NOT A DICT LITERAL: `DISCOVER_TABLE` and `DISCOVER_LABEL` are declared + several hundred lines BELOW this point, so a dict evaluated here would NameError at import. + A function body resolves at call time and does not care. (`PROFILE_SOURCE_IG`'s note directly + above records the same import-order trap costing a constant its natural home.) + + ⛔ AND WHY IT EXISTS AT ALL. These three facts were written out inline at FIVE call sites — + `clean_config`, `graph`'s discovery arm, `graph`'s `det` dict, `_flow_table` and + `compose_sentence` — and wave 29 taught four of the five about TikTok by not touching them: + they tested the string `"discover_instagram"`, so a stored TikTok automation fell through to + a default meant for Instagram, or to no arm at all. Every one of those misses was SILENT and + every gate stayed green. `DISCOVERY_KINDS` answers *"is this a corpus search?"*; this answers + *"whose?"* — and between them a sixth platform is one tuple entry and one arm here, rather + than a hunt through the file for string comparisons somebody has to think to look for. + + ⚠ DELIBERATELY NOT A PLATFORM REGISTRY. The dataset ids, the field maps and the row mappers + stay in the connector modules that own them. This is the presentation-and-default quartet the + ENGINE needs, and widening it is how it becomes a second `SOURCES` with a longer name. + + ⭐ T06 ADDED THE FOURTH ELEMENT, `spawn_fields` — the field list `_presets_after_write` gives + the target database on SAVE. It belongs here rather than beside that function because it must + equal what the RUN-TIME `ut_ensure` uses, and wave 25's R10 exists precisely to stop those two + disagreeing: columns that change between Save and the first run are two different answers to + "what does this database look like". + ⚠ THE TWO PLATFORMS PASS DIFFERENT-LOOKING LISTS AND THAT IS CORRECT, not an oversight. + Instagram spawns `CANDIDATE_FIELDS` (= the preset set PLUS the discovery bookkeeping) because + its preset list alone is a subset of what its runner writes; TikTok's `TT_PROFILE_FIELDS` + already carries its own bookkeeping columns, so it IS the whole set. Each side is the list its + own runner ensures — which is the rule, rather than "both use the one named CANDIDATE". + """ + if kind == "discover_tiktok": + return (PLATFORM_TIKTOK, TT_PROFILE_TABLE, TT_TABLE_LABELS[TT_PROFILE_TABLE], + TT_PROFILE_FIELDS) + return PLATFORM_INSTAGRAM, DISCOVER_TABLE, DISCOVER_LABEL, CANDIDATE_FIELDS + +#: ⭐ WAVE 26 · R3 — THE TYPES ARE HONEST NOW, AND THE MIGRATION IS THE PRICE OF THAT. +#: +#: This block used to say the `text` types were deliberate, and the reasoning was sound as far as +#: it went: `ut_ensure` merges by KEY and never re-types an existing column, so re-declaring +#: `followers` as `int` gives NEW tables a schema every table already in production does not have +#: — one vocabulary with two shapes, the exact drift this list exists to prevent. That argument +#: was never wrong; it was an argument for doing the MIGRATION, and the note used it as a reason +#: not to. Owner, 2026-08-06: *"Why is everything that instagram field found is in a text format? +#: change this."* +#: ⛔ SO THE TWO HALVES ARE INSEPARABLE AND SHIP TOGETHER. Declaring a type here without +#: `migrate_ig_field_types()` reintroduces exactly the split-schema the old note feared — and it +#: would do it silently, because a merged-by-key field list produces no error when two tables +#: disagree about what a column IS. +#: +#: ⚠ `avg_engagement` CARRIES A UNIT CONVERSION, NOT JUST A TYPE (amendment C1-a). The vendor +#: sends a 0–1 fraction (MEASURED: 0.0074 / 0.0656 / 0.0014 / 0.0148 / 0.0274 / 0.0091) and this +#: product's `pct` renders the stored number with a `%` appended — so storing the raw fraction +#: would print every creator in the book as `0.0%`. It is stored ×100 from here on, at the +#: mapping and in the migration both. See `_pct100`. +PRESET_PROFILE_FIELDS = [ + # ⭐ WAVE 26 · R5 — WHICH NETWORK THIS HANDLE IS ON, and it is half of the identity now. + # `@inayma` on Instagram and `@inayma` on TikTok are two accounts owned by two different + # people as often as not, so the dedup key is (platform, handle) and never handle alone + # (C3). ⛔ NOT called `source`: `SNAPSHOT_FIELDS` already spends that word on *which rung + # answered* (`field_def("source", "Read via")`), and one word meaning two things across two + # tables in one product is the drift this whole list exists to prevent. + field_def("platform", "Platform"), + # ⭐⭐ 2026-08-07 (owner ruling) — THE HANDLE IS THE PRIMARY COLUMN *AND* THE ENRICH BINDING. + # Owner: *"if a user use this automation for instagram scraping, the Unique ID will always be + # REPLACED or made as the Handle. in any case the user can add their own record right and edit + # those records."* Two declarations, one field, and they are the same sentence read twice: + # + # `pinned` — the grid's identity column is `find(f => f.pinned) ?? fields[0]`, so WITHOUT + # this the primary is an accident of creation order. It is a DECLARATION, not a + # reorder: the stored field order is left alone (saved view orders keep working) + # and the client pins the column to position 0 wherever it sits. That is why this + # fixes tables that already exist without moving a single stored field. + # `profile` — C3/R7's flag. It makes `handle` the column `enrich_instagram` BINDS to + # (`profile_field_key` step 2), which is what turns "I added a profile to my + # database, how do I enrich it?" from an unanswerable question into a step. It + # also validates the cell (`@Name`, a pasted instagram.com link and a bare handle + # all normalise to the bare handle) and routes a typed value to the SHARED + # definition row rather than one user's overlay (C3-A1) — which is precisely what + # makes the owner's second clause true: a record you add BY HAND is a record the + # engine can read and enrich. + # + # ⛔ THE TWO KEYS MUST SURVIVE `_clean_field` UNCHANGED or `verify_api` W25-1 goes red. The flag + # is also why this field can never be retyped: `_clean_field` REFUSES a profile flag on + # anything but `text`. + field_def("handle", "Handle", pinned=True, profile={"source": PROFILE_SOURCE_IG}), + field_def("profile_url", "Profile", "url"), + field_def("full_name", "Name"), field_def("followers", "Followers", "int"), + field_def("following", "Following", "int"), + field_def("avg_engagement", "Avg engagement", "pct"), + field_def("bio", "Bio"), field_def("external_url", "Link in bio", "url"), + field_def("verified", "Verified", "checkbox"), field_def("category", "Category"), + # --- the enrichment half: what `run_field_instagram` already pulls, kept to the fields + # MEASURED ≥60% populated (see the note above). + field_def("posts_count", "Post count", "int"), + field_def("highlights_count", "Story highlight count", "int"), + field_def("is_business", "Business account", "checkbox"), + field_def("is_professional", "Professional account", "checkbox"), + # ⚠ STAYS `text`. It is a 17-digit opaque identifier, not a quantity — typing it `int` would + # invite a grid to sum it, and some of them exceed 2^53 so a JS reader would round it. + field_def("ig_id", "Instagram id"), + # --- ⭐ 2026-08-07: the rest of the vendor's profile schema, promoted by owner instruction + # (see the block comment above for what that reverses). Types are HONEST from birth, which + # costs nothing here: no table has ever carried these columns, so there is no stored `text` + # definition for the migration to convert — R3's conversion rule applies to the columns that + # already exist, and these are new everywhere. + field_def("business_category", "Business category"), + field_def("is_private", "Private account", "checkbox"), + field_def("bio_hashtags", "Bio hashtags"), + field_def("pronouns", "Pronouns"), + field_def("profile_name", "Profile name"), + field_def("is_joined_recently", "Joined recently", "checkbox"), + field_def("has_channel", "Has channel", "checkbox"), + field_def("partner_id", "Partner id"), + field_def("external_url_title", "Link title"), + # ⚠ `text` for `ig_id`'s reason, one identifier over: it is a name, not a quantity. + field_def("fbid", "Facebook id"), + field_def("related_accounts", "Related accounts"), + field_def("country_code", "Country"), + field_def("source_payload", "Source data", "json"), + # ⭐⭐ 2026-08-07 (owner instruction) — THE RELATION AND THE ROLLUPS OVER IT. + # + # Owner: *"an enrichment automation should spawn relevant Post/Comment database that is + # linked to the profile automatically… and this rollup needs to have formula that we can use + # to calculate things like average Views over last N posts."* Both halves are these five + # columns, and they are PRESETS rather than something a person assembles, because "linked + # automatically" is the instruction — a relation you have to wire up by hand is the feature + # not existing. + # + # ⛔ `from` IS NOT DECLARED, ON PURPOSE. `_link_from_key` falls back to the PROFILE-flagged + # column and then the pinned one, both of which are `handle` on every preset table — so this + # links correctly on a database where the handle column was renamed, and on one where the + # flag sits somewhere unexpected. Naming `handle` here would be the hard-coded subject + # [[gate-answers-the-wrong-question]] warns about, one layer down. + # ⚠ "Post rows", NOT "Post count" — one opens records and one reports the account total. + # Two count-like columns with the same label would be unreadable. Caught by the label-collision + # gate rather than on screen, which is what that gate is for. + field_def("posts_link", "Post rows", "link", + description="Post records linked to this profile.", + link={"table": IG_POSTS_TABLE, "on": "influencer_key"}), + field_def("profile_snapshots_link", "Profile history", "link", + description="Profile snapshots linked to this profile.", + link={"table": IG_SNAPSHOTS_TABLE, "on": "influencer_key"}), + field_def("post_snapshots_link", "Post measurement rows", "link", + description="Post engagement measurements linked to this profile.", + link={"table": IG_POST_SNAPSHOTS_TABLE, "on": "influencer_key"}), + field_def("comments_link", "Comment rows", "link", + description="Comment records linked to this profile.", + link={"table": IG_COMMENTS_TABLE, "on": "influencer_key"}), + # ⚠ THE ROLLUPS READ `ut_ig_posts`' OWN LATEST COLUMNS, which is why those exist — a rollup + # is ONE HOP (Airtable's rule and ours), and the engagement SERIES lives one table further + # out in `ut_ig_post_snapshots`. + # ⚠ 12 IS THIS MEASURE'S OWN WINDOW (`AVG_WINDOW_POSTS`), NOT THE CAPTURE CAP. It used to be + # `MAX_POSTS_PER_PULL` and read "the vendor's ceiling" — both halves are now wrong: the cap is 30 + # (owner, 2026-08-09) and 30 is not the vendor's limit either. Binding these four to the cap meant + # raising it silently redefined every column named `_12`. `sortBy` is mandatory alongside a + # `limit`, and this is why — "the last 12" has to name what makes one post later than another. + # ⭐ UN-RETIRED 2026-08-08 together with the column it averages. It was retired for four hours + # because its input was an account-grain constant; with the input now a real per-reel + # measurement the average means what its label says again. + field_def("avg_views_12", "Avg views · last 12 posts", "rollup", + rollup={"link": "posts_link", "field": "views", "fn": "average", + "limit": AVG_WINDOW_POSTS, "sortBy": "posted_at", "sortDir": "desc", + "distinctBy": "shortcode"}), + field_def("avg_plays_12", "Avg plays - last 12 posts", "rollup", + rollup={"link": "posts_link", "field": "plays", "fn": "average", + "limit": AVG_WINDOW_POSTS, "sortBy": "posted_at", "sortDir": "desc", + "distinctBy": "shortcode"}), + field_def("avg_likes_12", "Avg likes · last 12 posts", "rollup", + rollup={"link": "posts_link", "field": "likes", "fn": "average", + "limit": AVG_WINDOW_POSTS, "sortBy": "posted_at", "sortDir": "desc", + "distinctBy": "shortcode"}), + field_def("avg_comments_12", "Avg comments · last 12 posts", "rollup", + rollup={"link": "posts_link", "field": "comments", "fn": "average", + "limit": AVG_WINDOW_POSTS, "sortBy": "posted_at", "sortDir": "desc", + "distinctBy": "shortcode"}), + # ⭐ THE ONE HONEST POST COUNT WE HAVE. D-82: the vendor's `posts_count` is a FABRICATED ZERO + # on the paid rung (49/49 rows measured), so it is discarded and that column reads blank after + # a paid enrich. This counts the post rows actually captured — a different number and a true + # one, which is why it gets its own column and its own label rather than quietly filling + # `posts_count` with something that is not what that field means. + field_def("posts_captured", "Posts captured", "rollup", + rollup={"link": "posts_link", "fn": "countall", "distinctBy": "shortcode"}), + field_def("profile_reads", "Profile reads", "rollup", + rollup={"link": "profile_snapshots_link", "fn": "countall", + "distinctBy": "snapshot_key"}), + field_def("post_measurements_captured", "Post measurements captured", "rollup", + rollup={"link": "post_snapshots_link", "fn": "countall", + "distinctBy": "post_snapshot_key"}), + field_def("comments_captured", "Comments captured", "rollup", + rollup={"link": "comments_link", "fn": "countall", + "distinctBy": "comment_key"}), + # ⭐⭐ WAVE 27 ITEM 16 — WHERE THIS CREATOR ACTUALLY IS, DERIVED, FOR NOTHING. + # + # ⛔ THE VENDOR DOES NOT SELL THIS. `country_code` was MEASURED `None` on every corpus row we + # have ever looked at, and the filter API REFUSES it as a predicate — so residency is the one + # thing a buyer most wants and the one thing the profile row cannot answer. The competitor + # teardown found the same gap solved by GUESSING from bio words, two buckets deep + # ([[janney-ai-teardown]]). + # + # ⭐ WE ARE ALREADY HOLDING THE ANSWER AND WERE THROWING IT AWAY. Each post carries the place + # it was tagged in, and the enrich has ALREADY BOUGHT twelve of them: `tagged_location` on the + # post row, plus the vendor's fuller `location`/`location_details` inside the paid + # `source_payload` that `_bd_tagged_location` normalises away. The modal city across a + # creator's own posts is a far better residency signal than a word in a bio, and it costs a + # dict comprehension. **Zero new vendor spend** — this is the whole reason it is a v1. + # + # ⚠ IT IS A GUESS AND THE COLUMN SAYS SO, IN ITS NAME AND IN ITS NEIGHBOUR. A travel creator + # posting from twelve cities gets a low confidence rather than a confident wrong answer, and + # `location_confidence` is the number a person filters on before trusting the guess. A single + # geotagged post produces NO guess at all — n=1 dressed as a pattern is what + # `SEED_MIN_ROWS_SHARING` refuses fifty lines up, and it would read as 100% certain. + field_def("location_guess", "Location (guess)"), + field_def("location_confidence", "Location confidence", "pct"), + # ⭐ R3's stamp. WITHOUT IT THE WHOLE SET IS UNREADABLE: a blank `followers` means "never + # enriched" and a stale one means "enriched in March", and no cell on the row can tell them + # apart. It is the single field that turns the other fifteen from numbers into measurements. + field_def("enriched_at", "Enriched at", "date"), + # ⭐ WAVE 26 · R1 — THE LAST-N POST WINDOW, AND IT IS A VIEW RATHER THAN A STORE. + # ⛔ READ THE R3 NOTE ABOVE BEFORE CHANGING THIS. "One store for one series" still holds: the + # authoritative post record is `ut_ig_posts` (keyed by shortcode, so it ACCUMULATES) and the + # authoritative engagement series is `ut_ig_post_snapshots` (append-per-pull, carrying + # views/likes/comments at a `pulled_at`). This cell is a DERIVED window over those two, + # rewritten each run, so a person reading the profile row can see the recent posts without a + # join — and deleting it would cost a convenience, never a measurement. Shape: contract C2. +] +#: The keys the preset set owns — derived, so a field added above cannot be forgotten here. +PRESET_PROFILE_KEYS = tuple(f["key"] for f in PRESET_PROFILE_FIELDS) + +#: ⭐ 2026-08-07 — WHICH preset field declares the primary column, and WHICH declares the profile +#: flag. DERIVED for the same reason `PRESET_PROFILE_KEYS` is, and the negative control is what +#: argued for it: with `"handle"` hard-coded in the migration, stripping the declaration left the +#: migration cheerfully stamping a column the product no longer claimed. Moving a declaration now +#: moves the migration with it, and REMOVING one stops the migration rather than leaving it to +#: enforce a rule nobody declares any more. +#: ⚠ Both are `""` when nothing declares them, and every reader treats `""` as "do nothing" — +#: never as "field number zero". +PRESET_PINNED_KEY = next((f["key"] for f in PRESET_PROFILE_FIELDS if f.get("pinned")), "") +PRESET_FLAG_KEY = next((f["key"] for f in PRESET_PROFILE_FIELDS if f.get("profile")), "") + +#: Discovery's table is the preset set PLUS its own bookkeeping. ⛔ DERIVED, NEVER RE-TYPED: the +#: alternative is two lists that describe the same columns and drift one label at a time, which is +#: the failure C1 exists to prevent. The five below are facts about the SEARCH (how often we found +#: them, who for, and a human's decision) rather than about the profile, so they are discovery's +#: and not part of the cross-tenant set. +CANDIDATE_FIELDS = [ + *PRESET_PROFILE_FIELDS, + # ⭐ WAVE 26 · R3 — `first_found` / `last_found` ARE DATES, not ISO strings in a text cell. + # They were written by `_iso()`, so the cell read `2026-08-05T14:03:11+07:00` and the owner + # called that format "extremely confusing" — correctly: it is a machine timestamp shown to a + # person, unsortable as a date and unfilterable by "last 7 days". The STAMP still carries its + # offset everywhere it is a time axis (`ut_ig_snapshots.pulled_at` is untouched); what + # changed is that "when did we first see this account" is a DAY, and a day is all anybody + # asks it for. `migrate_ig_field_types()` converts the stored strings. + field_def("found_count", "Times found", "int"), + field_def("first_found", "First found", "date"), + field_def("last_found", "Last found", "date"), + # ⛔ DEBT D-73 — THIS NOTE STATED A RETIRED LAW AS CURRENT FACT, directly above the field it + # describes, which is the first place anybody looks up what this column means. It read: *"the + # candidate pool is PER-USER — the upsert key is the COMPOUND (handle, created_by), so two + # people discovering the same profile each get their own row, their own found_count and their + # own review card."* Wave 26 · R4/R5 retired every clause of that. + # + # ⭐ THE CURRENT LAW: the identity is `(platform, handle)` and the TENANT is the unit. Two + # people in one workspace who discover the same profile share ONE row, one `found_count` and + # one history — because they are looking at one company's leads, not two private lists. + # `created_by` survives as an INFORMATIONAL stamp only: "Found by", answering who saw it + # first. It is no part of the dedup key, and a run must never branch on it. + # ⚠ Stamped by the RUNNER (the automation's creator), never by `_candidate_row` — unchanged, + # and the one clause of the old note that was still true. + field_def("created_by", "Found by"), + # ⛔ WAVE 26 · R6 — `tracked` WAS HERE AND IS DELETED. Owner, 2026-08-06: *"We already have + # a Field called stage to track the progress of the Automation per Record. We don't need + # another checkbox for this."* Correct, and it had been true since wave 22 shipped the stage + # column: every candidate carried BOTH a `stage_` select saying where it was AND a + # boolean saying whether it was kept — two progress fields that could disagree, with no rule + # about which one won. The stage field is the one progress column. Nothing replaces this. +] + + +def _profile_backlink_field(table_key, table_label, profile_key): + """A deterministic reciprocal link from one canonical IG table to one profile database. + + The key includes the target table identity, so ten separate Profile databases can all point + into the same canonical Posts/Comments/history tables without one relation overwriting the + next. The join is derived in both directions and therefore needs no cross-table fan-out write. + """ + digest = hashlib.sha1(str(table_key).encode("utf-8")).hexdigest()[:10] + return field_def( + f"profiles_{digest}", f"Profiles - {str(table_label or table_key)[:48]}", "link", + description=f"Profile records from {str(table_label or table_key)[:48]} linked to this row.", + link={"table": str(table_key), "on": str(profile_key), "from": "influencer_key"}, + ) + + +def _profile_schema_for(bound_key): + """Preset fields for an existing Profile table without inventing a second identity column.""" + out = [] + for field in PRESET_PROFILE_FIELDS: + item = dict(field) + if item.get("key") == PRESET_FLAG_KEY and bound_key != PRESET_FLAG_KEY: + item.pop("profile", None) + out.append(item) + return out + + +def _tt_profile_schema_for(bound_key): + """⭐ WAVE 30 · T08 — TikTok preset columns for a database whose profile column is `bound_key`. + + ⛔ NOT `_profile_schema_for` WITH A LIST ARGUMENT, and the difference is not stylistic. That + function walks `PRESET_PROFILE_FIELDS` and only ever removes the flag from `PRESET_FLAG_KEY`, + because on the Instagram side the flag lives on exactly one known column. TikTok's binding may + be ANY column a person named (`profile_field_key` step 1), so the rule here has to be stated + the other way round: **every field that is not the bound one arrives as DATA**, stripped of + both the identity flag and `pinned`. + ⚠ Otherwise a database whose TikTok column is `creator` would gain a rival `handle` carrying + `profile: {source: "tiktok"}` — a SECOND profile column, which `user_tables` refuses at both + write doors, so the whole top-up would be rejected and every preset cell would then be dropped + for want of a column. One shared helper for the save-time and run-time paths, so those two + cannot answer "which columns does a TikTok enrich need" differently. + """ + out = [] + for field in TT_PROFILE_FIELDS: + item = dict(field) + if str(item.get("key") or "") != str(bound_key or ""): + item.pop("profile", None) + item.pop("pinned", None) + out.append(item) + return out + + +def _locked_ig_field(field): + """One canonical Instagram field with the immutable preset declaration attached.""" + item = dict(field) + automation = dict(item.get("automation") or {}) + automation["preset"] = True + item["automation"] = automation + return item + + +def _profile_binding(table): + """The declared Profile identity field, falling back only to the canonical handle.""" + fields = list((table or {}).get("fields") or []) + flagged = next((str(f.get("key") or "") for f in fields + if isinstance(f.get("profile"), dict)), "") + if flagged: + return flagged + return PRESET_FLAG_KEY if any(f.get("key") == PRESET_FLAG_KEY for f in fields) else "" + + +def _ig_schema_contract(rt, profile_tables=None): + """The complete per-tenant Instagram graph contract, including every Profile backlink. + + The fixed child tables are shared. Profile tables are discovered structurally, so an older + user-named target and the built-in candidate database receive the same preset columns, Links, + Rollups, locks, and reciprocal fields. Retired machine keys are the only columns deleted. + """ + tables = ut_all(rt) + # ⭐ WAVE 32 · T42 — `_ig_contract_tables`, NOT `_ig_profile_tables`: this function decides + # which databases RECEIVE Instagram's 48 columns, and the owner's rule is that a database gets + # them only if it is used for Instagram. See that function for why the detector stays wider. + profile_keys = sorted(set(profile_tables if profile_tables is not None + else _ig_contract_tables(rt))) + backlinks, wanted, drops = [], {}, { + IG_SNAPSHOTS_TABLE: {"post_hashtags"}, + # ⚠ `views` IS DELIBERATELY ABSENT FROM THIS DROP SET AGAIN. It was dropped earlier today + # while it carried Bright Data's account-grain number; it is now declared in POST_FIELDS + # and fed by the `ig_post_views` capability. Leaving it here would have made the migration + # delete, on every authenticated read, the column the same release just added — the + # drop set and the field list are two halves of ONE contract and must move together. + } + for table_key in profile_keys: + table = tables.get(table_key) or {} + bound = _profile_binding(table) + if not bound: + continue + label = str(table.get("label") or table_key) + backlinks.append(_profile_backlink_field(table_key, label, bound)) + schema = _profile_schema_for(bound) + # ⭐⭐ 2026-08-10 (owner: *"retire the old hardcoded method and replace the Instagram + # database work with correct Rollup and Link fields"*) — THE DERIVED OVERLAY, AND IT + # BELONGS HERE RATHER THAN IN A SEPARATE MIGRATION. + # + # ⛔ A standalone converter would LOSE. `_reconcile_ig_graph_fields` overwrites every + # contract key it owns — `rollup` included, by its own note — so a column converted beside + # the contract is re-typed back to `int` on the next reconcile, silently, hours later. + # Making the CONTRACT itself say "this column is derived" leaves exactly one writer of the + # schema instead of two that disagree. + # + # ⚠ THE SET IS PER TENANT AND IS A PROOF, NOT A LIST. `_derived_profile_columns` converts a + # column only where this tenant's own rows show the fold already equals the stored value — + # measured, because a named list would have blanked 221 live cells on nurilab alone + # (`avg_engagement` and `category`, 62 each). A tenant whose series is thinner converts + # fewer columns and loses nothing; as its history fills in, later passes convert more. + derived = set(_derived_profile_columns(table, tables.get(IG_SNAPSHOTS_TABLE) or {})) + wanted[table_key] = [_as_derived_field(f) if str(f.get("key")) in derived else f + for f in schema] + # `tracked` was the pre-Stage progress checkbox. Keeping both allows two progress states + # to disagree, so it is retired by key just like the old nested Posts JSON. + drops[table_key] = {"posts", "post_hashtags", "tracked"} + wanted.update({ + IG_SNAPSHOTS_TABLE: [*SNAPSHOT_FIELDS, *backlinks], + IG_POSTS_TABLE: [*POST_FIELDS, *backlinks], + IG_POST_SNAPSHOTS_TABLE: [*POST_SNAPSHOT_FIELDS, *backlinks], + IG_COMMENTS_TABLE: [*COMMENT_FIELDS, *backlinks], + }) + return wanted, drops + + +# Only a unit-changing retype needs a cell conversion. Integer/checkbox/date cells already use +# the scalar strings their renderers expect; engagement is the exception because Bright Data's +# 0-1 fraction becomes this product's 0-100 percentage. +_IG_RETYPE_CONVERTERS = { + (IG_SNAPSHOTS_TABLE, "avg_engagement"): _pct100, +} + + +def _reconcile_ig_graph_fields(rt, wanted_by_table, drop_by_table=None): + """Repair machine-owned IG field declarations in one coalesced store update. + + `ut_ensure` deliberately merges only missing keys. That is correct for user columns but not + sufficient for a canonical relation: a stale `link.table`, rollup function, or type would + survive forever. This pass overwrites the contract keys for the fields we own, preserves + unrelated/user fields, and removes only explicitly retired preset columns plus their cells. + """ + drop_by_table = drop_by_table or {} + wanted_by_table = { + key: [_locked_ig_field(f) for f in fields] + for key, fields in wanted_by_table.items() + } + changed = False + + def _up(cur): + nonlocal changed + cur = cur if isinstance(cur, dict) else {} + for table_key, wanted_fields in wanted_by_table.items(): + table = cur.get(table_key) + if table is None: + continue + wanted = {str(f.get("key")): dict(f) for f in wanted_fields} + drops = set(drop_by_table.get(table_key) or ()) + fields, seen = [], set() + drops = _guarded_drops(table, drops) + for stored in table.get("fields") or []: + key = str(stored.get("key") or "") + if key in drops: + changed = True + continue + desired = wanted.get(key) + if desired is None: + fields.append(stored) + continue + # ⭐⭐ 2026-08-09 (owner: *"everything is custom and changeable always"*) — A + # COLUMN A HUMAN HAS TAKEN OVER IS LEFT ALONE. Every key below is overwritten + # from the shipped contract, `rollup` included, so without this the newly + # unlocked "edit a preset rollup" would save, render, recompute and then revert + # at the next enrichment run — an edit that looks like it worked and undoes + # itself hours later. `user_tables.user_edited` is the ONE reader of the stamp. + if _ut().user_edited(stored): + fields.append(stored) + seen.add(key) + continue + repaired = dict(stored) + for dkey, value in desired.items(): + if dkey != "automation": + repaired[dkey] = value + automation = dict(stored.get("automation") or {}) + automation.update(desired.get("automation") or {}) + repaired["automation"] = automation + # These bags define behaviour, not decoration. If the desired field does not use + # one, a stale bag from an earlier type must not remain attached to it. + for bag in ("link", "rollup", "profile", "pinned"): + if bag not in desired: + repaired.pop(bag, None) + if desired.get("type") not in ("select", "multiselect"): + repaired.pop("options", None) + converter = _IG_RETYPE_CONVERTERS.get((table_key, key)) + if (converter and stored.get("type") in ("text", "", None) + and repaired.get("type") != stored.get("type")): + for row in (table.get("rows") or {}).values(): + if str(row.get(key) or "").strip(): + converted = converter(row.get(key)) + if converted: + row[key] = converted + changed = True + if repaired != stored: + changed = True + fields.append(repaired) + seen.add(key) + for key, desired in wanted.items(): + if key not in seen and not any(str(f.get("key") or "") == key for f in fields): + fields.append(desired) + changed = True + if drops: + for row in (table.get("rows") or {}).values(): + for key in drops: + if key in row: + row.pop(key, None) + changed = True + table["fields"] = fields + return cur + + # Avoid a commit when the declarations are already exact. + snapshot = ut_all(rt) + needs = False + for table_key, wanted_fields in wanted_by_table.items(): + if table_key not in snapshot: + continue + table = snapshot.get(table_key) or {} + by_key = {str(f.get("key") or ""): f for f in table.get("fields") or []} + # ⚠ THE SAME GUARD, OR THE MIGRATION STOPS BEING IDEMPOTENT. This precheck exists to skip + # the commit when nothing would change; a spared `tracked` column left in the drop set + # answers "yes, work to do" on every single call, forever, and the schema pass would + # rewrite the table on every authenticated read without ever changing a byte. + drops = _guarded_drops(table, drop_by_table.get(table_key) or ()) + if drops & set(by_key): + needs = True + break + for desired in wanted_fields: + stored = by_key.get(str(desired.get("key") or "")) + mismatch = stored is None + if stored is not None: + for key, value in desired.items(): + if key == "automation": + if any((stored.get("automation") or {}).get(k) != v + for k, v in value.items()): + mismatch = True + break + elif stored.get(key) != value: + mismatch = True + break + if not mismatch: + stale_bags = {"link", "rollup", "profile", "pinned"} - set(desired) + mismatch = any(k in stored for k in stale_bags) + if (not mismatch and desired.get("type") not in ("select", "multiselect") + and "options" in stored): + mismatch = True + if mismatch: + needs = True + break + if needs: + break + if needs: + rt.update(UT_STORE_KEY, _up, flush="sync") + return changed + + +def ensure_ig_graph(rt, username="automation", flow_tag="", profile_table="", profile_field=""): + """Ensure the ONE per-tenant Instagram relational graph and return its canonical keys. + + Every enrichment path calls this function. Fixed table keys prevent a flow aimed at a new + Profile database from spawning `IG posts 2`; deterministic reciprocal link fields connect + each Profile database to the same Posts, Comments, profile-history, and post-history stores. + """ + profile = ut_get(rt, profile_table) if profile_table else None + bound = str(profile_field or "").strip() + if profile is not None and not bound: + bound = next((str(f.get("key") or "") for f in profile.get("fields") or [] + if isinstance(f.get("profile"), dict)), "") + profile_label = str((profile or {}).get("label") or profile_table) + backlink = [_profile_backlink_field(profile_table, profile_label, bound)] \ + if profile_table and profile is not None and bound else [] + + # ⭐ WAVE 32 · T41 — DERIVED from `IG_TABLE_FIELDS`/`IG_TABLE_LABELS`, which used to be this + # literal. The backlinks stay HERE because they are per-tenant, per-profile-database facts; the + # schema is a module constant. Splitting it that way is what let A's delivery sweep ask this + # module for Instagram's child set instead of hand-typing a fifth copy of these four names. + graph = {key: (IG_TABLE_LABELS[key], [*fields, *backlink]) + for key, fields in IG_TABLE_FIELDS.items()} + keys = {} + for key, (label, fields) in graph.items(): + keys[key] = ut_ensure(rt, label, fields, username, key=key, flow_tag=flow_tag, + record_mode=AUTOMATION_RECORD_MODE, lock_fields=True) + + if profile_table and profile is not None and bound: + profile_fields = _profile_schema_for(bound) + ut_ensure(rt, profile_label or profile_table, profile_fields, username, + key=profile_table, flow_tag=flow_tag, lock_fields=True) + # Reconcile the WHOLE tenant graph, not just this run's target. That is what keeps an older + # Profile database and IG candidates on the same universal schema while all of them point to + # the same four children. Existing flow provenance is merged, never replaced. + wanted, drops = _ig_schema_contract(rt) + _reconcile_ig_graph_fields(rt, wanted, drops) + return keys + + +# ── ⭐ WAVE 26 · THE MIGRATION (owner rulings R3 + R4/R5, contracts C1-a and C3) ─────────────── +# +# Two changes land on data that already exists, and neither is optional once the field defs move: +# +# R3 the preset columns take honest types, so the CELLS have to match them — an ISO stamp in a +# `date` column and a "20872" string in an `int` column are the split-schema the old +# "types are text on purpose" note correctly feared. Re-typing without converting is worse +# than not re-typing. +# R4 the candidate identity becomes `(platform, handle)`, so rows that exist today as +# `(handle, created_by)` DUPLICATES must be merged rather than left to collide. +# +# ⛔ IDEMPOTENCY IS THE WHOLE DESIGN, and the key is the STORED FIELD TYPE — never a flag, never a +# marker column. Run twice, an `avg_engagement` of 0.0074 would go 0.74 then 74, and nothing would +# look wrong until somebody read a 74% engagement rate off a creator with 3,000 followers. So a +# column is converted only while its STORED definition still says `text`; the moment it says `pct` +# the work is done and re-running is a no-op. That makes the migration safe to call on every boot, +# which is what it is for. +# +# ⚠ THE TABLE LIST IS DERIVED, NOT NAMED — D-71's lesson, one wave old and already paid for twice. +# `observed_categories` named its two source tables and went silently empty the day wave 25 let a +# user point an automation somewhere else. Naming tables here would leave exactly those user-named +# databases on the old schema, which is the same bug wearing a migration's clothes. + +#: A table is an Instagram profile table if it declares `handle` plus a real slice of the preset +#: vocabulary. Deliberately structural: it finds `ut_beauty_influencer_leads` without being told. +MIGRATE_MIN_PRESET_FIELDS = 3 +#: key -> the converter its new type needs. `checkbox` needs none (the writers already emit '1'/''). +_MIGRATE_CONVERT = { + "first_found": _day, "last_found": _day, "enriched_at": _day, + "avg_engagement": _pct100, +} + + +#: ⭐⭐ WAVE 30 · T08 — THE COLUMNS THAT CAN ONLY BE TIKTOK'S, DERIVED rather than typed out, so +#: that adding a column to either declaration keeps this set correct instead of quietly emptying +#: it. Eight today: `tt_id`, `like_engagement`, `comment_engagement`, `likes_received`, `region`, +#: `predicted_lang`, `account_created_at`, `source`. +TT_ONLY_PROFILE_KEYS = (frozenset(f["key"] for f in TT_PROFILE_FIELDS) + - frozenset(PRESET_PROFILE_KEYS) + - frozenset(f["key"] for f in CANDIDATE_FIELDS)) + + +def _is_tt_profile_table(tbl): + """⭐⭐ WAVE 30 · T08 — is this database a TIKTOK profile table? + + ⛔ THE DEFECT THIS EXISTS TO CLOSE WAS LIVE AND SHIPPED, and it converted a customer's TikTok + database into an Instagram one on the SECOND write to it. MEASURED: a `ut_tt_profile` spawned + by W30-T06 comes out correct — `handle` carrying `profile: {source: "tiktok"}` — and one more + `ut_ensure` on that table (a re-save, the discovery runner's own ensure, the next run) leaves + it carrying `profile: {source: "instagram"}` and the description *"Instagram username without + the @ symbol"*. `profile_field_key(..., source="tiktok")` then answers `""` forever, so every + TikTok enrich step on that database reports UNBOUND, and every row gets stamped + `platform: Instagram` — which is half of W26/R4's `(platform, handle)` identity, so two + different people's accounts merge under one key with nothing red anywhere. + ⚠ It hid because the migration SKIPS a table that does not exist yet, so the spawn itself is + always clean and only the second write is not. A gate that creates and asserts once cannot see + it; the assertion has to survive a second `ut_ensure`. + + ⭐ TWO LEGS, and the second is the one that still works after the first has been eaten: + 1. the table SAYS SO — a column declaring `profile: {source: "tiktok"}` is the database's own + statement of which network it is about, and it is the same declaration `profile_field_key` + reads, so there is one answer to "whose profile table is this"; + 2. it declares a column only TikTok has. Needed because leg 1 is exactly what the defect + destroys: on a table already corrupted in production, the flag now says Instagram, and a + detector resting on it alone would agree with the corruption and keep re-applying it. + """ + fields = (tbl or {}).get("fields") or [] + for f in fields: + p = f.get("profile") + if isinstance(p, dict) and str(p.get("source") or "") == PROFILE_SOURCE_TT: + return True + return bool({f.get("key") for f in fields} & TT_ONLY_PROFILE_KEYS) + + +#: ⭐⭐ WAVE 32 · T40 — THE MIRROR OF `TT_ONLY_PROFILE_KEYS`, and it is the subject of owner item 1: +#: the profile columns that can only ever be INSTAGRAM's. 26 today. Derived by the same subtraction +#: in the other direction, so a column moved between the two declarations changes both sets at once +#: rather than leaving one of them quietly asserting a stale fence. +#: ⚠ IT IS DERIVED FROM `CANDIDATE_FIELDS`, NOT `PRESET_PROFILE_FIELDS` — discovery's bookkeeping +#: columns (`found_count`, `first_found`, `last_found`, `created_by`) are declared on BOTH platforms, +#: so subtracting the preset list alone would report four keys as Instagram-only that TikTok's own +#: table has carried since wave 29. +IG_ONLY_PROFILE_KEYS = (frozenset(f["key"] for f in CANDIDATE_FIELDS) + - frozenset(f["key"] for f in TT_PROFILE_FIELDS)) + +#: ⭐⭐ WAVE 32 · T42 — the preset columns NOBODY TYPES: every `link` and `rollup` in the Instagram +#: preset set. Derived from the declaration, so adding a rollup to the contract widens this for +#: free. `_carries_ig_presets` uses it as the leg that survives when the `automation.preset` stamp +#: is absent — a hand-made creator list has `handle` and `followers`; it does not have `posts_link`. +PRESET_MACHINE_ONLY_KEYS = frozenset( + f["key"] for f in PRESET_PROFILE_FIELDS if f.get("type") in ("link", "rollup")) + + +def platform_schema(platform): + """⭐⭐ WAVE 32 · T41 — ONE ASKABLE DECLARATION OF WHAT A PLATFORM'S DATABASES ARE. + + Returns, for `PLATFORM_INSTAGRAM` or `PLATFORM_TIKTOK`: + + {"platform", "profile_table", "profile_label", "profile_fields", "preset_keys", + "only_keys", "locked_tables", "children": {key: {"label", "fields", "record_mode"}}} + + ⛔ **WHY IT IS A FUNCTION AND NOT A DICT LITERAL — the same import-order trap `discovery_facts` + records twenty lines from its own declaration.** `CANDIDATE_FIELDS` and `DISCOVER_TABLE` are + declared HUNDREDS of lines below `TT_TABLE_FIELDS`, so any module-level dict spanning both + platforms NameErrors at import. A function body resolves at call time and does not care. That is + not a style preference here; it is the reason two earlier attempts at a platform registry in this + file ended up as five inline copies instead. + + ⛔⛔ **AND IT IMPORTS NOTHING FROM `core`, WHICH IS THE ACTUAL CONTRACT WITH `main.py`.** This + module is deliberately dependency-light on the API's boot path (`MAX_UT_ROWS`, `MACHINE_OWNERS` + and `LOCKED_CHILD_TABLES` all say so in their own notes), and the REGISTRAR — the thing that + turns these names into `core.user_tables` state — lives in the composition root. So this returns + plain lists and dicts: a caller can `register_locked_records(...)`, `ut_ensure(...)` or diff it + against a live tenant without `core` ever appearing on the engine's import path. `verify_ + automation` asserts that, by importing this module in a CLEAN interpreter and checking + `core.user_tables` is absent from `sys.modules` afterwards — [[artifact-with-no-importer]] in + reverse: a declaration whose registrar cannot reach it is the same defect as a registrar with + nothing to register. + + ⚠ FIELD DICTS ARE COPIED ONE LEVEL. Every existing call site passes the module lists straight + into `ut_ensure`, which is safe only because nobody has yet appended to one; a boot sweep looping + over both platforms is exactly the caller that would. The copy costs a few hundred dict + constructions once per boot and removes the whole question. + + ⚠ `profile_fields` is the platform's PRESET set as spawned — Instagram's is `CANDIDATE_FIELDS` + (the preset set PLUS discovery's bookkeeping), not `PRESET_PROFILE_FIELDS`, because that is what + `discovery_facts` actually hands the spawn. Two answers to "Instagram's field set" is the drift + this function exists to end, so it gives the one the product uses. + """ + name = str(platform or "") + if name == PLATFORM_TIKTOK: + children = {k: {"label": TT_TABLE_LABELS[k], "fields": [dict(f) for f in TT_TABLE_FIELDS[k]], + "record_mode": tt_record_mode(k)} + for k in sorted(TT_TABLE_FIELDS) if k != TT_PROFILE_TABLE} + return { + "platform": PLATFORM_TIKTOK, + "profile_table": TT_PROFILE_TABLE, + "profile_label": TT_TABLE_LABELS[TT_PROFILE_TABLE], + "profile_fields": [dict(f) for f in TT_PROFILE_FIELDS], + "preset_keys": tuple(f["key"] for f in TT_PROFILE_FIELDS), + "only_keys": TT_ONLY_PROFILE_KEYS, + "locked_tables": TT_LOCKED_TABLES, + "children": children, + } + if name == PLATFORM_INSTAGRAM: + children = {k: {"label": IG_TABLE_LABELS[k], "fields": [dict(f) for f in fields], + # ⚠ NOT `tt_record_mode`'s twin by accident: `ensure_ig_graph` passes + # `AUTOMATION_RECORD_MODE` for all four unconditionally, and + # `IG_LOCKED_TABLES` is exactly those four. Derived from the SET so that + # unlocking one there unlocks it here, rather than from the constant. + "record_mode": (AUTOMATION_RECORD_MODE if k in IG_LOCKED_TABLES else "")} + for k, fields in sorted(IG_TABLE_FIELDS.items())} + return { + "platform": PLATFORM_INSTAGRAM, + "profile_table": DISCOVER_TABLE, + "profile_label": DISCOVER_LABEL, + "profile_fields": [dict(f) for f in CANDIDATE_FIELDS], + "preset_keys": tuple(f["key"] for f in CANDIDATE_FIELDS), + "only_keys": IG_ONLY_PROFILE_KEYS, + "locked_tables": IG_LOCKED_TABLES, + "children": children, + } + raise ValueError(f"no schema is declared for platform {platform!r}") + + +def definition_platforms(defn): + """Which networks ONE automation uses — `{"Instagram"}`, `{"TikTok"}`, both, or empty. + + ⭐ WAVE 32 · T42. Three ways a definition names a network, and all three count, because the + owner's rule is about what a database is USED FOR rather than about how the automation was + built: its discovery KIND (a corpus search is a network by construction), its enrich ACTIONS + (a `plain` flow that enriches is using that network), and the retired `field_instagram` kind, + which is uncreatable but alive on stored definitions (D-65) and still runs Instagram. + """ + defn = defn or {} + kind = str(defn.get("kind") or "") + out = set() + if kind in DISCOVERY_KINDS: + out.add(discovery_facts(kind)[0]) + if kind == "field_instagram": + out.add(PLATFORM_INSTAGRAM) + actions = (defn.get("flow") or {}).get("actions") or [] + if _actions_of_kind(actions, "enrich_instagram"): + out.add(PLATFORM_INSTAGRAM) + if _actions_of_kind(actions, "enrich_tiktok"): + out.add(PLATFORM_TIKTOK) + return out + + +def automation_platforms_by_table(rt, definitions=None): + """⭐⭐ WAVE 32 · T42 (owner item 1, third clause) — WHICH NETWORKS EACH DATABASE IS USED FOR. + + `{table_key: {"Instagram", "TikTok"}}`, over every stored automation. The owner's words are the + whole specification: *"if a user decide to use one database for both Tiktok and Instagram + scraping, only then can the pre-set Fields can exist in the same database."* — so the question a + schema pass has to be able to ask is *"which networks target THIS database"*, and until now + nothing in this module could answer it. + + ⛔ ASKABLE AND `core`-FREE, for `platform_schema`'s reason one function up: it reads only the + automations bucket through `all_definitions`, returns plain sets, and the T41 subprocess probe + covers it — a second askable declaration that quietly needed `core` would make the import-purity + guarantee mean "true for the things we remembered". + + ⚠ `definitions` is a LENT list, same contract as `ut_ensure`'s `tables=`: a caller already + holding the bucket must not pay for a second deep copy of it. + """ + # ⚠ `all_definitions` answers a DICT keyed by id, not a list — and iterating it directly hands + # every consumer a STRING that looks like a definition until the first `.get`. Normalised here + # so a caller lending its own list gets the same treatment. + defs = all_definitions(rt) if definitions is None else definitions + defs = list(defs.values()) if isinstance(defs, dict) else list(defs or []) + out = {} + for defn in defs: + if not isinstance(defn, dict): + continue + table = str(_flow_table(defn) or (defn.get("config") or {}).get("targetTable") or "") + if not table: + continue + found = definition_platforms(defn) + if found: + out.setdefault(table, set()).update(found) + return out + + +def platform_schemas(): + """Both declarations, in `PLATFORMS` order — what a boot sweep loops over. + + ⚠ `PLATFORM_FACEBOOK` is in `PLATFORMS` (it is a value the `platform` COLUMN may hold) and has + no schema, so this filters rather than raising: a caller sweeping every declared platform must + not be broken by a vocabulary entry that names no databases. `platform_schema` still raises for + it, which is the right answer to somebody ASKING for a schema that does not exist. + """ + out = [] + for name in PLATFORMS: + try: + out.append(platform_schema(name)) + except ValueError: + continue + return out + + +def _ig_profile_tables(rt, tables=None): + """Every `ut_*` table in this tenant that looks like an Instagram profile table. + + ``tables`` is the optional lent snapshot (W29-T01): read-only walk, so a caller that already + holds the bucket must not pay for a second deep copy of it. + + ⛔ WAVE 30 · T08 — AND THE TEST IS NOW EXCLUSIVE, because "looks like" was network-blind and a + TikTok profile table looks EXACTLY like one: 18 of `ut_tt_profile`'s 30 columns are Instagram + preset keys, and the bar here is `handle` plus three. Every caller of this function then treats + the match as an Instagram table — `migrate_ig_tables` rewrites its declarations to Instagram's + contract, and `discover_default_table` offers it as the default target for an Instagram search. + """ + out = [] + for key, tbl in sorted(((ut_all(rt) if tables is None else tables) or {}).items()): + keys = {f.get("key") for f in (tbl or {}).get("fields") or []} + if "handle" not in keys or len(keys & set(PRESET_PROFILE_KEYS)) < MIGRATE_MIN_PRESET_FIELDS: + continue + if _is_tt_profile_table(tbl): + continue + out.append(key) + return out + + +def _ig_contract_tables(rt, tables=None): + """⭐⭐ WAVE 32 · T42 — the tables Instagram's preset CONTRACT may be applied to. + + ⛔⛔ THIS IS A DIFFERENT QUESTION FROM `_ig_profile_tables` AND CONFLATING THEM COST FIVE RED + CHECKS. That function answers *"does this database LOOK like an Instagram profile table?"* and + two callers need exactly that: `discover_default_table`, which elects an existing table so a new + automation does not mint a second empty one — and it necessarily runs BEFORE any automation + targets that table, so a targeted-only test makes it elect nothing forever — and the detector's + own negative control. THIS function answers *"may we write Instagram's 48 columns into it?"*, + which is the owner's rule and a strictly narrower set. One normalizer answering two questions is + how a fix in one becomes a silent regression in the other ([[one-question-two-normalizers]]). + + A table qualifies when it looks like one AND any of: + - it is `ut_ig_profile`, Instagram's canonical database by declaration; + - it ALREADY carries the contract (`_carries_ig_presets`) — narrowing must never orphan a + table that has the columns, or its links and rollups silently stop being repaired; + - an INSTAGRAM automation targets it, which is the owner's rule stated in code. + + ⚠ The automations bucket is read LAZILY, on the first candidate that needs the question asked — + a tenant whose tables all already carry the contract pays nothing, on a path this wave is + otherwise trying to make faster. + """ + targets = None + out = [] + for key in _ig_profile_tables(rt, tables=tables): + tbl = (ut_all(rt) if tables is None else tables).get(key) or {} + if key != DISCOVER_TABLE and not _carries_ig_presets(tbl): + if targets is None: + targets = automation_platforms_by_table(rt) + if PLATFORM_INSTAGRAM not in targets.get(key, set()): + continue + out.append(key) + return out + + +def _preset_owned(field): + """Is this column MACHINE-authored, i.e. may `retract_foreign_presets` delete it? + + ⛔ MODULE-LEVEL, NOT A CLOSURE, AND THAT IS WHY IT MOVED. It was a nested `_machine` and the + negative control written against it came out BLIND: nothing outside the function could break + the one guard standing between a boot-time repair and a customer's own column, so the control + ended up patching the key SETS instead and proved something else. A guard a control cannot + reach is a guard nobody has tested [[gate-negative-control]]. + + Two legs, the second for tables that predate the stamp: the `automation.preset` mark that + `ut_ensure(lock_fields=True)` and `_locked_ig_field` both write, or a key that is a preset + LINK/ROLLUP — nobody types `posts_link` by hand. + """ + automation = (field or {}).get("automation") + if isinstance(automation, dict) and automation.get("preset") is True: + return True + return str((field or {}).get("key") or "") in PRESET_MACHINE_ONLY_KEYS + + +def _filled_count(rows, key): + """How many rows carry a non-blank value in `key` — the number that decides REPORT vs DELETE. + + ⛔ ALSO MODULE-LEVEL FOR ITS CONTROL'S SAKE. This one number is the whole of W30/R6's second + sentence inside `retract_foreign_presets`: a foreign column with cells in it is reported, never + deleted. A control that cannot make this lie cannot prove the report is doing anything. + """ + return sum(1 for r in (rows or {}).values() + if isinstance(r, dict) and str(r.get(key) or "").strip()) + + +def retract_foreign_presets(rt, log=print): + """⭐⭐ WAVE 32 · T46 / DEBT D-152 — REMOVE THE PRESET COLUMNS OF A NETWORK A DATABASE IS NOT + USED FOR. The RETRACTION half of `_ig_contract_tables`' recruitment rule, and A's boot sweep + (`W32-T07`) calls it once per tenant. + + ⛔ WHY IT HAS TO EXIST AT ALL, and this is the wave's own thesis one layer down. W30-T08 fixed + the DETECTOR, so the corruption cannot recur — and nothing undoes it. MEASURED on a fixture: a + `ut_tt_profile` corrupted before that fix keeps ALL 26 Instagram-only columns and its + `profile: {source: "instagram"}` flag through `ut_ensure`, `migrate_ig_tables` AND the discovery + spawn. Three write paths, zero repairs. *"The corruption cannot recur"* and *"the corruption is + gone"* are different claims [[a-migration-that-runs-on-the-next-write]]. + + Returns `{"tables", "columns", "cells", "flags", "kept"}`. **`kept` is the important one** — + see guarantee 4. + + THE FOUR GUARANTEES, in the order they matter: + 1. **Only machine-authored columns go.** A field must be stamped `automation.preset: True` + (or be a link/rollup this contract owns) to be eligible. A column a PERSON typed is never + touched, whatever it is named — the D-152 row calls this *"a VALUE BACKFILL over live + customer rows"*, and that is the line it must not cross. + 2. **Only the EXCLUSIVE sets.** `IG_ONLY_PROFILE_KEYS` (26) and `TT_ONLY_PROFILE_KEYS` (8), + both derived. A key both platforms declare — `platform`, `handle`, `followers`, the four + discovery bookkeeping columns — is shared vocabulary and is never a foreign column. + 3. **Idempotent.** A second call finds nothing and writes nothing, so it is safe on every + boot, which is what makes it callable from `main.py` rather than being a script somebody + has to remember to run (the whole reason D-201 is still open). + 4. ⛔ **A FOREIGN COLUMN THAT HOLDS DATA IS REPORTED, NOT DELETED.** W30/R6's second sentence: + a limit that cannot be removed is reported with its cause. Deleting a customer's populated + cells to make a grid look clean is not a repair, and this is precisely the *"deserves its + own supervision"* clause of D-152. `kept` names every such column with its row count, so + the caller can print it and a person can decide. + + ⚠ THE FLAG IS REPAIRED TOO, and it is the half D-152 names explicitly: a TikTok profile table + whose `handle` was re-declared `profile: {source: "instagram"}` answers `""` to + `profile_field_key(..., source="tiktok")` forever, so every TikTok enrich on it reports UNBOUND. + Repaired only where `_is_tt_profile_table` recognises the table by its OWN TikTok-only columns, + i.e. by the leg the corruption cannot have eaten. + """ + stats = {"tables": 0, "columns": 0, "cells": 0, "flags": 0, "kept": []} + targets = automation_platforms_by_table(rt) + exclusive = {PLATFORM_INSTAGRAM: IG_ONLY_PROFILE_KEYS, + PLATFORM_TIKTOK: TT_ONLY_PROFILE_KEYS} + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + for key, tbl in sorted(cur.items()): + if not isinstance(tbl, dict): + continue + fields = tbl.get("fields") or [] + keys = {str(f.get("key") or "") for f in fields} + # Only PROFILE databases are in scope: `handle` plus a real slice of either preset + # vocabulary. A posts or comments table shares no profile columns and is never a + # candidate; a database with nothing in common with either is not one either. + if "handle" not in keys: + continue + tt = _is_tt_profile_table(tbl) + used = set(targets.get(key) or ()) + if not used: + # Nothing targets it. Its own declaration is then the only statement of what it is + # for — and a table that says TikTok is not an Instagram table, whatever columns a + # past migration wrote into it. + used = {PLATFORM_TIKTOK} if tt else {PLATFORM_INSTAGRAM} + foreign = set() + for platform, own in exclusive.items(): + if platform not in used: + foreign |= set(own) + if not foreign: + continue + rows = tbl.get("rows") or {} + drop, touched = [], False + for f in fields: + fkey = str(f.get("key") or "") + if fkey not in foreign or not _preset_owned(f): + continue + filled = _filled_count(rows, fkey) + if filled: + # Guarantee 4 — REPORTED, never deleted. + stats["kept"].append({"table": key, "column": fkey, "rows": filled}) + continue + drop.append(fkey) + if drop: + tbl["fields"] = [f for f in fields if str(f.get("key") or "") not in drop] + for r in rows.values(): + if not isinstance(r, dict): + continue + for fkey in drop: + if fkey in r: + r.pop(fkey, None) + stats["cells"] += 1 + stats["columns"] += len(drop) + touched = True + # ⚠ THE FLAG, and only on a table TikTok's own columns still identify. + if tt: + for f in tbl.get("fields") or []: + p = f.get("profile") + if isinstance(p, dict) and str(p.get("source") or "") == PROFILE_SOURCE_IG: + f["profile"] = {**p, "source": PROFILE_SOURCE_TT} + stats["flags"] += 1 + touched = True + if touched: + stats["tables"] += 1 + return cur + + # ⚠ The precheck is the WHOLE mutation, run against a snapshot, so the common case (nothing to + # do) spends no commit — the same posture `_reconcile_ig_graph_fields` takes, and the reason + # this is safe to call on every boot. + probe = copy.deepcopy(ut_all(rt)) + _up(probe) + if not (stats["columns"] or stats["flags"]): + if stats["kept"]: + log(f"[aios-auto] retract: {len(stats['kept'])} foreign column(s) KEPT because they " + f"hold data. " + ", ".join(f"{k['table']}.{k['column']} ({k['rows']} rows)" + for k in stats["kept"])) + return stats + stats = {"tables": 0, "columns": 0, "cells": 0, "flags": 0, "kept": []} + rt.update(UT_STORE_KEY, _up, flush="sync") + log(f"[aios-auto] retract: {stats['columns']} foreign column(s), {stats['cells']} cell(s) and " + f"{stats['flags']} mis-declared flag(s) across {stats['tables']} database(s)") + if stats["kept"]: + log(f"[aios-auto] retract: {len(stats['kept'])} foreign column(s) KEPT because they hold " + f"data. " + ", ".join(f"{k['table']}.{k['column']} ({k['rows']} rows)" + for k in stats["kept"])) + return stats + + +def _carries_ig_presets(tbl): + """Has this table ALREADY been given Instagram's preset contract? + + ⭐⭐ WAVE 32 · T42 — THE RECRUIT/REPAIR LINE, AND IT RESTS ON A STAMP THE CODE ALREADY WRITES. + `ut_ensure(lock_fields=True)` and `_locked_ig_field` both set `automation: {preset: True}` on + every preset column, and `migrate_ig_tables` BACKFILLS it on a table that predates the stamp + (MEASURED: an unstamped 44-column fixture comes out 44/44 stamped after one pass). So "already + carries the contract" is a declaration to read, not a threshold to invent — and inventing one + was the alternative, because `MIGRATE_MIN_PRESET_FIELDS` is 3 and cannot tell a recruited table + from a hand-made creator list that happens to have `handle`, `followers` and `bio`. + + ⛔ WHY THE LINE EXISTS AT ALL (owner item 1's third clause, MEASURED): a hand-made 5-column + database that NO automation targets went to **45 columns** the moment an Instagram automation + aimed at a DIFFERENT database was SAVED — because `_ig_schema_contract` walks every + structurally-matching table in the tenant and hands each one the full 48-column contract. The + owner's rule is that the preset fields may live in a database only if it is used for that + scraping; recruitment by resemblance is the opposite of that rule. + + ⚠ NARROWING NEVER DROPS A COLUMN. `_reconcile_ig_graph_fields` deletes only the fixed retired-key + `drops` set, so a de-recruited table keeps every field and cell it has — it stops being ADDED to + and REPAIRED, which is the whole of the change. Asserted on a fixture rather than reasoned. + ⚠ AND THE COST IS PAID ONLY WHEN IT HAS TO BE: `_ig_profile_tables` reads the automations bucket + lazily, on the first candidate that is neither stamped nor the canonical `ut_ig_profile` — so a + tenant whose tables are all genuinely Instagram's pays nothing on a path this wave is otherwise + trying to make faster. + + ⛔⛔ TWO LEGS, AND THE SECOND IS THE ONE THAT KEEPS THIS FROM BREAKING A LIVE TENANT. The stamp + is written by the very passes this predicate now gates, so a table recruited BEFORE the stamp + existed and targeted by no surviving automation would be de-recruited and never stamped — + silently losing its rollup/link repair, which is the slowest and worst failure available here + (MEASURED on a stripped fixture: 44 columns, 0 stamps, 0 repaired). So a table ALSO counts as + already-carrying when it declares one of the preset set's LINK or ROLLUP columns. Those are + machine-authored by construction — nobody types `posts_link` or `avg_views_12` into a hand-made + creator list — so the second leg is derived from the contract itself, not a threshold somebody + picked. [[gate-and-nc-must-not-share-a-binding]]'s cousin: a discriminator written by the thing + it discriminates needs an independent second leg, exactly as `_is_tt_profile_table` needed one. + """ + fields = (tbl or {}).get("fields") or [] + for f in fields: + if str(f.get("key") or "") not in PRESET_PROFILE_KEYS: + continue + automation = f.get("automation") + if isinstance(automation, dict) and automation.get("preset") is True: + return True + return bool({str(f.get("key") or "") for f in fields} & PRESET_MACHINE_ONLY_KEYS) + + +def discover_default_table(rt, tables=None): + """Which database should a discovery automation write to when nobody has said? (2026-08-10) + + Owner: *"the damn database is supposed to be dynamic for whatever instagram profile is there. + We only need ONE IG profile so it's not confusing."* + + ⛔ THE COMPLAINT WAS ABOUT A SECOND EMPTY DATABASE, NOT A HARDCODED KEY. `ut_beauty_influencer_ + leads` appears NOWHERE in this product as a literal — it is a slug `ut_ensure` minted from the + label the tenant typed, and `_ig_profile_tables` finds it structurally (that function's own + note says so). What IS hardcoded is `DISCOVER_TABLE`, the FALLBACK target — and because it is + a constant rather than a question, a tenant that already had an Instagram profile database got + a SECOND, empty one the first time a discovery automation was saved without a target. + MEASURED on nurilab: `ut_ig_candidates` (0 rows, 44 preset columns) sitting beside + `ut_beauty_influencer_leads` (105 rows), both matching the profile predicate, one of them + pure confusion. + + So the default becomes a question asked of the tenant: + + exactly one profile database (excluding the fallback) -> that one + several, exactly one of which holds rows -> the one in use + several in use, or none at all -> "" (the caller keeps DISCOVER_TABLE) + + ⚠ RETURNS "" RATHER THAN GUESSING. Two populated profile databases is a real ambiguity and + picking one silently would write a paid discovery run into a database the user did not name — + worse than the empty table this exists to prevent. "" means "no opinion", and the caller's + existing fallback stands, which is exactly today's behaviour for that case. + ⚠ THE FALLBACK IS EXCLUDED FROM ITS OWN ELECTION. `ut_ig_candidates` carries the full preset + schema, so it matches `_ig_profile_tables` — without this line a tenant that already has the + empty table would keep re-electing it and nothing would ever change. + + ⭐ WAVE 29 (W29-T01) — ONE READ, NOT TWO. This function read the whole `user_tables` bucket + here and `_ig_profile_tables` read it AGAIN one line below: two 35.8 MB-ceiling deep copies to + answer one question, on a route the automation surface calls on every open. The election is + now computed from a single snapshot, which is also the only way it can be internally + consistent — the two reads could disagree with each other under a concurrent write. + ``tables`` lets `GET /automations` lend the copy it already holds; the answer is still derived + fresh on every call, so there is no memo to invalidate when a database is created or deleted. + """ + try: + tables = (ut_all(rt) if tables is None else tables) or {} + found = [k for k in _ig_profile_tables(rt, tables=tables) if k != DISCOVER_TABLE] + except Exception: # noqa: BLE001 + return "" + if len(found) == 1: + return found[0] + if len(found) > 1: + used = [k for k in found if (tables.get(k) or {}).get("rows")] + if len(used) == 1: + return used[0] + return "" + + +def _apply_discover_default(rt, raw): + """Fill in a discovery automation's target BEFORE the pure validator invents one. + + ⛔ IT HAS TO HAPPEN HERE, AND THE REASON IS THE WHOLE DESIGN. `clean_config` is the thing that + turns a missing target into `DISCOVER_TABLE` — and it takes no `rt`, by design (it is a pure + validator with a two-argument contract every gate and route depends on). Resolving only at the + RUN sites would be cosmetic: the literal is written into the STORED config at save time, so + `cfg.get("targetTable")` is truthy forever after and no runtime resolver is ever consulted. + `create`/`patch` are the one pair that both know the tenant and sit above that validator. + ⚠ ONLY WHEN THE CALLER SAID NOTHING. An explicit target — including one a previous save + stored — is the user's choice and is never rewritten. + """ + cfg = raw.get("config") + if not isinstance(cfg, dict) or str(cfg.get("targetTable") or "").strip(): + return raw + default = discover_default_table(rt) + if not default: + return raw + return {**raw, "config": {**cfg, "targetTable": default}} + + +def _vestigial_name_field(fields, rows): + """⭐ 2026-08-07 — the born-blank `Name` column, or None. The owner's *"REPLACED"* half. + + `user_tables.create()` mints EXACTLY ONE column on a hand-made database — + `{key:'name', label:'Name', type:'text', default:True}` — and `ut_ensure` then APPENDS the + automation's columns after it, never reordering. So on every Instagram database somebody + created before pointing an automation at it, column 1 is a `Name` nothing will ever write. + + ⛔ IT IS ONLY VESTIGIAL IF NOBODY EVER USED IT, and the test is deliberately conservative + because this is the one branch in the migration that DESTROYS something. A column carrying any + declaration (an automation binding, a metric, a profile flag) is somebody's work; a column with + a value in any row is somebody's data. Either way it survives as an ordinary column and merely + stops being the locked primary, which the `pinned` half achieves on its own. + + ⚠ `key == 'name'` IS ALREADY DECISIVE and the rest is belt: a user-created column is minted + `custom__` (`buildOverlayField`) and every machine one comes from a `field_def` + list, so nothing but `create()`'s default can hold this key. The label and type are checked + anyway — a renamed or retyped column is a column someone touched on purpose. + """ + f = next((f for f in fields if f.get("key") == "name"), None) + if f is None or f.get("label") != "Name" or f.get("type") != "text": + return None + if any(f.get(k) for k in ("automation", "metric", "profile", "measure", "formula")): + return None + if any(str(r.get("name") or "").strip() for r in rows.values()): + return None + return f + + +def _merge_candidates(rows): + """C3's merge rule, applied to rows now colliding on `(platform, handle)`. + + Earliest `first_found` wins - latest `last_found` wins - `found_count` SUMS - every other cell + takes the first non-blank, preferring the most recently enriched row. + + ⚠ THE SURVIVING ROW KEEPS THE LOWEST ID. Row ids are referenced by saved views, comments and + board cards; minting a new one would orphan all of them, so a merge picks a survivor rather + than creating one. + """ + groups, keepers = {}, {} + for rid in sorted(rows, key=lambda r: (len(str(r)), str(r))): + r = rows[rid] + h = str(r.get("handle") or "").strip() + if not h: + # ⛔⛔ THESE USED TO BE `continue`d, AND THE OUTPUT REPLACES THE TABLE'S ROWS — so a + # row with no handle was SILENTLY DELETED. Owner, 2026-08-07: *"How come when I add a + # manual handle in my 'Beauty influencer leads' database, after an automation run, it + # gets deleted?"* + # + # A row you add by hand is born blank and stays blank until you type into it, so the + # window is not narrow — it is every row, from creation until the cell is committed. + # And before the profile flag shipped (834decd) a typed handle landed in the TYPIST'S + # OVERLAY stratum, leaving the definition row's `handle` empty forever: the row was + # dropped even after it looked filled in on screen. + # + # ⚠ THE FUNCTION'S JOB IS TO MERGE DUPLICATE CANDIDATES, and a row with no handle is + # not a candidate — it is somebody's row. It cannot collide with anything (there is + # nothing to key it on), so it is carried through UNTOUCHED rather than judged. A + # merge pass that deletes what it cannot classify is not a merge, it is a filter. + keepers[rid] = r + continue + groups.setdefault((str(r.get("platform") or PLATFORM_INSTAGRAM).strip(), h), + []).append((rid, r)) + out, merged = dict(keepers), 0 + for _key, members in groups.items(): + if len(members) == 1: + out[members[0][0]] = members[0][1] + continue + merged += len(members) - 1 + # Most-recently-enriched first, so "first non-blank" prefers the freshest measurement. + ordered = sorted(members, key=lambda m: str(m[1].get("enriched_at") or ""), reverse=True) + keep_id = sorted(rid for rid, _r in members)[0] + acc = {} + for _rid, r in ordered: + for k, v in r.items(): + if k not in acc and str(v or "").strip(): + acc[k] = v + firsts = [str(r.get("first_found") or "").strip() for _i, r in members if r.get("first_found")] + lasts = [str(r.get("last_found") or "").strip() for _i, r in members if r.get("last_found")] + if firsts: + acc["first_found"] = min(firsts) + if lasts: + acc["last_found"] = max(lasts) + total = sum(_ig_int(r.get("found_count")) or 0 for _i, r in members) + if total: + acc["found_count"] = str(total) + out[keep_id] = acc + return out, merged + + +#: ⛔ DEBT D-74 + D-81 (wave 27 item 12) — `tracked` WAS DELETED FROM THE VOCABULARY AND NEVER +#: FROM THE TABLES. W26/R6 removed the column from `CANDIDATE_FIELDS` and from ~14 engine sites +#: *"we already have a Field called stage… we don't need another checkbox for this"* — and +#: MEASURED 2026-08-07 on `royal-imports/aios-nurilab-data`, `ut_beauty_influencer_leads` still +#: declared it, filled on 1 of 41 rows. A checkbox on the owner's flagship database that nothing +#: writes, nothing reads, and anybody can still click. +#: +#: ⚠ THE DECISION WAS WHICH, NOT WHETHER (D-74 stated both futures): sweep the ticks and lose the +#: record of who approved what before the wave, or leave them and let the next hand-made `tracked` +#: column silently inherit them. The wave takes the sweep — R6 deleted the concept, so a surviving +#: tick is not a decision anybody can act on, and an inert cell that reappears in the next EXPORT +#: as a column nobody declared is the worse half. +TRACKED_KEY = "tracked" + + +def _retired_tracked_field(fields): + """The retired `tracked` column when it is still the MACHINE's own, else None. + + ⛔ GUARDED THE WAY `_vestigial_name_field` IS, and for the same reason: this is a branch that + DESTROYS something. `tracked` was minted `{type: "checkbox"}` by the discovery preset, so a + column still declaring a checkbox is the one R6 retired. A column somebody has since RETYPED, + or bound a formula/link/rollup/measure to, is their work wearing an old key — it survives as + an ordinary column and merely stops being fed by anything, which was already true. + + ⚠ THE ASYMMETRY WITH THE TWO KEYS BESIDE IT IS DELIBERATE. `posts` and `post_hashtags` are + dropped unguarded: both are machine-derived cells with no history of anyone editing them. + `tracked` is the one a HUMAN ticked, so it is the one that gets the conservative test. + """ + f = next((f for f in fields if str(f.get("key") or "") == TRACKED_KEY), None) + if f is None or f.get("type") not in ("checkbox", "", None): + return None + if any(f.get(k) for k in ("formula", "link", "rollup", "measure", "metric")): + return None + return f + + +def _guarded_drops(table, drops): + """`drops` minus any key this table is allowed to keep — today, exactly `tracked` (D-81). + + ⛔ THE GUARD HAS TO LIVE WHERE THE DROP HAPPENS, and finding that out cost a red check. + `tracked` is deleted from TWO places: `migrate_ig_tables`' profile loop, which asks + `_retired_tracked_field` whether the column is still the machine's own, and the schema + reconciliation, which drops by KEY with no question asked. Guarding only the first was + decorative — the schema pass runs immediately afterwards on the same table and deleted the + retyped column the guard had just spared. Two paths to one deletion with one of them checked + is the shape where a control looks present and enforces nothing + ([[defects-that-mask-each-other]]). + """ + drops = set(drops or ()) + if TRACKED_KEY in drops: + stored = [f for f in (table or {}).get("fields") or [] + if str(f.get("key") or "") == TRACKED_KEY] + if stored and _retired_tracked_field(stored) is None: + drops.discard(TRACKED_KEY) + return drops + + +def _orphan_tracked_rows(fields, rows): + """Row ids carrying a `tracked` CELL that no field declares — D-74's actual subject. + + ⛔ THE COLUMN AND THE CELLS WERE RETIRED SEPARATELY, WHICH IS WHY THIS IS NOT COVERED BY THE + BRANCH ABOVE. R6 deleted the field DEFINITION from `CANDIDATE_FIELDS`; a table whose + declaration was already dropped keeps every `tracked` key in its row dicts, invisible on every + surface that renders from `fields` — and a migration that only looks at `fields` finds nothing + to do and `continue`s past the table. Storage, export and any future re-declaration of the key + all still see them. + """ + if any(str(f.get("key") or "") == TRACKED_KEY for f in fields): + return [] # the column above owns its own cells + return [rid for rid, r in rows.items() if TRACKED_KEY in (r or {})] + + +def backfill_corpus_snapshots(rt, log=print): + """The BACKFILL half: every profile row holding a measurement with no series behind it gets + its corpus observation. Returns how many were written. + + ⛔ ONE BUILDER, TWO CALLERS — `run_discover_instagram` for rows arriving now, this for rows + that arrived before the forward fix existed. A second row-shape here would be two ideas of + what a corpus observation is, and they would disagree on the next promoted column. + + ⚠ THE CONDITION IS "NO SERIES AT ALL", not "no series for this day", and the narrowness is + deliberate. A profile that HAS been enriched already owns real observations; synthesising a + corpus row dated `last_found` for it could out-rank the exact read in a `latest` rollup and + replace a measured number with a corpus one. The gap this closes is a row with NOTHING behind + it, which is the only case where a corpus observation can only improve matters. + + ⚠ IDEMPOTENT BY THE SNAPSHOT KEY (`@`), never by a flag — so this is safe on + every write, which is what `migrate_ig_tables`' placement requires. After one pass the row has + a series, so it stops matching and the scan costs a dict comparison. + """ + snaps = ut_get(rt, IG_SNAPSHOTS_TABLE) + if snaps is None: + return 0 # no series store — see `append_ig_snapshots` on why we never mint one + known = {str((r or {}).get("influencer_key") or "").strip().lower() + for r in (snaps.get("rows") or {}).values()} + known.discard("") + #: A cell counts as a MEASUREMENT if the snapshot schema has somewhere to put it — derived + #: from `SNAPSHOT_FIELDS` rather than named, for `_ig_profile_tables`' own reason (D-71: a + #: named list goes silently empty the day the schema moves). + measured = tuple(fd["key"] for fd in SNAPSHOT_FIELDS + if fd["key"] not in _SNAPSHOT_OWN_KEYS + and fd.get("type") in ("int", "pct")) + incoming = [] + for table_key in _ig_profile_tables(rt): + for row in ((ut_get(rt, table_key) or {}).get("rows") or {}).values(): + handle = str((row or {}).get("handle") or "").strip().lstrip("@").lower() + if not handle or handle in known: + continue + if not any(str((row or {}).get(k) or "").strip() for k in measured): + continue # nothing was ever read about this row — there is no observation to record + built = corpus_snapshot_row(row) + if built: + incoming.append(built) + known.add(handle) # two profile tables holding one handle write ONE observation + return append_ig_snapshots(rt, incoming, log=log) + + +def backfill_post_snapshot_grain(rt, log=print): + """Copy `posted_at`/`type` off the POST row onto every measurement missing them. + + ⛔ STRUCTURAL, NEVER A FLAG: the row matches when the snapshot's cell is blank AND the post's + is not, so a pass that has already run matches nothing and a post that genuinely has no date + is never re-visited on a promise it cannot keep. Same idempotency discipline as the migration + around it. + + ⚠ IT FILLS BLANKS AND NEVER OVERWRITES. A snapshot's `posted_at` is what the vendor said WHEN + THE MEASUREMENT WAS TAKEN; if a later pull disagrees with the post row, the measurement's own + record of the moment is the one to keep — a fact table that lets a dimension rewrite its + history is not a fact table. + """ + psnaps = ut_get(rt, IG_POST_SNAPSHOTS_TABLE) + posts = ut_get(rt, IG_POSTS_TABLE) + if psnaps is None or posts is None: + return 0 + have = {f.get("key") for f in (psnaps.get("fields") or [])} + carried = [k for k in ("posted_at", "type") if k in have] + if not carried: + return 0 # the columns are not declared yet — `ut_ensure` adds them first + by_shortcode = {} + for row in (posts.get("rows") or {}).values(): + code = str((row or {}).get("shortcode") or "").strip() + if code: + by_shortcode[code] = row or {} + changes = {} + for rid, row in (psnaps.get("rows") or {}).items(): + post = by_shortcode.get(str((row or {}).get("shortcode") or "").strip()) + if not post: + continue + for key in carried: + want = str(post.get(key) or "").strip() + if want and not str((row or {}).get(key) or "").strip(): + changes.setdefault(str(rid), {})[key] = _s(want, 200) + if not changes: + return 0 + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + table = cur.get(IG_POST_SNAPSHOTS_TABLE) + if table is not None: + for rid, values in changes.items(): + table.setdefault("rows", {}).setdefault(rid, {}).update(values) + return cur + + rt.update(UT_STORE_KEY, _up, flush="sync") + log(f"[ig-migrate] post-measurement grain: {len(changes)} row(s) gained a post date/kind") + return len(changes) + + +#: ⭐⭐ 2026-08-10 (owner: *"retire the old hardcoded method and replace the Instagram database work +#: with correct Rollup and Link fields"*) — THE PROFILE COLUMNS THAT MAY BECOME ROLLUPS. +#: +#: A column qualifies when the SERIES has somewhere to put it: `SNAPSHOT_FIELDS` carries the same +#: key, so `latest` over `profile_snapshots_link` can re-derive it. That is 27 of the 44 preset +#: columns — the other 17 are identity/bookkeeping (`handle`, `platform`, `enriched_at`), already +#: relational (the 12 links and rollups), or have no observation behind them at all +#: (`location_guess`/`location_confidence`, which WE infer rather than read). +#: +#: ⛔⛔ AND "DERIVABLE" IS NOT "SAFE", WHICH IS THE WHOLE REASON THIS IS A PREDICATE AND NOT A LIST. +#: MEASURED on nurilab before any conversion: a bare `latest` would BLANK 221 filled cells — +#: `avg_engagement` and `category` lose 62 each — because the discovery reads that produced those +#: values were never recorded as observations (the hole this same wave closed going forward, in its +#: historical form: 62 profiles hold exactly ONE observation, the enrichment scrape, dated three +#: days AFTER the discovery run whose numbers are on their row). A migration that converts on a +#: NAMED LIST would do that damage on any tenant whose series is thinner than the list's author +#: assumed. +#: +#: ⇒ So the migration converts a column only where it can PROVE, on this tenant's own rows, that +#: the derived value equals the stored one everywhere. See `_convertible_profile_columns`. +#: ⚠ Re-runnable by design: as the series fills in, later passes convert more. A column that is not +#: safe today is not refused forever, it is simply not converted yet. +IG_DERIVABLE_KEYS = tuple( + fd["key"] for fd in PRESET_PROFILE_FIELDS + if fd.get("type") not in ("link", "rollup") + and fd["key"] not in ("platform", "handle", "enriched_at", "first_found", "last_found", + "found_count", "created_by") + and fd["key"] in {s["key"] for s in SNAPSHOT_FIELDS}) + + +def _derived_profile_bag(key): + """The rollup that replaces the mapper's write of ONE profile column. + + ⚠ `where: is_not_empty` IS NOT DECORATION. A bare `latest` returns the newest observation's + value INCLUDING ITS BLANK, so a cheap pull that did not read the field would erase what an + expensive one learned — the exact asymmetry `upsert_rows`' merge exists to prevent on the + scalar side. `where` runs BEFORE the ranking (contract C1), so this reads "the newest + observation that ACTUALLY READ this field", which is what the materialised cell meant. + MEASURED: it rescues 5 of the 12 columns a bare `latest` would damage; the other 7 need the + observation itself, which is why the guard below is a proof and not a hope. + """ + return {"link": "profile_snapshots_link", "field": key, "fn": "latest", + "sortBy": "pulled_at", "sortDir": "desc", + "where": [{"field": key, "op": "is_not_empty"}], "whereConj": "and"} + + +def _derived_profile_columns(table, snapshots): + """Which of `IG_DERIVABLE_KEYS` should be DECLARED as rollups on THIS table. + + = the ones already converted, PLUS the ones whose derived value equals the stored value on + every row. Returns keys. + + ⛔⛔ IT MUST RE-ASSERT THE ONES ALREADY CONVERTED, and forgetting that is a one-line revert with + a several-hour fuse. `_reconcile_ig_graph_fields` overwrites every contract key it owns — + `rollup` included, by design — so a converted column that this function stopped naming would be + re-typed back to `int`/`text` on the next reconcile, its bag dropped, and its value re-written + by the mapper. The edit would look applied and undo itself later, which is exactly + [[preset-unlock-needs-a-custody-stamp]]'s shape. An already-converted column is therefore + included unconditionally rather than re-proved: its cells ARE the derived values, so a proof + would be comparing the fold against itself. + + ⛔ THE PROOF RUNS PER TENANT, PER COLUMN, AT MIGRATION TIME — never against a list somebody + measured on one workspace. `followers` derives exactly on nurilab and could be thin somewhere + else; `category` is damaged on nurilab and might be perfect elsewhere. The only honest + authority is the data in front of the migration. + ⚠ A column with NO stored values anywhere converts too, and that is deliberate rather than an + oversight: there is nothing to lose, and leaving it materialised would mean the schema differs + between two tenants for no reason a reader could discover. + """ + fields = {str(f.get("key")): f for f in (table or {}).get("fields") or []} + rows = (table or {}).get("rows") or {} + by_subject = {} + for snap in ((snapshots or {}).get("rows") or {}).values(): + key = str((snap or {}).get("influencer_key") or "").strip().lower() + if key: + by_subject.setdefault(key, []).append(snap or {}) + # The engine's own ordering, once per subject — newest first, unrankable rows last. + ordered = {} + for subject, observations in by_subject.items(): + keyed = [(o, _sort_key(o.get("pulled_at"), "date")) for o in observations] + rankable = [(o, k) for o, k in keyed if k is not None] + rankable.sort(key=lambda ok: ok[1], reverse=True) + ordered[subject] = [o for o, _k in rankable] + [o for o, k in keyed if k is None] + # ⛔ A rollup needs the LINK it folds. A profile table that has not been through the graph + # reconcile has no `profile_snapshots_link`, and declaring one there would mint a column whose + # link resolves to nothing — a blank cell wearing a configured column's clothes. + if "profile_snapshots_link" not in fields: + return [] + out = [] + for key in IG_DERIVABLE_KEYS: + field = fields.get(key) + if not field: + continue + if field.get("type") == "rollup": + out.append(key) # already converted — see the note above on why + continue + declared = str(field.get("type") or "text") + safe = True + for row in rows.values(): + stored = str((row or {}).get(key) or "").strip() + window = [o for o in ordered.get( + str((row or {}).get("handle") or "").strip().lower(), []) + if str(o.get(key) or "").strip() != ""] # the `where` guard, applied here + derived = str(_rollup_fold("latest", [o.get(key) for o in window], declared) + or "").strip() + if stored != derived: + safe = False + break + if safe: + out.append(key) + return out + + +def _as_derived_field(field): + """One preset scalar declaration → the same column declared as a rollup over the series.""" + key = str(field.get("key") or "") + return {**{k: v for k, v in field.items() if k not in ("agg",)}, + "type": "rollup", "rollup": _derived_profile_bag(key)} + + +def migrate_ig_tables(rt, log=print, only=""): + """Bring every Instagram Profile and canonical child table onto one current contract. + + Returns counted changes rather than a reassuring line. Profile cells are converted before + their type declarations change; then the graph reconciliation adds any missing preset + Links/Rollups, repairs child types, stamps locks, adds all reciprocal backlinks, and deletes + only retired machine fields. + + `only` narrows to a single table, which is how `ut_ensure` calls it: the migration then rides + the WRITE PATH, so a table is brought forward immediately before anything appends to it and no + caller has to remember to run anything. + """ + stats = {"tables": 0, "retyped": 0, "converted": 0, "stamped": 0, "merged": 0, + # ⭐ 2026-08-07 (owner ruling) — the primary-column half. Counted separately from + # `stamped` because they answer different questions: that one is "how many ROWS got a + # platform", these are "how many TABLES changed shape". + "pinned": 0, "flagged": 0, "droppedName": 0, + "droppedRecentPosts": 0, "droppedPostHashtags": 0, + "droppedTracked": 0, "droppedTrackedCells": 0, "schema": 0} + want = {f["key"]: f["type"] for f in CANDIDATE_FIELDS} + # ⛔ `only` NARROWS THE SET, IT DOES NOT BYPASS THE TEST — and skipping that cost six red + # checks the first time. `ut_ensure` calls this with the key it is ABOUT TO CREATE, so on a + # first run the table does not exist yet: with the detection bypassed it fell through to the + # writer and stamped a stub table with empty fields, clobbering the creation that was the whole + # point of the call (the owner stamp and the flow tag went with it). A table that does not + # exist, or that is not an Instagram profile table, has nothing to migrate. + # ⭐ WAVE 32 · T42 — the CONTRACT set, not the detector's. This loop retypes columns, stamps + # `platform` and drops retired keys, i.e. it edits the schema — so it answers to the owner's + # rule about which databases Instagram may write into, exactly like `_ig_schema_contract` at + # the bottom of this function. `_ig_profile_tables` stays the wider question and keeps its own + # callers (`discover_default_table`, `backfill_corpus_snapshots`, the detector's NC). + detected = set(_ig_contract_tables(rt)) + targets = ([only] if only in detected else []) if only else sorted(detected) + # ⭐⭐ WAVE 31 · T30 — `only` NOW NARROWS THE WHOLE FUNCTION, AND IT DID NOT. + # + # ⛔ THE BUG WAS THAT `only` NARROWED THE LOOP ABOVE AND NOTHING ELSE. The tenant-wide passes + # at the bottom — two full `_ig_schema_contract` + `_reconcile_ig_graph_fields` rounds and two + # backfills — ignore `only` entirely, so `migrate_ig_tables(only=X)` paid a whole-tenant + # Instagram reconciliation no matter what X was. And `ut_ensure` calls it that way on EVERY + # ensure whose field set intersects `PRESET_PROFILE_KEYS` — which TikTok's profile schema does, + # because the two networks share column names. + # MEASURED on the wave-30 fixture, `only="ut_tt_profile"`: `_ig_profile_tables` returns `[]`, + # `stats` comes back completely empty and ZERO store writes happen — at a cost of **10 + # whole-document reads**, each a `Store.get` deep copy (documented ceiling 35.8 MB / ~1.4 s). + # Across one save's four `ut_ensure` calls that was **40 of the 41** reads behind owner item 7's + # *"it just say Saving... and takes a long time"*. The migration was not slow; it was running at + # all. + # ⚠ THE COMMENT ONE SCREEN DOWN IS WHY THIS WENT UNSEEN FOR THREE WAVES: *"Both are guarded on + # 'is there anything to do' and cost a dict scan in the steady state, which is what makes them + # safe on a function that rides every `ut_ensure`."* That guard is on the WRITE. The read was + # never guarded, and a scan of a freshly deep-copied 35 MB document is not a dict scan + # [[read-a-gates-predicate-for-what-it-excludes]]. + # ⛔ SCOPED TO THE PROVEN-EMPTY CASE ONLY, deliberately. When `only` IS a detected profile table + # the function is unchanged, tenant-wide passes included — that is the Instagram path every + # migration check in `verify_automation` exercises. And a caller passing no `only` still gets + # the full sweep. The claim being made here is narrow and checkable: *a table that is not in the + # Instagram graph has no Instagram migration*, which is the same thing the `detected` test above + # already decided one line earlier and then threw away. + # ⛔ AND IT IS NOT A LEND. Lending one snapshot through the rest of this function would be the + # obvious next fix and it is WRONG: the second contract pass exists precisely because it must + # read what the first pass WROTE (see its own note below), so a read-write-read sequence cannot + # answer out of one snapshot without silently un-fixing that idempotency. + if only and not targets: + return stats + for table_key in targets: + tbl = ut_get(rt, table_key) or {} + fields = [dict(f) for f in tbl.get("fields") or []] + rows = {rid: dict(r) for rid, r in (tbl.get("rows") or {}).items()} + # ⭐ THE IDEMPOTENCY KEY. Only a column whose STORED type is still the old one is touched, + # so the cell conversions below can never run twice on the same value. + stale = [f for f in fields + if f.get("key") in want and f.get("type") != want[f["key"]] + and f.get("type") in ("text", "", None)] + stale_keys = {f["key"] for f in stale} + has_platform = any(f.get("key") == "platform" for f in fields) + needs_stamp = [rid for rid, r in rows.items() if not str(r.get("platform") or "").strip()] + # ⭐⭐ 2026-08-07 (owner ruling) — MAKE THE HANDLE THE PRIMARY COLUMN AND THE BINDING. + # `ut_ensure` merges by KEY and never updates a field that already exists, so declaring the + # two keys on `PRESET_PROFILE_FIELDS` reaches NEW tables only. Every table already in + # production needs them stamped here, and that is the whole reason this branch exists. + # + # ⚠ IDEMPOTENT BY READING THE STORED DECLARATION, never by a marker column — W26's rule, + # and it is free here: re-stamping a boolean and a two-key dict is a no-op by nature, which + # is exactly why these are safe to run on every write where a value conversion would not be. + pin_f = next((f for f in fields if f.get("key") == PRESET_PINNED_KEY), None) \ + if PRESET_PINNED_KEY else None + flag_f = next((f for f in fields if f.get("key") == PRESET_FLAG_KEY), None) \ + if PRESET_FLAG_KEY else None + # ⛔ AT MOST ONE PROFILE COLUMN PER TABLE is the law `user_tables` enforces at both write + # doors, and a migration is not exempt from it. If somebody has already flagged another + # column on this table, stamping the handle too would leave the table in a state its own + # validator refuses — and `_profile_of` returns the FIRST match, so the enrich action would + # silently bind to whichever came first in the field list. Leave it alone and pin only. + other_profile = next((f for f in fields if f is not flag_f + and isinstance(f.get("profile"), dict)), None) + want_pin = pin_f is not None and pin_f.get("pinned") is not True + want_flag = (flag_f is not None and other_profile is None + and not isinstance(flag_f.get("profile"), dict)) + vestigial = _vestigial_name_field(fields, rows) + retired = {f.get("key") for f in fields} & {"posts", "post_hashtags"} + if _retired_tracked_field(fields) is not None: + retired.add(TRACKED_KEY) + orphan_tracked = _orphan_tracked_rows(fields, rows) + if (not stale_keys and has_platform and not needs_stamp + and not want_pin and not want_flag and vestigial is None and not retired + and not orphan_tracked): + continue + stats["tables"] += 1 + if want_pin: + pin_f["pinned"] = True + stats["pinned"] += 1 + if want_flag: + flag_f["profile"] = {"source": PROFILE_SOURCE_IG} + stats["flagged"] += 1 + if vestigial is not None: + # ⚠ THE CELLS GO WITH THE COLUMN. A row dict keeping a `name` key whose field no longer + # exists is invisible everywhere except the next export, where it reappears as a column + # nobody declared. `_vestigial_name_field` has already proven every one of them blank. + fields = [f for f in fields if f is not vestigial] + for r in rows.values(): + r.pop("name", None) + stats["droppedName"] += 1 + if retired: + fields = [f for f in fields if f.get("key") not in retired] + for r in rows.values(): + for key in retired: + r.pop(key, None) + stats["droppedRecentPosts"] += int("posts" in retired) + stats["droppedPostHashtags"] += int("post_hashtags" in retired) + stats["droppedTracked"] += int(TRACKED_KEY in retired) + for rid in orphan_tracked: + # D-74: a tick whose column was already gone. Counted apart from the column drop + # because they answer different questions — "how many tables still declared it" and + # "how many rows were still carrying it after the declaration went". + rows[rid].pop(TRACKED_KEY, None) + stats["droppedTrackedCells"] += 1 + + for key in stale_keys: + conv = _MIGRATE_CONVERT.get(key) + if not conv: + continue # int/checkbox already store the right text shape + for r in rows.values(): + if str(r.get(key) or "").strip(): + new = conv(r[key]) + # "" from a converter means UNREADABLE, and the cell keeps its original text. + # A migration that empties a cell is indistinguishable from one that moved it. + if new: + r[key], stats["converted"] = new, stats["converted"] + 1 + for f in stale: + f["type"] = want[f["key"]] + stats["retyped"] += 1 + for rid in needs_stamp: + rows[rid]["platform"] = PLATFORM_INSTAGRAM + stats["stamped"] += 1 + if not has_platform: + fields = [dict(f) for f in CANDIDATE_FIELDS if f["key"] == "platform"] + fields + rows, merged = _merge_candidates(rows) + stats["merged"] += merged + + def _up(cur, _f=fields, _r=rows): + cur = cur or {} + t = dict(cur.get(table_key) or {}) + t["fields"], t["rows"] = _f, _r + cur[table_key] = t + return cur + + rt.update(UT_STORE_KEY, _up) + # ⭐⭐ WAVE 33 (D-187) — A SINGLE-TABLE CALL STOPS PAYING FOR A TENANT-WIDE RECONCILIATION. + # + # W31-T30 closed HALF of this: `only` naming a table outside the Instagram graph returns early + # (`if only and not targets` above). What it left is the case the row is actually about — `only` + # naming a table that IS in the graph, i.e. **every save an Instagram tenant makes**. The four + # passes below are tenant-wide by construction: two `_ig_schema_contract` + + # `_reconcile_ig_graph_fields` rounds and two backfills, each walking the whole document, none + # of them reading `only`. So one `ut_ensure` on one table paid a whole-tenant Instagram + # reconciliation, and `ut_ensure` runs several times per save. + # + # ⛔ THE ROW'S EXIT OFFERS TWO BRANCHES AND THIS TAKES THE FIRST ONE — *"the tenant-wide passes + # are reachable from a scheduled/boot caller"* — because they ARE, and were before this change: + # `routes_tables.py:882` (`_ig_forward`) calls `migrate_ig_tables(rt)` with NO `only`, once per + # tenant per process, guarded by `_IG_FORWARDED`. That is the right home for a whole-tenant + # repair: once, off the save path, rather than on every ensure. + # ⚠ THE SECOND BRANCH — "bound them by an anything-to-do test guarded on the READ" — is NOT + # available here and the comment above says why: the second contract pass exists to read what + # the first pass WROTE, so it cannot be answered out of one snapshot without silently + # un-fixing the idempotency everything else depends on. + # ⛔ NARROW AND CHECKABLE, like its W31-T30 sibling: a call that NAMES ONE TABLE does that + # table's migration. A call that names none still does the whole sweep, unchanged, and every + # tenant-wide check in `verify_automation` goes through that path. + if only: + return stats + # The profile loop owns value conversions and deduplication. The schema pass owns the + # declarations across ALL related databases and is safe only after those conversions, because + # the stored old type is the idempotency key for unit-changing values such as engagement. + wanted, drops = _ig_schema_contract(rt) + stats["schema"] = int(_reconcile_ig_graph_fields(rt, wanted, drops)) + # ⛔⛔ A SECOND CONTRACT PASS, AND IT IS NOT BELT-AND-BRACES — WITHOUT IT THIS FUNCTION IS NOT + # IDEMPOTENT, which is the one property everything else here is built on. + # + # The derived overlay (`_derived_profile_columns`) decides by COMPARING the profile cell to the + # fold of the series. The pass above REPAIRS THE SERIES — it retypes the child columns and runs + # `_IG_RETYPE_CONVERTERS`, which rewrites `avg_engagement` from the vendor's 0-1 fraction to our + # 0-100 percent. So on a tenant whose children are stale, the first proof compares a repaired + # profile cell against an UNREPAIRED observation, finds them different, and declines to convert + # a column that is in fact perfectly derivable. The next call then converts it — and + # `verify_automation`'s "a SECOND pass is a no-op" check went red, which is exactly what that + # check is for. MEASURED on the wave-26 fixture: 24 columns converted on pass 2, none on pass 1. + # + # ⚠ The module already knew this shape one level up — the profile loop's own note says the + # schema pass "is safe only after those conversions, because the stored old type is the + # idempotency key". Same argument, one table further down. + # ⚠ CHEAP WHEN THERE IS NOTHING TO DO: on a current tenant the contract recomputes to the same + # declarations and `_reconcile_ig_graph_fields` writes nothing. Twice, not looped: the repair is + # what moves the data, and it has already run. + wanted2, drops2 = _ig_schema_contract(rt) + stats["schema"] = int(bool(stats["schema"] + | int(_reconcile_ig_graph_fields(rt, wanted2, drops2)))) + # ⭐⭐ 2026-08-10 — THE TWO VALUE BACKFILLS, and they run AFTER the schema pass on purpose: + # `backfill_post_snapshot_grain` writes into columns that pass has just declared, and running + # it first would find nothing to fill and report a truthful zero about the wrong moment. + # Both are guarded on "is there anything to do" and cost a dict scan in the steady state, which + # is what makes them safe on a function that rides every `ut_ensure`. + stats["series"] = backfill_corpus_snapshots(rt, log=log) + stats["postGrain"] = backfill_post_snapshot_grain(rt, log=log) + if stats["tables"] or stats["schema"] or stats["series"] or stats["postGrain"]: + log(f"[ig-migrate] {stats}") + return stats + + +# ── WAVE 25 · C6 (owner ruling R5) — SEEDING A SEARCH FROM A VIEW OR COHORT. ────────────────── +# "Find me more accounts like the ones in this view." The server reads the seed rows, ranks their +# SHARED characteristics, and FILLS IN the discovery conditions — visible and editable, never +# hidden (R5). +# +# ⛔ THE RANKING IS RESTRICTED TWICE, AND BOTH RESTRICTIONS ARE MEASURED RATHER THAN CHOSEN: +# 1. R5 restricts it to `BD_POPULATED_FIELDS` — a characteristic extracted from a field the +# corpus barely populates produces a filter that returns nothing and looks exactly like +# "no such accounts exist" (the trap `BD_FILTER_LEAD` already exists to warn about). +# 2. A derived predicate must NARROW, or `narrowing_refusal` refuses to save it. The +# intersection of the two sets is MEASURED at exactly eight fields — `account`, `biography`, +# `external_url`, `fbid`, `full_name`, `id`, `profile_name`, `profile_url` — and each of them +# narrows under its own `default_operator`, asserted in the gate rather than assumed. +# +# ⭐ AND FOUR OF THOSE EIGHT CANNOT GENERALISE, which is the part worth stating: `account`, `id`, +# `fbid` and `profile_url` are IDENTITIES. A filter derived from them re-finds the seed rows and +# nothing else — a search that returns what you gave it, at full price. So the ranker reads only +# the three that describe a KIND of account: shared words in the bio, a shared domain in the link, +# and shared words in the name. +SEED_SOURCES = ("view", "cohort") +SEED_MAX_ROWS = 200 # bounded and DISCLOSED in `basis.rows` — never a silent [:N] +SEED_MAX_PREDICATES = 3 # a filter of ten derived guesses is not a filter +SEED_MIN_COVERAGE = 0.5 # a "shared" characteristic under half the rows share is not shared +SEED_MIN_ROWS = 2 # one row has nothing to share WITH +#: ⛔ AND A COVERAGE BAR ALONE IS NOT ENOUGH, which the gate caught rather than review: at two +#: seed rows, `coverage >= 0.5` is satisfied by a word appearing in exactly ONE of them. So every +#: bio word in a two-row seed qualified as "shared", and the surface would have offered a +#: characteristic derived from a single record as the thing those records have in common. A +#: shared characteristic needs at least two rows to be shared BY — n=1 dressed as a pattern is +#: the fabrication this module refuses everywhere else. +SEED_MIN_ROWS_SHARING = 2 +#: Which ROW column feeds which VENDOR field. The row keys are the C1 preset set's; the vendor +#: names are `BD_FILTER_FIELDS`'. Two vocabularies for one thing, so the mapping is written down. +SEED_FROM_ROW = (("bio", "biography"), ("external_url", "external_url"), + ("full_name", "full_name")) +#: Words that are shared by everything and therefore discriminate nothing. Deliberately SHORT — +#: an aggressive list would silently drop a real signal, and the coverage bar already removes most +#: noise. Anything ≤2 characters is dropped by length instead of by enumeration. +SEED_STOPWORDS = frozenset({ + "the", "and", "for", "with", "you", "your", "our", "are", "not", "all", "any", "from", + "this", "that", "have", "has", "was", "com", "www", "http", "https", "out", "new", "more", + "who", "how", "その", "инст"} | {"official", "welcome", "contact", "email", "dm", "link"}) + + +def _seed_tokens(text): + """One cell → the distinct words worth matching on. Lowercased, de-punctuated, ≥3 chars.""" + words = re.findall(r"[a-z0-9]{3,}", str(text or "").lower()) + return {w for w in words if w not in SEED_STOPWORDS} + + +def _seed_domain(url): + """A link-in-bio → its registrable-ish host, or ''. `linktr.ee/x` and `linktr.ee/y` share a + domain and that IS the shared characteristic; the path is the thing that differs.""" + raw = str(url or "").strip() + if not raw: + return "" + host = (urlparse(raw if "//" in raw else "https://" + raw).hostname or "").lower() + return host[4:] if host.startswith("www.") else host + + +def seed_predicates(rows): + """R5: seed rows → `(derived_predicates, basis)`. PURE — no store, no vendor, no network. + + `basis` is what makes the offer honest rather than magic: how many rows were read, which + characteristics were extracted, and **the MEASURED coverage of each**, so a weak signal reads + as weak instead of being presented with the same confidence as a strong one. + + ⛔ EVERY DERIVED PREDICATE IS ASSERTED AGAINST `narrowing_refusal` BEFORE IT IS OFFERED + (HARD RULE 11). A suggestion the product's own save door then refuses is the post-W24 + default-versus-guard defect rebuilt — and it would be worse here, because the user did not + type it: they would be told their own product's suggestion is invalid. + + ⚠ AN EMPTY ANSWER IS A LEGITIMATE ANSWER. When the seed rows share nothing above the coverage + bar, `derived` is `[]` and `basis` says why. An honest nothing beats a filter that looks + plausible, returns zero rows, and reads as "no such accounts exist". + """ + rows = [r for r in (rows or []) if isinstance(r, dict)] + total = len(rows) + basis = {"rows": total, "fields": [], + # R5's second lane, reported as the MEASUREMENT it is rather than sold as a feature. + # `related_accounts` is 22% populated corpus-wide (D-25) and is not part of the C1 + # preset set, so on our own seed rows it is usually simply absent — which the surface + # must SAY, or a lane that adds nothing reads as a lane that failed. + "related": {"tried": total, + "found": sum(1 for r in rows + if str(r.get("related_accounts") or "").strip())}, + "note": ""} + if total < SEED_MIN_ROWS: + basis["note"] = ("a seed needs at least two records. One record has nothing to share " + "with anything") + return [], basis + ranked = [] + for row_key, vendor_field in SEED_FROM_ROW: + counts = {} + for r in rows: + vals = ({_seed_domain(r.get(row_key))} if vendor_field == "external_url" + else _seed_tokens(r.get(row_key))) + for v in vals: + if v: + counts[v] = counts.get(v, 0) + 1 + for value, n in counts.items(): + cov = n / total + if cov >= SEED_MIN_COVERAGE and n >= SEED_MIN_ROWS_SHARING: + ranked.append({"name": vendor_field, "value": value, "coverage": round(cov, 3), + "rows": n}) + # Strongest first; ties broken by the LONGER value, which is the more specific one. + ranked.sort(key=lambda f: (-f["coverage"], -len(f["value"]), f["name"])) + derived, seen_fields = [], set() + for f in ranked: + if len(derived) >= SEED_MAX_PREDICATES: + break + # One predicate per FIELD: two `biography includes` rows AND-ed together narrow to the + # accounts carrying both words, which is a much smaller search than the seed implies. + if f["name"] in seen_fields: + continue + p = {"name": f["name"], "operator": default_operator(f["name"]), "value": f["value"]} + if not predicate_narrows(p): + continue # cannot be offered; it would be refused at the save door + seen_fields.add(f["name"]) + derived.append(p) + basis["fields"].append({"name": f["name"], "label": field_label(f["name"]), + "value": f["value"], "coverage": f["coverage"]}) + if derived and narrowing_refusal(derived, "and"): + # Belt AND braces: the per-predicate check above should make this unreachable, and it is + # asserted in the gate. If the LAW ever changes shape, this fails closed with an honest + # empty rather than offering something the save door will reject. + basis["note"] = ("the shared characteristics found are not specific enough to search on " + "add a condition of your own") + return [], basis + if not derived: + basis["note"] = basis["note"] or ( + "these records share no bio words, link domain or name in common. Nothing could be " + "derived, so the conditions are yours to write") + return derived, basis + + +def seed_rows(rt, table_key, view_id=""): + """The rows a seed source selects. `(rows, problem)` — bounded, and the bound is DISCLOSED by + `basis.rows` rather than silently applied ([[no-unverifiable-aggregates]]).""" + t = ut_get(rt, str(table_key or "")) + if t is None: + return [], f"{table_key!r} is not a database in this workspace" + rows = list((t.get("rows") or {}).values()) + if str(view_id or "").strip(): + # ⛔ THE SAME RESOLVER `enters_view` USES, not a second one. A seed that selected a + # different row set from the view it names would be describing a view nobody has. + tree, _fields, problem = view_filter(rt, table_key, view_id) + if problem: + return [], problem + rows = [r for r in rows if lane_match(tree, r)] + return rows[:SEED_MAX_ROWS], "" + + +def automation_tables(defn): + """Every `ut_*` table THIS automation writes into — its config target plus every + `create_record` action's table, deduped, order preserved. + + ⚠ THE SAME TWO AUTHORITIES `observed_category_tables` READS, narrowed to one definition. + Split out rather than parameterised because the two questions are different: that one asks + "where could a Category value be, anywhere in this tenant" and is deliberately generous; this + one asks "which rows has THIS automation already found" and must not reach into a table it + does not write, or a nightly search would start excluding another automation's finds. + """ + keys, seen = [], set() + cfg = (defn or {}).get("config") or {} + for raw in [cfg.get("targetTable"), + *(((a or {}).get("config") or {}).get("table") + for a in walk_actions(((defn or {}).get("flow") or {}).get("actions")))]: + k = str(raw or "").strip() + if k.startswith(UT_PREFIX) and k not in seen: + seen.add(k) + keys.append(k) + return keys + + +def candidate_key(platform, handle): + """The discovery upsert identity: `(platform, handle)`, as one string. + + ⭐ WAVE 29 — LIFTED OUT OF `run_discover_instagram`, where it was a closure, because a second + runner now needs the same identity. Two closures computing "the same" key is how one of them + grows a different default for a blank `platform` and the two networks quietly start sharing + rows — which is the exact data-loss shape wave 26's R4 added `platform` to prevent. + + ⚠ A BLANK PLATFORM DEFAULTS TO INSTAGRAM, and that is a FACT about the stored data rather than + an assumption: every row written before wave 26 came from the Instagram runner. Blank-keyed + rows would fail to match their own re-find and duplicate the whole table on the next run. + """ + return f"{str(platform or PLATFORM_INSTAGRAM).strip()}\n{str(handle or '').strip()}" + + +def already_found_handles(rt, defn, cap=None): + """The handles this automation has already written, for the vendor-side exclusion (item 7). + + ⭐⭐ THE POINT IS THE BILL, not the row count. Discovery upserts by handle, so re-finding a + profile has always been harmless — and never free: the vendor bills for every record it + returns, so a nightly search over stable keywords pays again, in full, for the same accounts + every night and reports them as `seen_again`. MEASURED shape of the waste: nurilab's beauty + scout re-found its whole result set on every run. This is the list that stops it, sent as one + `not_in` the vendor evaluates BEFORE billing. + + ⚠ SCOPED TO THIS AUTOMATION'S OWN TABLES. Excluding handles another automation found would + hide profiles this one has never seen — cheaper, and wrong: two scouts with different keywords + are two questions, and one must not answer with "somebody already looked at that". + """ + cap = BD_EXCLUDE_MAX if cap is None else int(cap) + out, seen = [], set() + for table_key in automation_tables(defn): + for row in ((ut_get(rt, table_key) or {}).get("rows") or {}).values(): + h = str((row or {}).get("handle") or "").strip().lstrip("@").lower() + if h and h not in seen: + seen.add(h) + out.append(h) + if len(out) >= cap: + return out + return out + + +def observed_category_tables(rt): + """Every table a Category value could have been written into, DERIVED from the automations + this tenant actually has — the default discovery table and the snapshot table are the FLOOR, + not the list. + + ⭐ Two authorities, because wave 24 and wave 25 each moved where the target is declared: + `config.targetTable` (the discovery runner's own write) and any `create_record` action's + `config.table` (R2 — authoritative since wave 25, and the field the Builder's Database picker + writes). Both are read, deduped, order preserved so the defaults come first. + """ + keys, seen = [], set() + + def _add(k): + k = str(k or "").strip() + if k and k.startswith("ut_") and k not in seen: + seen.add(k) + keys.append(k) + + _add(DISCOVER_TABLE) + _add("ut_ig_snapshots") + try: + for defn in (all_definitions(rt) or {}).values(): + cfg = (defn or {}).get("config") or {} + _add(cfg.get("targetTable")) + for act in walk_actions(((defn or {}).get("flow") or {}).get("actions")): + _add(((act or {}).get("config") or {}).get("table")) + except Exception: # noqa: BLE001 + # A definition set that cannot be read costs the DERIVED lanes and keeps the defaults — + # the same posture as the master lane below: degrade to less vocabulary, never to an error. + pass + return keys + + +def observed_categories(rt, limit=60): + """⭐ DEBT D-59 — the Category values WE HAVE ACTUALLY SEEN, for a combobox that still accepts + free text. + + D-59's exit condition, verbatim: *"EITHER derive the options from values we have actually seen + (the `category` column on `ut_ig_candidates` + the master store, offered as a combobox that + still accepts free text), OR buy a sample large enough to enumerate the real vocabulary. **Not**: + transcribe a published taxonomy and hope it lines up."* This is the first branch. + + ⛔ WHY A PUBLISHED TAXONOMY WOULD HAVE BEEN WORSE THAN NO DROPDOWN. A filter on a value the + corpus does not use returns zero rows and looks exactly like an honest "no such accounts + exist" — so a picker built from Instagram's own category list would mislead precisely when it + looked most authoritative. Every option here has been observed on a real row, and each carries + its COUNT so a value seen once reads differently from one seen forty times. + + ⚠ IT STAYS A COMBOBOX. These are the values we have seen, not the values that exist — a + control that refused anything else would be a second, quieter version of the same lie. + """ + seen = {} + + def _eat(rows, key): + for r in rows or []: + v = " ".join(str((r or {}).get(key) or "").split())[:60] + if v: + seen[v] = seen.get(v, 0) + 1 + + # ⛔ THE TABLES ARE DERIVED, NOT NAMED — and this is a REGRESSION BY OMISSION that shipped + # green. The two constants below were the whole list, which was correct until **wave 25 R2 + # made the Create record action's `config.table` AUTHORITATIVE**: from that ruling on, a + # discovery automation writes wherever the user pointed it, and `ut_ig_candidates` is merely + # the DEFAULT nobody keeps. MEASURED on nurilab 2026-08-06 — `ut_ig_candidates` held 0 rows + # while the tenant's two real target tables held 20 each, so this function honestly reported + # `{options: []}` and the combobox it feeds had nothing to offer. A vocabulary harvester that + # names its sources goes quietly empty the moment the product lets a user choose one. + for key in observed_category_tables(rt): + _eat(((ut_get(rt, key) or {}).get("rows") or {}).values(), "category") + try: + # ...and the PLATFORM master, which is the whole point of pooling it: a tenant with three + # candidates still gets a vocabulary drawn from every profile the platform has captured. + import ig_master + if ig_master.configured(): + # ⚠ `_handle().get(SNAP_BUCKET)` — a dict of rows keyed by id. NOT `_upsert()`, which + # returns the upsert FUNCTION; calling that and reading `.get(...)` off it answers + # None, so the master lane would have degraded to nothing INSIDE the except below and + # this would have shipped as a silent no-op with the gate green. Verified against + # `ig_master.series_for`, which reads the same bucket the same way. + _eat((ig_master._handle().get(ig_master.SNAP_BUCKET) or {}).values(), "category") + except Exception: # noqa: BLE001 + # A master that cannot be read costs the tenant its own values and nothing else. + pass + return [{"value": v, "count": n} + for v, n in sorted(seen.items(), key=lambda kv: (-kv[1], kv[0]))[:limit]] + + +def preset_plan(rt, table_key): + """C1 / R2b: the preset set diffed against ONE database's CURRENT fields. + + `{table, fields: [{key, label, type, present}], willUse: [...], willCreate: [...]}` + + ⛔ COMPOSED HERE, NOT IN THE CLIENT. The owner's ask is a config panel that says which columns + an automation will USE and which it will CREATE — and the only thing that can answer it is + whatever holds both lists. A client that diffed the preset set against a table's fields would + be a second implementation of `ut_ensure`'s merge rule, free to disagree with the merge that + actually runs; it would be wrong precisely when the two lists differ, which is the only case + anybody is asking about. + + ⚠ A TABLE THAT DOES NOT EXIST IS NOT AN ERROR — it is the ordinary state at R10's spawn-on-save + moment, and the honest answer is "all of them will be created". `present` is a fact about the + table, so it reads False for every field rather than the call refusing. + + ⛔ 2026-08-07 — THE PROJECTION IS FOUR KEYS AND DELIBERATELY DROPS EVERY DECLARATION BAG + (`link`, `rollup`, `metric`, `profile`, `pinned`). Measured off the live Space the day the + relational pair shipped: `posts_link` and the four rollups arrive here with their bags EMPTY. + That is CORRECT for this payload — it answers "which columns will be used vs created", where + a name and a type are the whole question — and it is safe today because nothing RENDERS a + column from this list: the grid reads full field dicts from `/tables` (`scoped_pool` passes + `dict(f)` straight through), and the column itself is spawned server-side by `ut_ensure` from + `PRESET_PROFILE_FIELDS`, which carries the bag. + ⚠ **It stops being safe the moment somebody previews a rollup from this payload** — they would + render "Avg views - last 12 posts" configured by nothing. Read the field off `/tables`, or + widen this projection deliberately; do not assume a bag is here because the field has one. + """ + have = {str(f.get("key")) for f in ((ut_get(rt, str(table_key or "")) or {}).get("fields") + or [])} + fields = [{"key": f["key"], "label": f["label"], "type": f.get("type") or "text", + "present": f["key"] in have} + for f in PRESET_PROFILE_FIELDS] + return {"table": str(table_key or ""), + "exists": ut_get(rt, str(table_key or "")) is not None, + "fields": fields, + "willUse": [f for f in fields if f["present"]], + "willCreate": [f for f in fields if not f["present"]]} + + +def clean_predicates(raw, operator="and", kind=""): + """Validate a discovery filter into the vendor's shape. Returns `(predicates, error)`. + + Refuses rather than coerces: a predicate naming a field the API rejects comes back as a + 400 with the field named, because the alternative — dropping it — turns "find me verified + accounts in Georgia" into "find me any account" and bills for the difference. + + ⭐ WAVE 32 · T46 (D-167) — `kind` NARROWS THE VOCABULARY TO THE PLATFORM'S. Default `""` keeps + Instagram's 21 names, so every existing caller and every stored Instagram automation is + unchanged; a `discover_tiktok` is validated against the 5 names its corpus actually has. The + refusal it produces is the same sentence, listing the platform's own fields — which is the + difference between a search that says why it cannot be built and one that is built, paid for, + and comes back empty. + """ + fields, _lead = filter_fields(kind) + out = [] + for p in (raw or [])[:12]: + if not isinstance(p, dict): + continue + name = _s(p.get("name") or p.get("field"), 60).strip() + op = _s(p.get("operator") or p.get("op"), 20).strip() + if name not in fields: + return None, ("that field cannot be searched. The searchable ones are: " + + ", ".join(field_label(f) for f in fields)) + # ⭐ PER-FIELD, not the global list. `is_business_account >= 3` used to validate cleanly + # because every operator was legal on every field; a yes/no column offering "at least" + # is a question with no meaning that the vendor is nevertheless asked. + allowed = ops_for(name) + if op not in allowed: + return None, (f"{field_label(name)} cannot be asked {BD_OP_LABELS.get(op, op)!r}. " + "it takes: " + + ", ".join(BD_OP_LABELS.get(o, o) for o in allowed)) + entry = {"name": name, "operator": op} + if op not in BD_NULLARY_OPS: + raw_v = p.get("value") + # ONE value or SEVERAL — several is the "contains any of" shape (owner item 3), and + # it is stored as a list rather than as N sibling predicates so the row a person sees + # and the row that is stored are the same thing. + vals = raw_v if isinstance(raw_v, list) else [raw_v] + vals = [v.strip() if isinstance(v, str) else v for v in vals] + vals = [v for v in vals if not (v is None or (isinstance(v, str) and not v))] + if not vals: + return None, f"give {field_label(name)} something to compare against" + if len(vals) > 1 and op not in BD_MULTI_VALUE_OPS: + return None, (f"{BD_OP_LABELS.get(op, op)!r} takes one value. " + f"{field_label(name)} can only be given a list with " + + ", ".join(BD_OP_LABELS[o] for o in ops_for(name) + if o in BD_MULTI_VALUE_OPS)) + if len(vals) > MAX_PREDICATE_VALUES: + return None, (f"{field_label(name)} takes at most {MAX_PREDICATE_VALUES} values " + "in one condition") + if field_kind(name) == "boolean": + ok = {o["value"] for o in BD_BOOLEAN_OPTIONS} + bad = [v for v in vals if str(v).lower() not in ok] + if bad: + return None, f"{field_label(name)} is answered Yes or No" + # ⛔ A REAL BOOLEAN, NOT THE STRING. MEASURED 2026-08-06: the filter API answers + # **400** to `{"name": "is_verified", "operator": "=", "value": "true"}` and + # accepts `True` (`snap_msh2rp4kh3v833q0u`). Nobody had ever sent one — the field + # was a free-text box until this wave, so the Yes/No dropdown that makes it easy + # to ask is also the thing that would have made every boolean condition fail. + # The dropdown's option VALUES stay "true"/"false" (a carries strings); - # the conversion belongs here, at the edge that talks to the vendor. - vals = [str(v).lower() == "true" for v in vals] - if field_kind(name) == "number": - try: - vals = [float(v) if "." in str(v) else int(v) for v in vals] - except (TypeError, ValueError): - return None, f"{field_label(name)} takes a number" - # A single value stays a SCALAR on the wire. The vendor has only ever been sent - # scalars for these operators; the list form is expanded at send time and the - # one-value case must not quietly start exercising an untested shape. - entry["value"] = vals[0] if len(vals) == 1 else vals - out.append(entry) - if not out: - # ⭐ WAVE 24 · AMENDMENT A2 — AN EMPTY FILTER IS INCOMPLETE, NOT WRONG, so it STORES. - # This used to refuse, which was right while a wizard collected the filters before the - # automation existed. C-TRIG law 1 inverts that: picking the `ig_profile_match` trigger - # CREATES the automation, and the filters are typed afterwards — so a save-time refusal - # here meant the trigger could never be picked at all. It is the A3 - # stored-inert-with-`configured:false` pattern this module already runs on. - # - # ⛔ NOTHING IS LOST AT THE MONEY DOOR, which is why this is safe rather than convenient: - # `run_discover_instagram` ALREADY calls `narrowing_refusal(preds)` before starting a - # search, deliberately, because "a stored config can predate the law, and the vendor - # bills for breadth whether the filter was saved yesterday or last month". The RUN is the - # wall. Every MALFORMED predicate above is still refused here, and the narrowing law - # below still refuses a non-empty filter that narrows nothing — only EMPTY changed. - return [], None - # C4 (wave 22): breadth is refused at WRITE time too — see `narrowing_refusal` for the - # measured hang this guards against. Same sentence at create/patch and at run. - guard_err = narrowing_refusal(out, operator, kind) or depth_refusal(out, operator) - if guard_err: - return None, guard_err - return out, None - - -def discover_estimate(records_limit): - """The cost preview a Find node shows BEFORE it runs. **SPEC, never measured** — see the - section header. Returned as structured data so the UI cannot accidentally drop the caveat.""" - n = max(0, int(records_limit or 0)) - return {"records": n, "usd": round(n * BD_RECORD_PRICE_SPEC, 4), - "unitUsd": BD_RECORD_PRICE_SPEC, "basis": "SPEC", - "note": "estimated from the published rate. An exact price is only known after a run" - "a price before a run, and this account's token cannot read a balance"} - - -def _candidate_row(row, stamp): - """One corpus row → a `ut_ig_candidates` row. None when it has no handle. - - ⛔ NEVER EMITS `found_count` — it is arithmetic the runner does against what is already - stored, so writing it here would reset the counter to 1 on every re-find. (It never emitted - `tracked` either; R6 deleted that column in wave 26.) - """ - if not isinstance(row, dict): - return None - handle = str(_first(row, "account", "username", default="") or "").strip() - if not handle: - return None - return { - # ⭐ R5 — half the identity. See `PLATFORM_INSTAGRAM`: this runner only ever reads - # Instagram, so it is a constant here rather than something derived from the row; the - # TikTok runner will stamp its own and the (platform, handle) key keeps the two apart. - "platform": PLATFORM_INSTAGRAM, - "handle": handle, - "profile_url": str(_first(row, "profile_url", "url", - default=f"https://www.instagram.com/{handle}/")), - "full_name": str(_first(row, "full_name", "profile_name", default="") or ""), - "followers": _s(_ig_int(_first(row, "followers"))), - "following": _s(_ig_int(_first(row, "following"))), - # ⭐ POPULATED ON CORPUS ROWS and null on scrape rows — the two paths differ, and this is - # the path where it carries values (measured on all five of the 2026-08-04 result set). - # ⚠ ×100 since wave 26 (amendment C1-a): the vendor's fraction is not our `pct`. - "avg_engagement": _pct100(_first(row, "avg_engagement", default="")), - "bio": _s(_first(row, "biography", "bio", default=""), 500), - "external_url": _s(_bd_first_url(_first(row, "external_url", "external_urls")), 300), - "verified": "1" if _first(row, "is_verified", default=False) else "", - "category": _s(_first(row, "category_name", "business_category_name", default="")), - # R3: a DAY, matching the `date` type the column now declares. The full-precision stamp - # still exists on the snapshot series, which is where a time axis belongs. - "last_found": _day(stamp), - } - - -# ── ⭐⭐ 2026-08-10 — DISCOVERY'S MISSING OBSERVATION ────────────────────────────────────────── -# -# ⛔ THE DEFECT, MEASURED ON NURILAB BEFORE ANY OF THIS WAS WRITTEN: 105 profiles, 104 carrying a -# `followers` number, and only 97 with a single row of history behind it. Seven accounts had a -# measurement nothing could re-derive, date or audit — `19,448 followers`, as of never, read via -# nothing. `_candidate_row` above writes six MEASUREMENTS onto a profile row (followers, -# following, engagement, verified, category, bio) and the discovery runner wrote no snapshot at -# all, so discovery was the one rung in this module that read numbers and recorded no observation. -# -# ⛔ IT IS NOT A "MISSING ENRICHMENT". Those seven rows are not waiting to be enriched — they hold -# real corpus numbers that are already on screen and already filterable. The gap is that the -# ENTITY row is the only copy, which is the exact arrangement R3's "one store for one series" law -# exists to forbid one level up. -# -# ⚠ A CORPUS ROW IS AN HONEST OBSERVATION, NOT A FAKE MEASUREMENT, and the two columns that make -# it honest already existed: `source` says **Discovery** (this was read off the vendor's -# pre-collected corpus, not measured for you) and `approx` is checked. Together they say "do not -# read this as an exact count taken at `pulled_at`" — which is the whole difference between -# recording what we know and inventing what we do not. -# -# ⭐ AND THE DATE GRAIN MAKES THE TIE-BREAK COME OUT RIGHT, which is worth stating because it -# looks like an accident: `pulled_at` here is a DAY (`last_found`), so `_sort_key` reads it as -# MIDNIGHT, while an enrichment on that same day carries a real timestamp. A `latest` rollup over -# the series therefore prefers the exact read over the corpus read whenever both happened on one -# day — the ordering you would have to hand-write, falling out of the grain. - -#: The `via` a DISCOVERY read reports. `PUBLIC_SOURCE` maps it to the word in the cell. -IG_VIA_DISCOVERY = "brightdata:discovery" - -#: The five keys a snapshot row owns about ITSELF. Everything else it carries is a measurement -#: copied off the profile row, and the list of those is DERIVED from `SNAPSHOT_FIELDS` rather -#: than typed out — a hand-list is what silently stops carrying the next promoted column. -_SNAPSHOT_OWN_KEYS = ("snapshot_key", "influencer_key", "pulled_at", "source", "approx") - - -def corpus_snapshot_row(row, day=""): - """ONE profile row read from the corpus → its `ut_ig_snapshots` observation. None if unusable. - - `day` overrides the row's own `last_found`, which is what the BACKFILL passes when a row's - only date is `first_found`. - - ⛔ IDEMPOTENT BY ITS KEY, never by a flag or a marker column. `snapshot_key` is - `@`, so re-running this over the same rows on the same day rewrites the same key - and `upsert_rows` merges it — the migration is safe to call on every write, which is the only - way it can be safe to call at all (W26's rule, and the same reason `migrate_ig_tables` keys - idempotency on the stored TYPE). - - ⚠ BLANKS ARE OMITTED RATHER THAN WRITTEN. `capture_rows` writes every key including the empty - ones because a paid pull's blank means "this rung did not read it" and the row is append-keyed. - Here the merge is the hazard instead: a second discovery on the same day returning a thinner - corpus record would otherwise ERASE what the first one learned. An absent key renders exactly - as an empty cell, so nothing is lost by leaving it out. - """ - handle = str((row or {}).get("handle") or "").strip().lstrip("@").lower() - if not handle: - return None - when = _day(day or (row or {}).get("last_found") or (row or {}).get("first_found") or "") - if not when: - return None - out = {"snapshot_key": f"{handle}@{when}", "influencer_key": handle, "pulled_at": when, - "source": _s(public_source(IG_VIA_DISCOVERY)), "approx": "1"} - for fd in SNAPSHOT_FIELDS: - key = fd["key"] - if key in _SNAPSHOT_OWN_KEYS: - continue - value = (row or {}).get(key) - if str(value or "").strip(): - out[key] = str(value) if key == "source_payload" else _s(value, 400) - return out - - -def append_ig_snapshots(rt, incoming, snap_key=IG_SNAPSHOTS_TABLE, log=print): - """Merge corpus observations into an EXISTING `ut_ig_snapshots`. Returns rows written. - - ⛔ IT NEVER CREATES THE TABLE, and that refusal is structural rather than cautious: the only - honest way to create it is `ensure_ig_graph`, which calls `ut_ensure`, which calls - `migrate_ig_tables` — and the BACKFILL caller is inside `migrate_ig_tables`. Reaching for the - creator from there is unbounded recursion. A tenant with no series store has nothing to - back-fill INTO; the forward path (`run_ig_discovery`) creates the graph properly and this - function then has somewhere to write. - """ - rows_in = [r for r in (incoming or []) if r] - if not rows_in: - return 0 - existing = ut_get(rt, snap_key) - if existing is None: - return 0 - merged, counts = upsert_rows(dict(existing.get("rows") or {}), rows_in, "snapshot_key", - cap=row_cap(snap_key)) - if counts["capped"]: - # D-11's law: a full append table means the SERIES has stopped growing, which is the one - # failure a chart cannot show you. - log(f"[aios-auto] corpus series: {counts['capped']} observation(s) refused by " - f"{snap_key}'s row cap") - written = counts["inserted"] + counts["updated"] - if not written: - return 0 - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - if cur.get(snap_key) is not None: - cur[snap_key]["rows"] = merged - return cur - - rt.update(UT_STORE_KEY, _up, flush="sync") - return written - - -# --------------------------------------------------------------------------------------------- -# RUNNERS -# --------------------------------------------------------------------------------------------- - -def cap_note(capped, missing=()): - """The sentence(s) a cap breach adds to a run summary. - - `capped` = `[(table_key, rows_lost), …]` — the ROW ceiling was hit. - `missing` = `[table_key, …]` — the TABLE ceiling was hit: the database could not - be created at all. - - ⛔ IT NAMES THE TABLE. "142 rows skipped" sends somebody to look at the source page; "the - ut_ig_post_snapshots database is FULL" sends them to the actual problem. A loud failure that - does not say WHERE is only marginally better than a silent one. - - ⛔ AND THE TABLE CEILING IS NOW AS LOUD AS THE ROW ONE (closes DEBT D-11). `ut_ensure` refused - SILENTLY at `MAX_UT_TABLES`: it returned a key for a database it had not created, the - runners' `if tgt is not None` write guard then skipped that bucket, and the run reported - success over a database that does not exist. Exactly the silent-stop failure the row cap - already had, one level up — and harder to spot, because the table is not there to look at. - """ - out = [f"⚠ the {k} database is FULL ({row_cap(k):,} rows). {n} row{'' if n == 1 else 's'} " - f"from this run were NOT written" for k, n in (capped or []) if n] - out += [f"⚠ the {k} database could NOT BE CREATED. This workspace is at its " - f"{MAX_UT_TABLES}-database limit, so nothing from this run reached it" - for k in (missing or [])] - return "; ".join(out) - - -def ut_missing(rt, *keys): - """Which of `keys` do NOT exist after an `ut_ensure` — the table-ceiling detector (D-11).""" - have = ut_all(rt) - return [k for k in keys if k and k not in have] - - -def run_scrape_db(rt, defn, username="automation", log=print, step=_no_step, rows=None): - """Automation #1 (R6): a public page → a blank database, re-runnable, UPSERT by key.""" - cfg = defn.get("config") or {} - fmap, key_field = cfg.get("fieldMap") or {}, cfg.get("keyField") - dry = bool(cfg.get("dryRun")) - # Per-NODE outcomes for the canvas. MEASURED as the run walks, never inferred afterwards from - # the rollup — inferring is exactly how a green dot ended up over an empty table (see the - # rollup comment in `run_field_instagram`). - steps = {"trigger": "ok", "fetch": "idle", "extract": "idle", "write": "idle"} - step(f"Fetching {urlparse(str(cfg.get('url') or '')).hostname or 'the page'}") - status, final, body = fetch(cfg["url"]) - if not (200 <= status < 300): - steps["fetch"] = "error" - return ("error", f"{cfg['url']} answered {status}. Nothing was written", - {"status": status}, [], steps) - steps["fetch"] = "ok" - soup = _soup(body) - if cfg.get("extract") == "jsonld": - blocks = jsonld(soup) - flat = [] - for b in blocks: - if isinstance(b, list): - flat.extend([x for x in b if isinstance(x, dict)]) - elif isinstance(b, dict): - items = b.get("itemListElement") - flat.extend([x for x in items if isinstance(x, dict)] if isinstance(items, list) - else [b]) - raw_rows = [{k: _scalar(v) for k, v in d.items()} for d in flat] - else: - found = tables(soup) - idx = max(0, min(int(cfg.get("tableIndex") or 0), len(found) - 1)) if found else 0 - raw_rows = [r for r in (found[idx] if found else []) if isinstance(r, dict)] - if not raw_rows: - steps["extract"] = "error" - return ("error", "the page parsed but the selected table had no rows", {"rows": 0}, [], - steps) - steps["extract"] = "ok" - step(f"Extracting {len(raw_rows)} row{'' if len(raw_rows) == 1 else 's'}") - - incoming = [] - for r in raw_rows: - mapped = {tgt: _s(r.get(src, ""), 500) for src, tgt in fmap.items()} - if any(v for v in mapped.values()): - incoming.append(mapped) - - fields = [field_def(tgt, src if len(src) <= 60 else src[:60]) - for src, tgt in fmap.items()] - # the key column leads, so the table opens on the identity it is upserted by - fields.sort(key=lambda f: 0 if f["key"] == key_field else 1) - label = cfg.get("targetLabel") or defn.get("name") or "Scraped table" - if dry: - # THE WRITE NODE IS OFF — read, compute, report, touch nothing. Not even `ut_ensure`, - # which would create the table: a dry run that leaves a new empty database behind is not - # a dry run. - table_key = ut_key_for(label, cfg.get("targetTable") or None) - else: - table_key = ut_ensure(rt, label, fields, username, key=cfg.get("targetTable") or None) - t = ut_get(rt, table_key) or {} - # ⛔ D-116 — THE ONE CALL SITE THAT NEEDS A PRECEDENCE RULE, because it is the one whose rows - # come from a CORPUS. Every other `upsert_rows` caller writes measurements or appends a series. - rows, counts = upsert_rows(t.get("rows") or {}, incoming, key_field, - cap=row_cap(table_key), protect=corpus_protect) - touched = [rid for rid, row in rows.items() - if str(row.get(key_field, "")) in {str(i.get(key_field)) for i in incoming}] - if not dry: - ut_write_rows(rt, table_key, rows) - counts["source_rows"] = len(incoming) - affected = touched[:200] - summary = (f"{counts['inserted']} new, {counts['updated']} updated, " - f"{counts['unchanged']} unchanged, {counts['orphans']} no longer on the page " - f"(kept)") - # ⛔ D-116 — SAY IT. A search that quietly declines to overwrite six measured cells is doing - # the right thing invisibly, which is indistinguishable from a corpus that happened to agree — - # and the next person to wonder why a number did not move has nothing to read. The observation - # still lands in the series either way; this sentence is about the SCALAR. - if counts.get("held"): - summary += (f", {counts['held']} measured value(s) kept over the corpus " - f"(an enrichment read them exactly; the search's own numbers are in the " - f"snapshot series)") - capped = [(table_key, counts.get("capped", 0))] - missing = [] if dry else ut_missing(rt, table_key) - counts["missing_tables"] = len(missing) - note = cap_note(capped, missing) - if note: - summary += f". {note}" - steps["write"] = "partial" - if dry: - summary = f"Test run. Nothing saved. Would have written: {summary}" - steps["write"] = "skipped" - elif not note: - steps["write"] = "ok" - state = "partial" if (counts.get("capped") or counts.get("skipped") or missing - or dry) else "ok" - log(f"[aios-auto] scrape_db {defn.get('id')} -> {table_key}: {summary}") - return (state, summary, counts, affected, steps) - - -def capture_rows(res, pulled): - """ONE pull → `(snapshot_row, post_identity_rows, post_metric_rows, comment_rows)`. - - ⭐ WAVE 25 (C4) — FACTORED OUT OF `run_field_instagram` SO THE ENRICH ACTION CAN REUSE IT - RATHER THAN FORK IT. C4's instruction is literal: "Reuse `pull_profile` and the write path — - do not fork them." A second copy of this mapping is the shape that goes wrong invisibly, - because the two copies would each be *plausible* and would disagree only on the rows a - particular rung happened to return. - - ⚠ THE TWO WRITE DISCIPLINES ARE OPPOSITE HERE, AND BOTH ARE DELIBERATE: - * A SNAPSHOT ROW IS APPEND-KEYED (`@`), so every key is written every - time and a blank is unambiguous — it means THIS PULL did not read it. - * AN IDENTITY ROW OMITS ITS BLANKS, because it is UPSERTED: `posted_at`/`caption`/`type` are - readable on some rungs and not others (the Bright Data PROFILE row carries post identity - with `datetime: None`, measured 24/24, while the engagement rung knows `date_posted`), and - `upsert_rows` writes exactly the keys it is handed — so sending "" would let a cheap run - ERASE what an expensive one learned. - * AN ENGAGEMENT SNAPSHOT IS APPENDED ONLY WHEN SOMETHING WAS MEASURED. The post series is - the append table that FILLS (maxPosts rows per profile per pull); with the paid engagement - rung OFF every one of those rows would carry three blanks — pure noise, eating a 200k - ceiling and drawing a chart of nothing. A row in a time series should mean "this was true - then"; a row meaning "nobody looked" belongs nowhere. - """ - prof = (res or {}).get("profile") or {} - snap_row = { - "snapshot_key": f"{prof.get('username')}@{pulled}", - "influencer_key": prof.get("username"), "pulled_at": pulled, - # ⛔ THE PUBLIC WORD, NOT THE VENDOR KEY — this cell is rendered in a grid column called - # "Read via". `res["via"]` keeps the real key for the call graph and the server log. - "source": _s(public_source((res or {}).get("via"))), "approx": _s(prof.get("approx")), - "bio": _s(prof.get("bio"), 500), - "external_url": _s(prof.get("external_url"), 300), - } - for fd in SNAPSHOT_FIELDS: - k = fd["key"] - if k not in snap_row: - value = prof.get(k) - # Profile history declares the same percentage dialect as the current Profile row. - # Converting only the latest projection left the append series at 0-1 while its field - # now said pct (0-100), so the two views of one measurement disagreed by 100x. - if k == "source_payload": - snap_row[k] = str(value or "") - else: - snap_row[k] = _s(_pct100(value) if k == "avg_engagement" else value, 400) - idents, metrics_rows, comment_rows = [], [], [] - for p in (res or {}).get("posts") or []: - ident = {"shortcode": p["shortcode"], "influencer_key": prof.get("username"), - "url": _s(p.get("url"), 300)} - for k, n in (("posted_at", 200), ("type", 200), ("caption", 800), - ("paid_partnership", 8), ("partner", 120), ("hashtags", 400), - ("alt_text", 400), ("tagged_location", 400)): - if p.get(k): - ident[k] = _s(p.get(k), n) - if p.get("source_payload"): - ident["source_payload"] = str(p["source_payload"]) - metrics = {k: p.get(k) for k in ("likes", "comments", "views", "plays")} - # ⭐⭐ 2026-08-07 — THE LATEST ENGAGEMENT VALUES, ONTO THE POST ROW ITSELF. - # - # This is what keeps a rollup at ONE HOP (Airtable's rule and ours): without it, "average - # views over the last 12 posts" would have to walk profile → posts → each post's most - # recent snapshot, and a two-hop rollup is a much larger feature with a much worse - # invalidation story. The post row now carries its own latest, exactly as the profile row - # carries LATEST + `enriched_at` while its series lives in `ut_ig_snapshots` (R3). - # - # ⛔ ONE STORE FOR ONE SERIES IS UNTOUCHED: `ut_ig_post_snapshots` below is still the - # authoritative engagement series and still gets its appended row. These cells are a - # projection of the row being appended in the same breath, never a second source. - # ⛔ A KEY IS WRITTEN ONLY WHEN THE VENDOR ANSWERED. `upsert_rows` merges, so an absent - # key leaves the previous pull's value standing — which is the correct behaviour for a - # LATEST column and the reason this cannot be a blanket `_s(...)` over all three: writing - # "" on a run that did not buy metrics would ERASE what an earlier paid run learned, the - # same failure the snapshot-append rule above is written against. - measured = {k: v for k, v in metrics.items() if v is not None and str(v).strip() != ""} - if measured: - ident.update({k: _s(v, 40) for k, v in measured.items()}) - # The stamp is what makes the numbers readable: a blank `plays` beside a - # `measured_at` of last week means "we looked and the vendor had nothing", and with - # no stamp it means that AND "we never looked", indistinguishably. - ident["measured_at"] = _day(pulled) - idents.append(ident) - if any(v is not None for v in metrics.values()): - metrics_rows.append({ - "post_snapshot_key": f"{p['shortcode']}@{pulled}", - "shortcode": p["shortcode"], "influencer_key": prof.get("username"), - "pulled_at": pulled, - # The two denormalised post facts (see `POST_SNAPSHOT_FIELDS`). Written from the - # SAME vendor record the identity row above is built from, so they cannot disagree - # with it on this pull. - # ⚠ BLANK IS EXPECTED AND IS NOT A BUG on the profile rung: `wave20-split` - # measured `datetime` as None on 24/24 posts from the Profiles dataset, so a pull - # that learns a post's engagement often does not learn its date in the same - # breath. `backfill_post_snapshot_grain` fills those from the post row, which by - # then may have learned it from a different rung. - "posted_at": _s(p.get("posted_at"), 200), "type": _s(p.get("type"), 200), - "likes": _s(metrics["likes"]), "comments": _s(metrics["comments"]), - "views": _s(metrics["views"]), "plays": _s(metrics["plays"]), - "source_payload": str(p.get("source_payload") or "")}) - for comment in p.get("embedded_comments") or []: - # The embedded shape often names only the commenter. Its parent Post is the - # authoritative owner, so bind every preview to this profile and post explicitly. - comment_rows.append({**comment, "shortcode": p["shortcode"], - "influencer_key": prof.get("username") or ""}) - comment_rows.extend((res or {}).get("comments") or []) - # The same comment can be returned as both `latest_comments` and `top_comments`, or by the - # optional full endpoint after an embedded preview. The canonical key is the one row truth. - unique_comments = {} - for row in comment_rows: - key = str((row or {}).get("comment_key") or "") - if key: - unique_comments[key] = {**(unique_comments.get(key) or {}), **row} - return snap_row, idents, metrics_rows, list(unique_comments.values()) - - -#: C4 — how ONE pulled profile becomes the C1 preset CELLS on the record being enriched. -#: `profile key -> preset column key`, so the mapping is a table rather than sixteen lines of -#: `row[...] = prof.get(...)` that a future field addition can silently miss. -#: ⚠ Keys absent from a pull are simply not written (R3: a blank preset cell means "this pull did -#: not read it", and `enriched_at` is what distinguishes that from "never enriched"). -PRESET_FROM_PROFILE = { - "username": "handle", "profile_url": "profile_url", "full_name": "full_name", - "followers": "followers", "following": "following", "avg_engagement": "avg_engagement", - "bio": "bio", "external_url": "external_url", "verified": "verified", - "category": "category", "posts_count": "posts_count", - "highlights_count": "highlights_count", "is_business": "is_business", - "is_professional": "is_professional", "ig_id": "ig_id", - # ⭐ 2026-08-07 — the promoted fields. A preset column with no row in this table is a column - # that can only ever be blank, so "make the preset fields populated" is THIS half of the - # owner's instruction and the field list is only the other half. - # ⚠ The two lists are held in step by a derived gate rather than by care: every - # `PRESET_PROFILE_KEYS` entry must either be written by this map or be explicitly declared - # as written elsewhere (`platform`, `enriched_at`, `posts` are stamped by `preset_cells`). - "business_category": "business_category", "is_private": "is_private", - "bio_hashtags": "bio_hashtags", - "pronouns": "pronouns", "profile_name": "profile_name", - "is_joined_recently": "is_joined_recently", "has_channel": "has_channel", - "partner_id": "partner_id", "external_url_title": "external_url_title", - "fbid": "fbid", "related_accounts": "related_accounts", - "country_code": "country_code", "source_payload": "source_payload", -} - -#: ⭐ 2026-08-07 — the preset keys `PRESET_FROM_PROFILE` deliberately does NOT carry, because a -#: different writer stamps them. DERIVED gates compare the two lists, and without this the gate -#: could only be written as "these three are fine" — a hard-coded exception list, which is the -#: shape [[gate-answers-the-wrong-question]] warns about. -#: ⚠ The five relational columns are written by `compute_relation_cells` on the tick, NOT by an -#: enrichment run — which is the whole point of them: they stay true when the LINKED table -#: changes, and a pull that touched no profile still updates a profile's post count. -PRESET_WRITTEN_ELSEWHERE = ( - "platform", "handle", "enriched_at", - "posts_link", "profile_snapshots_link", "post_snapshots_link", "comments_link", - "avg_views_12", "avg_plays_12", "avg_likes_12", "avg_comments_12", - "posts_captured", "profile_reads", "post_measurements_captured", "comments_captured", -) - - -#: A location guess needs at least this many GEOTAGGED posts to agree with. One post is not a -#: pattern and would render as 100% confident, which is the fabrication `SEED_MIN_ROWS_SHARING` -#: refuses on the discovery side for exactly the same reason. -LOCATION_MIN_POSTS = 2 - - -def _post_place(post): - """The city a post was tagged in, or ''. Reads the normalised field FIRST, the paid payload - second. - - ⛔ THE SECOND READ IS THE POINT AND IT COSTS NOTHING. `_bd_tagged_location` flattens the - vendor's `location` array into "Capanema, Pará, Brasil" and DISCARDS the rest — including the - `lat`/`lng`/`name` object Bright Data returns on some rows and the whole `location_details` - shape. The complete vendor row is retained in `source_payload` on the same record, already - paid for, so a post whose normalised field came back blank can still be read here. - ⚠ THE FIRST COMPONENT IS THE CITY. The vendor's array is ordered outward — city, region, - country — so the head is the narrowest thing it told us, and "Jakarta" is the answer to - "where is this person"; "Indonesia" mostly is not. - """ - if not isinstance(post, dict): - return "" - raw = str(post.get("tagged_location") or "").strip() - if not raw: - try: - payload = json.loads(post.get("source_payload") or "{}") - except Exception: # noqa: BLE001 - payload = {} - raw = str(_bd_tagged_location(payload) or "") if isinstance(payload, dict) else "" - return raw.split(",")[0].strip() - - -def location_guess(posts): - """`(city, confidence_pct)` from the places a creator's own posts were tagged in. - - `confidence` is the share of GEOTAGGED posts that agree on the modal city — deliberately not - the share of ALL posts, because a creator who geotags three of twelve is telling us about - three, and dividing by twelve would report a real signal as weak. How thin the evidence is - stays visible a different way: `LOCATION_MIN_POSTS` refuses to answer at all below two. - """ - places = [p for p in (_post_place(x) for x in posts or []) if p] - if len(places) < LOCATION_MIN_POSTS: - return "", None - counts = {} - for p in places: - counts[p] = counts.get(p, 0) + 1 - # ⚠ TIE-BROKEN BY FIRST APPEARANCE, never by sort order: two cities at 3/3 would otherwise be - # resolved alphabetically, which is an arbitrary answer wearing a confident number. - best = max(places, key=lambda p: (counts[p], -places.index(p))) - return best, round(100.0 * counts[best] / len(places), 1) - - -def preset_cells(res, pulled): - """C4/R3: one pull → the LATEST-value cells written onto the enriched record. - - ⛔ `enriched_at` IS ALWAYS WRITTEN when a pull succeeded, and it is the field that makes the - other fifteen readable: without it a blank `followers` means both "never enriched" and - "enriched in March and the vendor had nothing", and no cell on the row can tell them apart. - """ - prof = (res or {}).get("profile") or {} - cells = {} - for src, dest in PRESET_FROM_PROFILE.items(): - v = prof.get(src) - if v is not None and v != "": - # ⚠ WAVE 26 / C1-a — the ONE key whose value is not carried straight across. The - # vendor's engagement is a 0–1 fraction and our `pct` column is 0–100, so a - # straight-through copy here would write the same wrong number the discovery path - # used to write. Both writers convert; neither is the exception. - cells[dest] = (_pct100(v) if dest == "avg_engagement" else - (str(v) if dest == "source_payload" else _s(v, 500))) - # ⭐ R5 — stamped by the writer, not carried from the pull: this function only ever sees an - # Instagram profile. It matters on an ENRICH of a row somebody typed by hand, which may have - # arrived with no platform at all — and a blank half of the dedup key is how one account - # becomes two rows. - cells["platform"] = PLATFORM_INSTAGRAM - # ⭐ ITEM 16 — the residency guess, off posts this pull already paid for. Written only when - # there is one: a blank is "we could not tell", and overwriting a good guess from a run whose - # twelve posts happened to carry no geotag would make the column worse the more it ran. - place, confidence = location_guess((res or {}).get("posts")) - if place: - cells["location_guess"] = _s(place, 120) - cells["location_confidence"] = str(confidence) - # R3: `enriched_at` is a `date` column now. It answers "how stale is this number", which is a - # question in days; the full stamp keeps its precision on the snapshot series. - cells["enriched_at"] = _day(pulled) - return cells - - -#: Keys the TikTok SNAPSHOT row owns itself, so the copy loop below never overwrites them from the -#: profile map. Mirrors `_SNAPSHOT_OWN_KEYS` on the Instagram side. -_TT_SNAPSHOT_OWN_KEYS = ("snapshot_key", "influencer_key", "pulled_at", "source", "approx") - - -def tt_preset_cells(res, pulled): - """⭐⭐ WAVE 30 · T08 — one TikTok pull → the LATEST-value cells written onto the record. - - ⛔ THIS IS NOT `preset_cells` WITH A FLAG, AND THAT IS THE WHOLE DESIGN DECISION. Instagram's - `preset_cells` walks `PRESET_FROM_PROFILE` (an Instagram map), stamps - `cells["platform"] = PLATFORM_INSTAGRAM` UNCONDITIONALLY, and computes a location guess from - posts. Passing a TikTok row through it would stamp every TikTok creator as Instagram — which is - the identity half of W26's `(platform, handle)` key, so it would not merely mislabel a row, it - would MERGE two different people's accounts under one key. - ⭐ AND THE TIKTOK ROW DOES NOT NEED A MAP AT ALL. `connectors_tt.normalize_profile` already - emits our column names — it is the SAME function the discovery runner uses — so a scraped row - and a corpus row cannot disagree about which vendor key became which column. All this adds is - what the FETCH knows and the map cannot: when it was read, and by which route. - - ⚠ FILTERED TO THE DECLARED SCHEMA. `ut_write_rows`/`_clean_field` DROP an undeclared key with - no error and a successful-looking run (the defect `section_w29_tiktok_schema`'s emit-vs-declare - sweep exists for), so an unknown key is refused here where it is visible rather than swallowed - three layers down. - """ - prof = (res or {}).get("profile") or {} - declared = {f["key"] for f in TT_PROFILE_FIELDS} - cells = {k: v for k, v in prof.items() if k in declared} - # `source` is the ROUTE that answered, not the vendor's name — the same thing `via` carries on - # the Instagram snapshot, and the reason a stored measurement can always say how it was read. - if (res or {}).get("via"): - cells["source"] = _s((res or {}).get("via"), 60) - # R3, as on the Instagram side: a `date`, because "how stale is this number" is a question in - # days. Without it a blank `followers` means both "never enriched" and "enriched and empty". - cells["enriched_at"] = _day(pulled) - return cells - - -#: The metric columns a TikTok post SNAPSHOT carries. ⛔ DERIVED from the declaration, minus the -#: keys the snapshot owns itself — so adding a metric to `TT_POST_SNAPSHOT_FIELDS` starts being -#: captured, and adding an IDENTITY column to it never does. -_TT_PSNAP_OWN_KEYS = ("platform", "post_snapshot_key", "shortcode", "influencer_key", "pulled_at", - "post_link") -TT_PSNAP_METRIC_KEYS = tuple(f["key"] for f in TT_POST_SNAPSHOT_FIELDS - if f["key"] not in _TT_PSNAP_OWN_KEYS) - - -def tt_capture_rows(res, pulled): - """⭐⭐ WAVE 30 · T10 — one TikTok pull → `(post_identity_rows, post_metric_rows, comment_rows)`. - - The TikTok twin of `capture_rows`, and separate for the same reason `tt_preset_cells` is: that - function walks Instagram's `SNAPSHOT_FIELDS`/`PRESET_FROM_PROFILE` and stamps Instagram's - platform. The mappers have already done the vendor→our-keys work here - (`connectors_tt.normalize_post` / `normalize_comment`), so this adds only what the FETCH knows. - - ⚠ THE TWO WRITE DISCIPLINES ARE OPPOSITE, exactly as on the Instagram side: - * AN IDENTITY ROW OMITS ITS BLANKS, because it is UPSERTED — `upsert_rows` writes the keys it - is handed, so sending "" would let a thin run ERASE what a full one learned. - * A METRIC SNAPSHOT IS APPENDED ONLY WHEN SOMETHING WAS MEASURED. A row in a time series - should mean "this was true then"; a row meaning "nobody looked" eats the cap and draws a - chart of nothing. - - ⛔⛔ D-117 IS NOT REPRODUCED HERE, AND THAT IS THE INSTRUCTION THE TICKET LEADS WITH. - Instagram denormalises `posted_at` and `type` onto its post SNAPSHOT rows, where they never - reconcile against the identity table and quietly become a second, ageing answer to a question - the posts table already answers. This snapshot carries the METRICS and the join key and nothing - else — `TT_PSNAP_METRIC_KEYS` is derived from the declaration minus the keys the snapshot owns, - so the exclusion is structural rather than a list somebody has to remember to keep short. - """ - prof = (res or {}).get("profile") or {} - who = str(prof.get("handle") or "").strip().lstrip("@").lower() - idents, metric_rows, comment_rows = [], [], [] - for p in (res or {}).get("posts") or []: - shortcode = str((p or {}).get("shortcode") or "").strip() - if not shortcode: - continue - ident = {k: v for k, v in p.items() if v not in (None, "")} - # ⚠ The Posts dataset carries `profile_username`, but a row that omits it must still join: - # the profile whose `top_videos` we scraped IS the influencer, and that is a fact of the - # call rather than of the row. - ident.setdefault("influencer_key", who) - ident["platform"] = PLATFORM_TIKTOK - idents.append(ident) - measured = {k: p.get(k) for k in TT_PSNAP_METRIC_KEYS - if k != "source_payload" and p.get(k) is not None} - if not measured: - continue - row = {"platform": PLATFORM_TIKTOK, "shortcode": shortcode, - "influencer_key": str(p.get("influencer_key") or who), - "post_snapshot_key": f"{shortcode}@{pulled}", "pulled_at": pulled} - for k, v in measured.items(): - row[k] = _s(v, 400) - metric_rows.append(row) - for c in (res or {}).get("comments") or []: - if not str((c or {}).get("comment_key") or "").strip(): - continue - row = {k: v for k, v in c.items() if v not in (None, "")} - # Same fact-of-the-call argument: the Comments dataset has no influencer field at all, so - # without this the comments table could never be filtered by creator. - row.setdefault("influencer_key", who) - row["platform"] = PLATFORM_TIKTOK - comment_rows.append(row) - return idents, metric_rows, comment_rows - - -def tt_snapshot_row(res, pulled): - """One TikTok pull → its `ut_tt_snapshots` observation, or None when there is no handle. - - ⛔ THE APPEND LAW, UNCHANGED FROM INSTAGRAM AND FOR THE SAME MEASURED REASON: TikTok's Profiles - dataset carries **no measurement timestamp** (`create_time` is when the ACCOUNT was made, not - when `followers` was true — probed, `tiktok-capture.md`). So the series is dated by when WE - read it, and the key is `@` — idempotent by its KEY rather than by a flag, - so re-running over the same rows rewrites the same row instead of growing the table. - ⚠ `pulled_at` keeps the FULL stamp while the record's `enriched_at` is a day: the record answers - "how stale", the series answers "when exactly", and collapsing the second into the first would - make two reads on one day indistinguishable. - """ - prof = (res or {}).get("profile") or {} - handle = str(prof.get("handle") or "").strip().lstrip("@").lower() - if not handle: - return None - out = {"platform": PLATFORM_TIKTOK, "snapshot_key": f"{handle}@{pulled}", - "influencer_key": handle, "pulled_at": pulled, - "source": _s((res or {}).get("via") or "brightdata", 60)} - for fd in TT_SNAPSHOT_FIELDS: - key = fd["key"] - if key in _TT_SNAPSHOT_OWN_KEYS or key == "platform": - continue - value = prof.get(key) - # ⚠ A ZERO IS A MEASUREMENT AND SURVIVES — the test is `is not None`, never truthiness. - # `str(value or "").strip()` would drop a genuine 0 follower count, and this is a series - # whose whole purpose is that a number moved. - if value is not None and str(value).strip() != "": - out[key] = str(value) if key == "source_payload" else _s(value, 400) - return out - - -def run_field_instagram(rt, defn, username="automation", log=print, step=_no_step, - rows=None): - """Automation #2 (R7): for every row of a database that carries a profile URL, pull the public - profile through the paid capability chain, write a status string into the automation column, - and append a timestamped row to each of the three IG tables. - - ⭐ WAVE 28 / R5 — there is no tier and no rung choice here any more. `pull_profile` routes per - CAPABILITY and reports `blocked` rather than downgrading to an approximate row, so the two - "which rung answered" counters this function used to keep have nothing left to distinguish.""" - cfg = defn.get("config") or {} - table_key, fkey = cfg.get("targetTable"), cfg.get("fieldKey") - post_metrics = bool(cfg.get("postMetrics")) - comment_metrics = bool(cfg.get("commentMetrics")) - dry = bool(cfg.get("dryRun")) - # ⚠ THE STEP KEYS ARE THE CANVAS NODE IDS (contract C3) and the two must move together — a - # status written under a node id `graph()` no longer emits is a dot nothing renders, which is - # indistinguishable from a step that never ran. - steps = {"trigger": "ok", "source": "idle", - "capture_posts": "idle" if post_metrics else "skipped", - "capture_comments": "idle" if comment_metrics else "skipped", - "write": "idle"} - t = ut_get(rt, table_key) - if not t: - steps["source"] = "error" - return ("error", f"{table_key} is not a database in this workspace", {}, [], steps) - url_field = cfg.get("urlField") or _auto_url_field(t, fkey) - if not url_field: - steps["source"] = "error" - return ("error", "the automation column has no URL field bound to it", {}, [], steps) - steps["source"] = "ok" - rows = dict(t.get("rows") or {}) - # the relational tables the pull lands in (R7): every row timestamped for time-range filters - if dry: - # The Write node is off: resolve the keys, create nothing. (`ut_ensure` writes.) - snap_key, post_key, ps_key, comment_key = (IG_SNAPSHOTS_TABLE, IG_POSTS_TABLE, - IG_POST_SNAPSHOTS_TABLE, IG_COMMENTS_TABLE) - else: - graph = ensure_ig_graph(rt, username, str(defn.get("id") or ""), - profile_table=table_key) - snap_key, post_key, ps_key, comment_key = (graph[IG_SNAPSHOTS_TABLE], graph[IG_POSTS_TABLE], - graph[IG_POST_SNAPSHOTS_TABLE], graph[IG_COMMENTS_TABLE]) - missing = [] if dry else ut_missing(rt, snap_key, post_key, ps_key, comment_key) - snaps = dict((ut_get(rt, snap_key) or {}).get("rows") or {}) - posts = dict((ut_get(rt, post_key) or {}).get("rows") or {}) - psnaps = dict((ut_get(rt, ps_key) or {}).get("rows") or {}) - comments = dict((ut_get(rt, comment_key) or {}).get("rows") or {}) - - counts = {"profiles": 0, "ok": 0, "partial": 0, "blocked": 0, "error": 0, - "posts": 0, "new_posts": 0, "paid": 0, "capped": 0, "metrics": 0, "comment_rows": 0, - "missing_tables": len(missing)} - cells, affected, notes = {}, [], [] - pending_metrics = [] - # ⛔ ACCUMULATE HERE, UPSERT ONCE PER TABLE AFTER THE LOOP. This used to call `upsert_rows` - # once per POST, which is O(existing) per call — survivable only while the cap was 5000 rows. - # `MAX_UT_IG_ROWS` makes the same loop hundreds of millions of dict copies, i.e. an automation - # that no longer finishes. **Raising a cap and batching its writer are ONE change.** (W19-C.) - in_snaps, in_posts, in_psnaps, in_comments = [], [], [], [] - targets = [(rid, str(r.get(url_field, "") or "").strip()) - for rid, r in rows.items() if str(r.get(url_field, "") or "").strip()] - for i, (rid, url) in enumerate(targets): - if i: - time.sleep(PACE_SECONDS) # ≥2 s between profiles (R7) - # ITEM 6: the other genuinely long runner — ≥2 s of pacing per profile means a 60-profile - # column automation blocks for minutes by design. A counting step is the difference - # between "working through them" and "stuck". - step(f"Capturing profile {i + 1} of {len(targets)}") - counts["profiles"] += 1 - res = pull_profile(url, max_posts=cfg.get("maxPosts") or DEFAULT_POSTS_PER_PULL, log=log, - post_metrics=post_metrics, comment_metrics=comment_metrics, - pending_metrics=(pending_metrics if not dry else None)) - pulled = _iso() - via = res.get("via") or "" - read_ok = res["state"] in ("ok", "partial") - # ⚠ `counts["paid"]` SURVIVES R5 AND IT IS NOT THE RETIRED TIER. It counts profiles a PAID - # vendor actually answered, which is the run's spend report — the thing the owner reads to - # reconcile a bill. What died is the free rung it used to be contrasted with, so the test - # is now simply "did a vendor answer" rather than "did the vendor we were told to try". - if read_ok and via in ("brightdata", "apify"): - counts["paid"] += 1 - if read_ok: - counts["ok" if res["state"] == "ok" else "partial"] += 1 - prof = res["profile"] - snap_row, ident_rows, metric_rows, captured_comments = capture_rows(res, pulled) - in_snaps.append(snap_row) - in_posts.extend(ident_rows) - in_psnaps.extend(metric_rows) - in_comments.extend(captured_comments) - counts["metrics"] += len(metric_rows) - counts["comment_rows"] += len(captured_comments) - counts["posts"] += len(res.get("posts") or []) - detail = f"{prof.get('followers') or '?'} followers" - if prof.get("approx"): - detail += " (approx)" - if res.get("posts"): - detail += f", {len(res['posts'])} posts" - elif res["state"] == "partial": - detail += ", posts not readable on the rung that answered" - cells[rid] = f"{res['state']} · {_stamp()} · {detail}" - elif res["state"] == "blocked": - counts["blocked"] += 1 - cells[rid] = f"blocked · {_stamp()} · {_s(res.get('note'), 90)}" - notes.append(res.get("note") or "blocked") - else: - counts["error"] += 1 - cells[rid] = f"error · {_stamp()} · {_s(res.get('note'), 90)}" - notes.append(res.get("note") or "error") - affected.append(rid) - - # --- THE THREE UPSERTS. Once each, over the whole run's accumulated rows. - snaps, c_snap = upsert_rows(snaps, in_snaps, "snapshot_key", cap=row_cap(snap_key)) - posts, collapsed_posts = dedupe_canonical_rows(posts, "shortcode", newest_by="measured_at") - posts, c_post = upsert_rows(posts, in_posts, "shortcode", cap=row_cap(post_key)) - c_post["duplicates"] += collapsed_posts - psnaps, c_ps = upsert_rows(psnaps, in_psnaps, "post_snapshot_key", cap=row_cap(ps_key)) - comments, collapsed_comments = dedupe_canonical_rows(comments, "comment_key") - comments, c_comments = upsert_rows(comments, in_comments, "comment_key", cap=row_cap(comment_key)) - c_comments["duplicates"] += collapsed_comments - counts["new_posts"] = c_post["inserted"] - capped = [(snap_key, c_snap["capped"]), (post_key, c_post["capped"]), - (ps_key, c_ps["capped"]), (comment_key, c_comments["capped"])] - counts["capped"] = sum(n for _k, n in capped) - - if not dry: - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - tt = cur.get(table_key) - if tt is not None: - for rid, val in cells.items(): - tt.setdefault("rows", {}).setdefault(str(rid), {})[fkey] = val - for k, rws in ((snap_key, snaps), (post_key, posts), (ps_key, psnaps), - (comment_key, comments)): - tgt = cur.get(k) - if tgt is not None: - tgt["rows"] = rws - _refresh_relations_inplace(cur, log=log) - return cur - - rt.update(UT_STORE_KEY, _up, flush="sync") # ONE coalesced update for all four tables - queued = queue_pending_metric_snapshots(rt, str(defn.get("id") or ""), pending_metrics) - if queued: - counts["metric_batches_pending"] = queued - - # --- C6 (R2): WRITE-THROUGH to the platform master. Three postures, never conflated: - # `ok` silent, `off` an honest aside (the tenant copy IS the story on this deployment), - # `error` a LOUD partial — a pooled history with silent holes is worse than none. - master_note = "" - if not dry and (in_snaps or in_posts or in_psnaps): - import ig_master - m_status, m_note = ig_master.append_run(getattr(rt, "key", ""), - in_snaps, in_posts, in_psnaps) - if m_status == "error": - master_note = f"⚠ the platform master copy FAILED. {m_note}; the tenant copy " \ - f"is complete and the next run re-appends" - counts["master_failed"] = 1 - elif m_status == "ok": - counts["master"] = len(in_snaps) + len(in_psnaps) - - # ⚠ A RUN IS ONLY 'ok' IF EVERY PROFILE WAS. The first live run rolled up to 'ok' while - # every CELL said `partial`, because the rollup only asked about blocked/error — so the rail - # showed a green dot over a table with no posts in it. A summary that disagrees with the - # cells it summarises is the failure this whole module's honest-status rule exists to - # prevent, so `partial` now propagates. Measured 2026-08-04. - read = counts["ok"] + counts["partial"] - if not read: - state = "error" if counts["error"] else "partial" if counts["blocked"] else "ok" - elif counts["blocked"] or counts["error"] or counts["partial"]: - state = "partial" - else: - state = "ok" - if counts.get("metric_batches_pending"): - state = "partial" - # --- per-node outcomes for the canvas (see the same note in `run_scrape_db`) - if not targets: - cap_state = "idle" - elif not read: - cap_state = "error" if counts["error"] else "blocked" - else: - cap_state = "partial" if (counts["blocked"] or counts["error"] - or counts["partial"]) else "ok" - # ⭐ C3 — THE CAPTURE FORK IS GONE, SO ITS OUTCOME LANDS ON `source`. There is one way to - # read a profile now, and the node that names the profile set is the honest owner of "did - # reading them work". The old `capture`/`capture_paid`/`capture_anon` trio described a branch - # that no longer exists in behaviour OR on screen. - steps["source"] = cap_state if targets else steps["source"] - # ⚠ MEASURED, LIKE EVERY OTHER DOT: it is `ok` only if an engagement row was actually - # appended. "It was switched on" is not the same fact as "it answered", and painting the - # second from the first is the fabrication `node_status` refuses to make. - # ⚠ AND `blocked` IS A CLAIM ABOUT THE VENDOR, so it needs something to have been ASKED. A - # run that found no posts to enrich did not have a rung refuse it — nothing was requested — - # so that reads `skipped`, the same word an off switch earns. - steps["capture_posts"] = ("ok" if counts["metrics"] - else "partial" if counts.get("metric_batches_pending") - else "blocked" if (post_metrics and counts["posts"]) - else "skipped") - steps["capture_comments"] = ("ok" if counts.get("comments") - else "blocked" if (comment_metrics and counts["posts"]) - else "skipped") - - summary = (f"{read}/{counts['profiles']} profiles read, {counts['posts']} posts " - f"({counts['new_posts']} new)") - if counts.get("metric_batches_pending"): - summary += (f", {counts['metric_batches_pending']} post-engagement batch" - f"{'' if counts['metric_batches_pending'] == 1 else 'es'} still building " - "(collected automatically)") - if counts["paid"]: - summary += f", {counts['paid']} with exact counts" - if counts["metrics"]: - summary += f", {counts['metrics']} post engagement snapshots" - elif post_metrics and counts["posts"] and not counts.get("metric_batches_pending"): - # ⚠ ASKED FOR AND NOT DELIVERED IS ITS OWN SENTENCE. Silence here would read as "there - # was no engagement", which is a claim about Instagram rather than about our run. - summary += ", no post engagement was readable" - if counts["comment_rows"]: - summary += f", {counts['comment_rows']} comment rows captured" - elif comment_metrics and counts["posts"]: - summary += ", no comment engagement was readable" - if counts["partial"]: - summary += f", {counts['partial']} profile-only (posts not readable)" - if counts["blocked"]: - summary += f", {counts['blocked']} blocked" - note = cap_note(capped, missing) - if note: - summary += f". {note}" - state = "partial" if state != "error" else state - steps["write"] = "partial" - elif not dry: - steps["write"] = "ok" - if master_note: - summary += f". {master_note}" - state = "partial" if state != "error" else state - steps["write"] = "partial" - if dry: - summary = f"Test run. Nothing saved. Would have written: {summary}" - steps["write"] = "skipped" - state = "partial" if state != "error" else state - if notes: - summary += f". {notes[0][:120]}" - if not dry: - # C7: the fresh master rows are exactly what the table's metric cells summarise. - try: - compute_metric_cells(rt, table_key) - except Exception as e: # noqa: BLE001 - log(f"[aios-auto] metric compute {table_key} failed: {type(e).__name__}: {e}") - return (state, summary, counts, affected, steps) - - -def _auto_url_field(table, fkey): - """The URL field an automation column is bound to: its own `automation.urlField`, else the - first url-typed column on the table. Never guessed silently — the caller reports which.""" - for f in table.get("fields") or []: - if f.get("key") == fkey: - bound = ((f.get("automation") or {}).get("urlField") or "").strip() - if bound: - return bound - for f in table.get("fields") or []: - if f.get("type") == "url": - return f.get("key") - return "" - - -#: ⭐ W29-T04 — the discovery-run bookkeeping columns, so `run_discover_tiktok` and the D-116 merge -#: rule agree on what discovery OWNS versus what enrichment owns. Anything not in this set is a -#: MEASUREMENT, and a corpus re-find must never overwrite an enriched measurement with its own -#: cheaper approximation. -TT_DISCOVERY_KEYS = frozenset({"platform", "handle", "created_by", "found_count", - "first_found", "last_found", "source", "source_payload"}) - - -def _tt_candidate_row(row, stamp): - """One TikTok corpus row → a `ut_tt_profile` candidate row. None without a handle. - - ⛔ NEVER EMITS `found_count` — the runner computes it against what is already stored, so - writing it here would reset the counter to 1 on every re-find. Same rule, same reason, as - `_candidate_row` on the Instagram side. - ⭐ THE FIELD MAP IS THE CONNECTOR'S, not a second one written here. `normalize_profile` is what - the enrich path will use too, so a corpus row and a scrape row cannot disagree about which - vendor key becomes which column. - """ - import connectors_tt as _tt - cells = _tt.normalize_profile(row if isinstance(row, dict) else {}) - if not str(cells.get("handle") or "").strip(): - return None - return {**cells, "last_found": stamp, "source": "corpus"} - - - -def _ig_discovery_series(rt, table_key, incoming, stamp, username, auto_id, counts, log): - """⭐⭐ THE FORWARD HALF (2026-08-10), Instagram only. Every candidate a run wrote also leaves - an OBSERVATION behind — see `corpus_snapshot_row`. Without it a backfill is theatre: the next - discovery run re-opens exactly the gap the backfill just closed, which is how a data defect - becomes a recurring one. - - ⚠ `ensure_ig_graph`, NOT a snapshot-only `ut_ensure`, and it is a real behaviour change worth - owning: a tenant that has never enriched gets the four canonical child databases the first time - discovery runs (four of `MAX_UT_TABLES = 40`). The alternative writes a series into a store the - profile table has no LINK to — the numbers would be recorded and unreachable, which is half a - feature wearing a whole one's clothes. - ⚠ `incoming`, not `merged`: the observation belongs to the profiles THIS RUN read, not to every - row the table happens to hold. - """ - try: - graph = ensure_ig_graph(rt, username, auto_id, profile_table=table_key) - counts["series"] = append_ig_snapshots( - rt, [corpus_snapshot_row(c, day=stamp) for c in incoming], - graph[IG_SNAPSHOTS_TABLE], log=log) - except Exception as e: # noqa: BLE001 - # The candidates already landed. A series write that fails must not throw away the rows we - # just paid the vendor for — the same posture the preset top-up takes. - log(f"[aios-auto] discover {auto_id}: corpus series write failed: " - f"{type(e).__name__}: {e}") - - -def _ig_discovery_metrics(rt, table_key, auto_id, log): - """C7 — the metric FIELDS computed off the master series. See the field runner.""" - try: - compute_metric_cells(rt, table_key) - except Exception as e: # noqa: BLE001 - log(f"[aios-auto] metric compute {table_key} failed: {type(e).__name__}: {e}") - - -#: ⭐⭐ WAVE 30 · T09 (DEBT D-129) — ONE DISCOVERY RUNNER, TWO CORPORA, AND EVERY DIFFERENCE BETWEEN -#: THEM IS A ROW IN THIS TABLE. -#: -#: `run_discover_tiktok` was written in wave 29 as a deliberate SIBLING of `run_discover_instagram` -#: — its own docstring said so and gave the reason: *"it would rewrite the one code path this -#: product bills money through"*. That was the right call with one platform's worth of evidence and -#: it stopped being right when the duplication reached ~210 lines, because a sibling does not -#: inherit fixes. The proof is already in the copy below: the TikTok runner carries the **D-116** -#: precedence rule (a corpus re-find must not overwrite an ENRICHED measurement with the corpus's -#: cheaper approximation) and the Instagram one, which is the one with paying rows in it, does not. -#: -#: ⛔ **THE COLLAPSE IS BEHAVIOUR-PRESERVING, DELIBERATELY, AND `discovery_keys` IS WHERE YOU CAN -#: SEE THAT DECISION RATHER THAN INFER IT.** Setting Instagram's to `IG_DISCOVERY_KEYS` would close -#: D-116 in one line — and it would change what a live, billed, nightly automation writes to rows -#: the owner reads, inside a refactor whose whole claim is that nothing moved. It is REPORTED -#: instead (see D-116's row), which is the standing rule for a limit that is not being removed -#: today. ⚠ The one-line fix is now visible in a table instead of buried 300 lines apart, which is -#: most of the value of doing this at all. -#: -#: ⚠ `dataset` IS A CALLABLE, not a string: `connectors_tt` imports this module at import time, so -#: the engine may only reach it from INSIDE a function. `mapper` has the same shape for the same -#: reason on the TikTok side. -#: ⚠ `handle_field` is the column the engine-added `not_in` exclusion names, and it is per platform -#: because B-20 MEASURED that Bright Data's TikTok Profiles dataset calls Instagram's `account` -#: `account_id`. The exclusion never appears as a condition row, so nothing a person types can -#: correct it. -DISCOVERY_SPECS = { - "discover_instagram": { - "noun": "profile", - "log_tag": "discover", - "platform": PLATFORM_INSTAGRAM, - "dataset": lambda: BD_DS_PROFILES, - "handle_field": "account", - "mapper": lambda row, stamp: _candidate_row(row, stamp), - "table": DISCOVER_TABLE, - "label": DISCOVER_LABEL, - "fields": CANDIDATE_FIELDS, - # ⛔ D-116 IS OPEN ON THIS SIDE. `None` = discovery overwrites whatever it re-finds. - "discovery_keys": None, - "on_write": _ig_discovery_series, - "after_write": _ig_discovery_metrics, - }, - "discover_tiktok": { - "noun": "TikTok profile", - "log_tag": "discover-tt", - "platform": PLATFORM_TIKTOK, - "dataset": lambda: _tt_module().TT_DS_PROFILES, - "handle_field": "account_id", - "mapper": lambda row, stamp: _tt_candidate_row(row, stamp), - "table": TT_PROFILE_TABLE, - "label": TT_TABLE_LABELS[TT_PROFILE_TABLE], - "fields": TT_PROFILE_FIELDS, - "discovery_keys": TT_DISCOVERY_KEYS, - # ⚠ NO series and NO metric fields on this side, and that is not an oversight: both hooks - # write into Instagram's own snapshot store, through Instagram's own vocabulary. TikTok's - # series is W29-T06's `ut_tt_post_snapshots`, which the ENRICH path owns. - "on_write": None, - "after_write": None, - }, -} - - -def _tt_module(): - """`connectors_tt`, imported LAZILY. It imports this module at module level, so a top-level - import here would close the cycle.""" - import connectors_tt as _tt - return _tt - - -def run_discovery(rt, defn, username="automation", log=print, step=_no_step, rows=None): - """Automation #3 (owner ruling R7 / D-23) and DEBT D-9: FIND handles nobody here has typed in, - on whichever corpus this automation's KIND names. - - Two-phase by construction — see the DISCOVERY section header. A run either STARTS a corpus - query and hands its snapshot id to the next run, or COLLECTS one a previous run started. Every - outcome is a sentence about what actually happened; none of them is a green zero. - - ⛔ THE PLATFORM COMES FROM `defn["kind"]`, WHICH IS THE ONLY THING THAT DECIDES IT NOW. Before - wave 30 the choice was made by WHICH FUNCTION `RUNNERS` pointed at, so a fixture (or a stored - definition) with a stale kind still ran the corpus its caller had in mind. It cannot any more: - an unknown kind is refused rather than defaulted, because defaulting here means billing one - platform's corpus for another platform's search. - """ - kind = str(defn.get("kind") or "") - spec = DISCOVERY_SPECS.get(kind) - cfg = defn.get("config") or {} - dry = bool(cfg.get("dryRun")) - limit = int(cfg.get("recordsLimit") or 5) - preds = list(cfg.get("predicates") or []) - auto_id = str(defn.get("id") or "") - steps = {"trigger": "ok", "find": "idle", "collect": "idle", "write": "idle"} - counts = {"asked": limit, "found": 0, "new": 0, "seen_again": 0, "capped": 0, - "missing_tables": 0} - if spec is None: - # ⛔ NEVER A DEFAULT. `RUNNERS` maps exactly the kinds in `DISCOVERY_SPECS` onto this - # function, so reaching here means somebody widened one of the two without the other. - steps["find"] = "blocked" - return ("error", f"this automation's kind ({kind or 'blank'}) has no profile corpus, so " - f"nothing was searched and nothing was charged", counts, [], steps) - noun = spec["noun"] - - if not bd_ready(): - # ⛔ THE FAIL-CLOSED PATH (C4). Discovery has NO anonymous rung to drop to — the corpus is - # the vendor's — so this is where the run stops, saying which env var is missing. - steps["find"] = "blocked" - return ("partial", "Profile search is not set up yet. Nothing was searched and nothing " - "was charged", counts, [], steps) - - pending = str((defn.get("state") or {}).get("pendingSnapshot") or "") - if not pending: - # C4 (wave 22): the guard holds at RUN too — a stored config can predate the law, and - # the vendor bills for breadth whether the filter was saved yesterday or last month. - # D-68: the SHAPE guard rides beside the breadth guard, at save AND at run - a config - # stored before this check existed must not reach the vendor either. - # ⭐ W32-T46: the RUN-time twin of the save-time guard says the right network's name too — - # `defn["kind"]` is what decides the corpus everywhere else in this function. - guard_err = (narrowing_refusal(preds, cfg.get("operator"), str(defn.get("kind") or "")) - or depth_refusal(preds, cfg.get("operator"))) - if guard_err: - steps["find"] = "blocked" - return ("error", f"the search was not started. {guard_err}", counts, [], steps) - if dry: - est = discover_estimate(limit) - steps["find"] = steps["write"] = "skipped" - return ("partial", - f"Test run. Nothing saved. Would look for up to {limit} " - f"{noun}{'' if limit == 1 else 's'} matching " - f"{_predicate_sentence(preds, cfg.get('operator'))}, for about " - f"${est['usd']}", counts, [], steps) - step("Starting the search") - # ⭐ ITEM 7 — DO NOT PAY FOR WHAT WE ALREADY HAVE. The exclusion is engine-added: it never - # appears as a condition row, and `bd_filter_start` drops it rather than risk a shape the - # vendor refuses. Reported either way, because "why did this run find fewer" and "why did - # this run cost the same as last night" are both questions this line answers. - # ⚠ `already_found_handles` NEEDS NO PER-PLATFORM BRANCH — it is scoped to THIS definition's - # own tables (`automation_tables`), so a TikTok automation targeting `ut_tt_profile` - # excludes exactly the handles it has itself found. Verified rather than assumed: the - # function reads `config.targetTable` plus each `create_record` action's table, and names - # no IG constant. - excl = {} - sid, note = bd_filter_start(preds, cfg.get("operator") or "and", limit, - dataset_id=spec["dataset"](), - exclude_handles=already_found_handles(rt, defn), - applied=excl, handle_field=spec["handle_field"]) - if excl.get("excluded"): - step(f"excluded {excl['excluded']} already-found " - f"profile{'' if excl['excluded'] == 1 else 's'} at the provider") - elif excl.get("dropped"): - step(f"searching without the already-found list. {excl['dropped']}") - if note: - steps["find"] = "blocked" - return ("error", f"the search was not started. {note}", counts, [], steps) - pending = sid - # ⛔ LOG THE ID *BEFORE* PERSISTING IT, and the order is the whole point. `set_state` goes - # through `rt.update`, which can RAISE on an unavailable store — and that exception - # becomes an error run, so a `log()` placed after it never executes. The snapshot would - # then exist, be billed, and have its id in neither the store nor the logs. Logging first - # means the worst case is still recoverable by a human reading the Space output. - log(f"[aios-auto] {spec['log_tag']} {auto_id}: started {sid}") - # PERSIST BEFORE POLLING. A snapshot the vendor is already building does not stop - # existing because this process dies thirty seconds later, and a lost id is a set we - # paid attention to and can never collect. - set_state(rt, auto_id, {"pendingSnapshot": sid, "pendingSince": _iso()}) - steps["find"] = "ok" - - waited, status, size, note = 0.0, "", 0, "" - while True: - # ⭐ ITEM 6 — THE ONE PLACE THE LIVE STEP HAS TO MOVE. This loop blocks for up to - # BD_FILTER_WAIT at the vendor, and D measured that this wait IS the whole of "Run once now - # is laggy / looks stuck". A counter here is what makes a legitimate wait distinguishable - # from a hung thread; without it the rendered step reads the same word for both, which is a - # progress indicator that cannot indicate progress. - step(f"Searching. {int(waited)}s of " - f"{int(BD_FILTER_WAIT)}s") - status, size, note = bd_filter_status(pending) - if note: - steps["collect"] = "error" - set_state(rt, auto_id, {"pendingSnapshot": None, "pendingSince": None}) - return ("error", note, counts, [], steps) - if status != "building" or waited >= BD_FILTER_WAIT: - break - time.sleep(BD_FILTER_POLL) - waited += BD_FILTER_POLL - step("Reading the profiles the search found") - - if status == "building": - # ⚠ THE HANDOFF, AND IT IS A SUCCESSFUL OUTCOME OF A SORT. Measured build latency is ~20 - # minutes; holding a worker thread that long on a free-tier container to poll would be - # the wrong shape. Say plainly that it is still running and who collects it. - # ⚠ THE SENTENCE NAMES WHO COLLECTS IT, because the old one did not and that is what made - # this read as "stuck": *"The next run picks up the results"* is a promise about a run - # that, on a MANUAL automation, nobody had scheduled. `pending_collect_ids` makes the tick - # finish it, so the promise is now kept by something rather than by the reader. - # ⭐ WAVE 27 ITEM 14 (owner) — THE BRAG IS GONE. The sentence used to explain the wait by - # naming the corpus size. It was TRUE, and it was answering a question the person had not - # asked. It was also the one place a vendor's catalogue size was quoted to a customer, so - # it would have gone stale the day the vendor grew. THE SHAPE OF THE ANSWER SURVIVES and is - # the part that mattered: the wait does not shrink when you ask for fewer records, and a - # person who does not know that reads "20 minutes for 10 rows" as a fault. - mins = 0 - try: - since = (defn.get("state") or {}).get("pendingSince") - if since: - mins = max(0, int((_dt.datetime.now(_dt.timezone.utc) - - _dt.datetime.fromisoformat(str(since))).total_seconds() - // 60)) - except Exception: # noqa: BLE001 - mins = 0 - been = f". {mins} min so far" if mins else "" - steps["collect"] = "partial" - return ("partial", - f"Still searching at the provider{been}. A corpus search takes about 20 minutes " - f"however few records you asked for. Nothing is lost, you are not charged twice, " - f"and the results are collected automatically as soon as they are ready", - counts, [], steps) - if status == "empty": - # A search that matched nothing RAN CORRECTLY. It is the ordinary result of a keyword - # that is too specific, so it says what to do about it instead of reporting a fault. - steps["collect"] = "ok" - steps["write"] = "skipped" - set_state(rt, auto_id, {"pendingSnapshot": None, "pendingSince": None}) - return ("partial", "No profiles matched. Try a shorter or more common keyword. " - "'floral' finds more accounts than 'floral design studio'", - counts, [], steps) - if status != "ready": - steps["collect"] = "error" - set_state(rt, auto_id, {"pendingSnapshot": None, "pendingSince": None}) - return ("error", f"the search ended as {status or 'unreadable'} and returned nothing", - counts, [], steps) - - rows, dnote = bd_filter_rows(pending) - if dnote: - # Delivery lags `ready` by minutes (measured). Keep the id; the next run collects it. - steps["collect"] = "partial" - return ("partial", f"the search finished with {size} match" - f"{'' if size == 1 else 'es'} but {dnote}", counts, [], steps) - set_state(rt, auto_id, {"pendingSnapshot": None, "pendingSince": None}) - steps["collect"] = "ok" - - stamp = _iso() - _map = spec["mapper"] - incoming = [c for c in (_map(r, stamp) for r in rows) if c] - counts["found"] = len(incoming) - # ⚠ COUNT THE ROWS WE COULD NOT USE. A corpus row with no handle cannot be a candidate, and - # dropping it silently means "the vendor delivered 50 and we kept 3" reads identically to - # "the vendor delivered 3". It also catches the shape where an error document comes back - # down the rows path and parses as one unusable row instead of an error. - counts["dropped"] = max(0, len(rows) - len(incoming)) - label = cfg.get("targetLabel") or spec["label"] - table_key = (ut_key_for(label, cfg.get("targetTable") or spec["table"]) if dry - else ut_ensure(rt, label, spec["fields"], username, - key=cfg.get("targetTable") or spec["table"], - flow_tag=auto_id, lock_fields=True)) - existing = dict((ut_get(rt, table_key) or {}).get("rows") or {}) - # --- ⭐ WAVE 26 · C3 / owner ruling R4 — THE UPSERT KEY IS `(platform, handle)`. - # - # ⛔ THIS RETIRES WAVE 22's C6 COMPOUND `(handle, created_by)`, deliberately, and the reasoning - # that built it is worth stating before it is discarded rather than after: per-user rows gave - # each person their own `found_count` and their own review card, so two colleagues scouting the - # same market never edited each other's work. Owner, 2026-08-06: one profile is ONE row. The - # tenant is the unit, not the user. `created_by` survives as "Found by" — informational, and no - # longer part of the identity. - # - # ⚠ AND THE OWNER ADDED THE GUARD THAT MAKES IT SAFE, WHICH IS THE HALF THAT WOULD HAVE BEEN - # MISSED: a handle is only unique WITHIN a network. `@inayma` on Instagram and `@inayma` on - # TikTok are routinely different people, so keying on the handle ALONE would have silently - # merged two accounts into one row the day the TikTok runner ships (D-9) — a data-loss bug with - # no error, discovered months later by someone wondering why a creator's followers halved. - # Hence `platform`, and hence it being part of the key rather than a label beside it. - _ck = candidate_key - - # ⚠ A ROW WITH NO `platform` CELL IS STAMPED WITH **THIS CORPUS'S** PLATFORM, and the default is - # per spec rather than global. On the Instagram table every legacy row came from the Instagram - # runner, so `Instagram` is a FACT about the stored data; in a `ut_tt_*` table the same blank - # means TikTok for exactly the same reason. Blank-keyed rows would fail to match their own - # re-find and duplicate the whole table on the next run. - for r in existing.values(): - if not str(r.get("platform") or "").strip(): - r["platform"] = spec["platform"] - existing2 = {rid: {**r, "_ckey": _ck(r.get("platform"), r.get("handle"))} - for rid, r in existing.items()} - # `found_count` is ARITHMETIC OVER WHAT IS STORED, not a value from the vendor — re-finding - # a profile is the signal that it keeps matching, so the counter grows instead of - # resetting. `first_found` is written only when the pair is new, so it never moves. - seen = {r["_ckey"]: r for r in existing2.values()} - _dkeys = spec["discovery_keys"] - for c in incoming: - # "Found by" — the first finder is recorded and later finders do not overwrite them, which - # is what the column means now that it is no longer part of the identity. - c["created_by"] = username - c["_ckey"] = _ck(c.get("platform"), c["handle"]) - prev_row = seen.get(c["_ckey"]) - if prev_row and str(prev_row.get("created_by") or "").strip(): - c["created_by"] = prev_row["created_by"] - if prev_row: - counts["seen_again"] += 1 - c["found_count"] = str((_ig_int(prev_row.get("found_count")) or 0) + 1) - # ⛔⛔ DEBT D-116 — THE ENRICHED-MEASUREMENT PRECEDENCE RULE, AND IT RUNS ONLY WHERE THE - # SPEC DECLARES A `discovery_keys` SET. A discovery re-find OVERWRITES an enriched - # `followers` with the corpus number — the corpus is cheaper, rounder and older than a - # scrape, so the row silently gets WORSE every night while the automation reports - # success. The fix is a precedence rule, not a blank check: a row that has been ENRICHED - # (`enriched_at` is stamped only by the enrich path) keeps its exact measurements, and - # discovery may only fill what is genuinely empty and update its own bookkeeping. - # ⚠ The declared set is what discovery OWNS; everything else on a corpus row is a - # measurement, and an approximation must never replace an exact one. - if _dkeys and str(prev_row.get("enriched_at") or "").strip(): - for k in list(c): - if (k not in _dkeys and k != "_ckey" - and str(prev_row.get(k) or "").strip()): - c.pop(k) - else: - counts["new"] += 1 - c["found_count"] = "1" - c["first_found"] = stamp - # ⭐ WAVE 26 · ITEM 1 — THE STAGE STAMP IS PER-AUTOMATION, SO IT CANNOT HANG OFF - # "is this row new to the TABLE". - # - # ⛔ MEASURED DEFECT, two discovery automations pointed at one database: the second one's - # board was EMPTY. It reported "2 profiles found — 0 new, 2 seen before", which is true, - # and then drew nothing, because this stamp lived inside the `else` above. `skey` is - # `stage_`; the branch it sat in asks whether some OTHER automation had - # already created the row. So the first automation to reach a handle claimed it, and every - # later automation sharing that database silently had no cards — with a summary saying it - # had found them. - # - # ⚠ THE ORIGINAL LAW IS PRESERVED, and it is the reason this is a condition rather than an - # unconditional write: a re-find must never pull a card somebody has already moved back - # into Review. So the question is "has THIS automation placed this record yet?" — not "is - # this record new?" A blank stage cell for this automation means unplaced, which is - # exactly the state a new card is in. - merged, mc = upsert_rows(existing2, incoming, "_ckey", cap=row_cap(table_key)) - merged = {rid: {k: v for k, v in r.items() if k != "_ckey"} - for rid, r in merged.items()} - counts["capped"] = mc["capped"] - missing = [] if dry else ut_missing(rt, table_key) - counts["missing_tables"] = len(missing) - if not dry and not missing: - ut_write_rows(rt, table_key, merged) - steps["write"] = "ok" - if spec["on_write"]: - spec["on_write"](rt, table_key, incoming, stamp, username, auto_id, counts, log) - elif dry: - steps["write"] = "skipped" - - affected = [rid for rid, r in merged.items() - if str(r.get("handle")) in {c["handle"] for c in incoming}][:200] - est = discover_estimate(counts["found"]) - summary = (f"{counts['found']} {noun}{'' if counts['found'] == 1 else 's'} found. " - f"{counts['new']} new, {counts['seen_again']} seen before. About " - f"${est['usd']}") - if counts["dropped"]: - summary += (f". {counts['dropped']} result{'' if counts['dropped'] == 1 else 's'} had no " - f"username and could not be saved") - cnote = cap_note([(table_key, counts["capped"])], missing) - if cnote: - summary += f". {cnote}" - steps["write"] = "partial" - if dry: - summary = f"Test run. Nothing saved. Would have written: {summary}" - state = "partial" if (dry or counts["capped"] or missing or not counts["found"]) else "ok" - if not dry and not missing and spec["after_write"]: - spec["after_write"](rt, table_key, auto_id, log) - log(f"[aios-auto] {spec['log_tag']} {auto_id} -> {table_key}: {summary}") - return (state, summary, counts, affected, steps) - - -def _predicate_sentence(preds, operator="and"): - """A predicate list as something a person can read back. Used in summaries and the canvas. - - Human words on BOTH halves — it used to render `biography includes floral`, the vendor's - column name beside the vendor's operator token, in a sentence shown to a customer. - """ - joiner = " or " if str(operator or "").lower() == "or" else " and " - parts = [] - for p in preds or []: - v = p.get("value") - shown = " or ".join(str(x) for x in v) if isinstance(v, list) else v - parts.append(f"{field_label(p.get('name'))} " - f"{BD_OP_LABELS.get(p.get('operator'), p.get('operator'))}" - + ("" if shown is None else f" {shown}")) - return joiner.join(parts) or "no conditions" - - -#: How many records ONE `plain` run walks. Deliberately the ut table's OWN row ceiling rather -#: than a second, smaller number nobody could explain: a plain automation's records ARE its -#: table's rows. What this actually bounds is the APPEND tables (`MAX_UT_IG_ROWS`, 200k), which a -#: flow could be pointed at and which would not finish. -#: ⚠ W31-T36 (D-143 / R6) — THIS IS NOW THE FALLBACK ONLY: what a flow may walk when the table's -#: own limit cannot be resolved (no runtime, or a key `core.user_tables` does not know). The real -#: answer is per TABLE and comes from `core.user_tables.row_limit`; see `_flow_record_cap`. -#: ⛔ Do not re-point a caller at this constant — a private copy of "is this database bounded" is -#: exactly the second evaluator D-143 was. -MAX_FLOW_RECORDS = MAX_UT_ROWS - - -def _flow_record_cap(table_key, rt=None): - """How many records may a flow walk in this database? `None` means NO CAP (R6). - - ⛔ DELEGATES, NEVER DECIDES. `core.user_tables.row_limit` is the one evaluator for "is this - database bounded", and it distinguishes three states this module must not re-derive: `0` - (read-through — its rows are not in the document at all), `None` (connected and UNCAPPED, which - is R6's whole point), and `MAX_ROWS` (the editable substrate, genuinely bounded by the shared - document). A second copy here is exactly the defect D-143 IS. - - ⚠ `0` MEANS "NO ROWS LIVE HERE", NOT "WALK NOTHING". A read-through grid's rows arrive through - the mirror route, so a flow that reached this function already holds ids from somewhere else; - capping it at zero would refuse a walk whose records are in hand. Treated as uncapped, and the - honest place to fix a read-through walk is the walker. - """ - try: - import core.user_tables as _ut_cap - cap = _ut_cap.row_limit(str(table_key or ""), st=rt) - except Exception: # noqa: BLE001 - return MAX_FLOW_RECORDS # the fallback, and it is the conservative direction - return None if not cap else int(cap) - - -def _flow_cap_reason(table_key, rt=None): - """R6's SECOND sentence, appended to the run's own note: the cause and the recommended fix. - - ⛔ R6 IS TWO SENTENCES AND THE SECOND IS THE ONE THAT GETS DROPPED — *"if there is lag or it - can't be done, you need to explicitly tell me why and recommend a fix."* A run that says only - *"walked the first 5000"* has disclosed the number and hidden everything a person could act on. - `core.user_tables.limit_report` already carries both, so this READS them rather than writing a - second wording that would drift from the one the grid shows. - """ - try: - import core.user_tables as _ut_cap - rep = _ut_cap.limit_report(str(table_key or ""), st=rt) or {} - except Exception: # noqa: BLE001 - return "" - cause, fix = str(rep.get("cause") or ""), str(rep.get("recommendation") or "") - if not cause and not fix: - return "" - return f". {cause}." + (f" To walk them all: {fix}." if fix else "") - - -# --------------------------------------------------------------------------------------------- -# WAVE 35 · T36 / CONTRACT C8 / OWNER RULING R10 — THE STATEMENTS BATCH: ASSEMBLE, PARK, NEVER SEND -# -# ⛔⛔ THE TICKET'S `how:` SAID TO USE "the board's strict review posture (wave 22)". THAT POSTURE -# DOES NOT EXIST, and rebuilding it would REVERSE AN OWNER RULING rather than merely miss a symbol. -# Wave 27 R3 deleted `board()`, `move_card()`, `stages_for()`, `ensure_stage_field()` and the review -# branch of the action walk (the tombstone is above `LANE_OPS`), and its reason is a DATA reason, -# verbatim: *"the board wrote MACHINE COLUMNS INTO A TENANT'S OWN TABLE — a stage select, an `_at` -# stamp and a `_cycles` counter per automation — to render a view of state the RUN LOG already -# holds."* `retire_automation_stage_fields` and `migrate_ig_tables` still DROP those columns, so a -# batch parked in a stage field would be deleted by our own migration. -# ⇒ The batch parks on the AUTOMATION DEFINITION, the shape `runs` and `reviews` already use. Zero -# machine columns on a customer's database, and the migration has nothing to take away. -# -# ⛔ A STATEMENTS FLOW IS **BATCH**-SCOPED, NOT RECORD-SCOPED, and that is why it does not go -# through `apply_actions` at all. `run_plain` walks every row of the flow's bound table, so a -# per-record arm would assemble the whole worklist once per record; and a statements agent binds NO -# table (its customers come from the Odoo AR worklist, not a `ut_*` grid), so `run_plain` would -# have answered `partial: "no database is bound yet"` and the step would NEVER HAVE RUN. The -# precedent for a kind that defines its own scope is already here: the enrich-only branch below, -# whose saved View "ARE the set of records the automation was asked to process". -# ⚠ The `_walk` arm for this kind therefore stays a REFUSAL, not a duplicate: a `send_statement` -# dropped into an ordinary record-walking flow must say it did nothing, never half-send. -# -# ⛔⛔ AND NOTHING HERE SENDS. Not one function in this section imports the mail path. The send door -# is `routes_statements`, behind `admin_gate` + `_royal_only`, calling `collections_send. -# queue_statement`, whose SAFE_MODE guardrail lives in the DATA LAYER where no route, payload or UI -# can bypass it. This module ASSEMBLES and PARKS. A person clicks Send. - -#: How many statements one parked batch holds. Bounded for the reason `MAX_RUNS` and `MAX_REVIEWS` -#: are: this rides inside the automation DEFINITION, and an unbounded list is a serialisation cost -#: on every read of the automations bucket. -#: ⚠ WHEN IT BITES IT IS DISCLOSED, never a silent `[:N]` — R6's second sentence, and -#: [[no-unverifiable-aggregates]]. `assemble_statements` appends a note naming the number left out. -MAX_STATEMENT_BATCH = 200 - -#: The collection worklist loader, injectable so a gate can drive this without Odoo credentials. -#: Production leaves it None and the real data layer answers. Same shape as `_DRAFT_CHAT`. -_STATEMENT_ROWS = [None] - - -def _collections(): - """`modules.collections_send`, imported lazily — `platform/` is not on the path at import time - for every consumer of this module, and this is the only section that needs it.""" - import modules.collections_send as cs # noqa: PLC0415 - return cs - - -def statement_steps(defn): - """The ENABLED `send_statement` actions of this flow, top level only. - - ⚠ Top level only, deliberately, and `is_statement_flow` is what makes that safe: a batch flow - is exactly one enabled action. A `send_statement` nested inside an If is NOT a batch flow, so - it falls to the ordinary record walk and is refused there with a sentence. - """ - return [a for a in (((defn or {}).get("flow") or {}).get("actions") or []) - if isinstance(a, dict) and a.get("kind") == "send_statement" - and a.get("enabled", True)] - - -def is_statement_flow(defn): - """Is this automation a statements BATCH rather than a record walk? - - ⛔ EXACTLY ONE ENABLED ACTION, AND IT IS THIS KIND. The narrowness is the safety: a flow that - also updates records has record semantics that the batch path would silently drop, so it keeps - the ordinary walk (where the arm refuses and says so). This mirrors the enrich-only branch in - `run_plain`, which is narrow for the same stated reason. - """ - enabled = [a for a in (((defn or {}).get("flow") or {}).get("actions") or []) - if isinstance(a, dict) and a.get("enabled", True)] - return len(enabled) == 1 and enabled[0].get("kind") == "send_statement" - - -def assemble_statements(defn, log=print): - """`(batch, notes)` — the statements this configuration WOULD send, rendered, plus why anybody - was left out. **Reads Odoo read-only. Sends nothing. Writes nothing.** - - Each entry is `{customer, to, subject, html, tier, overdue}`. `html` is rendered by the SAME - `render_statement_html` the send door uses, so what a person approves is what goes out — a - review screen rendering its own approximation of the mail is a review of the wrong thing. - - ⚠ A CUSTOMER WITH NO EMAIL IS *SKIPPED AND NAMED*, never dropped. `routes_statements.send` - already separates "we could not" from "there was nowhere to send"; this keeps that distinction - at the assembly end, because a batch that silently shrinks is one nobody can reconcile. - """ - cfg = (statement_steps(defn) or [{}])[0].get("config") or {} - cs = _collections() - notes = [] - loader = _STATEMENT_ROWS[0] - rows = list(loader() if loader else cs.load_collection_list(cs.Odoo())) - tier = str(cfg.get("tier") or "").strip() - if tier: - before = len(rows) - rows = [r for r in rows if str(r.get("Tier") or "") == tier] - log(f"[aios-auto] statements: {len(rows)} of {before} customers are in tier {tier}") - subject_tpl = str(cfg.get("subject") or "") or cs.DEFAULT_SUBJECT - intro_tpl = str(cfg.get("intro") or "") or cs.DEFAULT_INTRO - footer_tpl = str(cfg.get("footer") or "") or cs.DEFAULT_FOOTER - month = _dt.date.today().strftime("%B %Y") - batch, no_email = [], [] - for row in rows: - if len(batch) >= MAX_STATEMENT_BATCH: - break - name = str(row.get("Customer") or "") - to = str(row.get("Email") or "").strip() - if not to: - no_email.append(name) - continue - try: - subject = subject_tpl.format(customer=name, company=cs.COMPANY, month=month) - except (KeyError, IndexError): - # An unknown placeholder is the person's typo, not a crash. Show what they typed — - # the same choice `routes_statements.preview` already makes. - subject = subject_tpl - batch.append({"customer": name, "to": to, "subject": _s(subject, 200), - "html": _s(cs.render_statement_html(row, intro_tpl, footer_tpl), 40000), - "tier": str(row.get("Tier") or ""), "overdue": row.get("Overdue")}) - if no_email: - shown = ", ".join(no_email[:5]) - one = len(no_email) == 1 - notes.append(f"{len(no_email)} customer{'' if one else 's'} " - f"{'has' if one else 'have'} no email address on their record and " - f"{'is' if one else 'are'} not in this batch " - f"({shown}{', and others' if len(no_email) > 5 else ''}). " - f"Add an address in Odoo and run it again") - left = len(rows) - len(batch) - len(no_email) - if left > 0: - # R6's second sentence: a limit that bites is REPORTED, with its cause and what to do. - notes.append(f"{left} more customers matched but one batch holds " - f"{MAX_STATEMENT_BATCH}. Send this batch, then run it again for the rest, or " - f"narrow the tier filter") - return batch, notes - - -def park_statements(rt, auto_id, batch, notes): - """Write the assembled batch onto the DEFINITION as `pendingStatements`. Replaces, never - appends: a batch is THIS run's worklist, and two runs' statements merged into one list is a - customer invoiced twice. - - ⚠ `flush="async"` for the reason every writer in this module gives — the read-your-writes - contract means the next `GET /automations` already sees it, and only the upload is deferred. - """ - entry = {"ts": _iso(), "count": len(batch), "sent": False, - "items": list(batch)[:MAX_STATEMENT_BATCH], - "notes": [_s(n, 300) for n in (notes or [])][:25]} - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - d = cur.get(str(auto_id)) - if d is not None: - d["pendingStatements"] = entry - return cur - - _store_update(rt, _up, flush="async") - return entry - - -def clear_statements(rt, auto_id): - """Drop a parked batch — after it is sent, or when somebody discards it.""" - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - d = cur.get(str(auto_id)) - if d is not None: - d.pop("pendingStatements", None) - return cur - - _store_update(rt, _up, flush="async") - - -def run_statements(rt, defn, username="automation", log=print, step=_no_step): - """The batch runner: assemble, park, tell somebody. Returns `run_plain`'s 5-tuple. - - ⛔ THE STATE IS `partial`, NEVER `ok`, AND THAT IS DELIBERATE. `ok` would put a green dot over - a job that is only half done — the statements exist and NOBODY HAS SENT THEM. `partial` is this - module's honest-progress state and the summary says exactly what is waiting, which is what makes - the run notification (`_notify_run`) read as "come and look" rather than "done". - """ - if not is_statement_tenant(rt): - # Belt and braces behind `clean_actions`' wall: a definition stored before the wall, or - # copied between tenants, must not assemble another company's customer list. - return ("error", "statements are not configured for this workspace", {}, [], {}) - step("reading the collection list") - try: - batch, notes = assemble_statements(defn, log=log) - except Exception as exc: # noqa: BLE001 - return ("error", f"the collection list could not be read: {str(exc)[:200]}", {}, [], {}) - step("rendering statements") - park_statements(rt, defn.get("id"), batch, notes) - counts = {"statements": len(batch)} - if not batch: - return ("partial", "no customers matched, so there is nothing to review", counts, [], {}) - one = len(batch) == 1 - return ("partial", - f"{len(batch)} statement{'' if one else 's'} " - f"{'is' if one else 'are'} ready for review. Nothing has been sent. Open this agent " - f"and click Send to release them", - counts, [], {}) - - -#: ⭐⭐ WAVE 35 · T37 — THE SEEDED ROYAL IMPORTS STATEMENTS AGENT. -#: A FIXED id, because the row's own EXISTENCE is the idempotency key (see `seed_statements_agent`). -STATEMENTS_AGENT_ID = "system:statements" -SYSTEM_STATEMENTS = "statements" -#: First of the month, 06:00. A monthly statement run is what the collections worklist is for, and -#: the hour matches the Odoo sync default so the numbers it reads are the night's. -STATEMENTS_DEFAULT_CRON = "0 6 1 * *" - - -def seed_statements_agent(rt, username="system"): - """Mint Royal Imports' "Monthly statements" agent ONCE, **switched OFF**. Returns the - definition if it wrote one, else None. - - ⛔⛔ IT ARRIVES `enabled: False` AND THAT IS THE TICKET'S OWN TRAP, not a preference. An agent - that ships enabled has scheduled itself against real customers before a single person has read - its configuration — and this one's action queues mail to the tenant's actual debtors. - - ⭐ WHY STORED, WHEN WAVE 34's SYSTEM AGENT IS DERIVED. `_odoo_sync_row` gives three reasons for - deriving and only one survives contact with THIS agent (raised as `ASK D-11`): - · "two copies of one cadence" — does not apply: Odoo's cadence already lives in the connector - config, so a stored copy could drift; a statements schedule has no other home to drift from. - · "stored means the failure-pause can disable it" — INVERTS: auto-pausing statements after K - consecutive failures is correct (a dead credential should stop it), where pausing the Odoo - cadence would silently break infrastructure. - · "nothing can seed it" — real, and answered here: ONE write per tenant ever, by the - CONTAINER (which is what D-195 asks for; what it forbids is a CLI writing while the Space - is up), behind the existence check below. - And deriving cannot meet T37 anyway: a derived row is not in the automations bucket, so edit, - toggle, schedule and run would each need a bespoke door — four new mechanisms to avoid one - guarded write. The Odoo agent escapes that only because a separate resync loop already does its - work. - - ⛔ THE EXISTENCE CHECK IS THE WHOLE IDEMPOTENCY STORY, AND IT IS SAFE ONLY BECAUSE THE ROW - CANNOT BE DELETED. `routes_automation.delete_automation` refuses any stored row carrying - `system` (409, with a sentence), so "the row is present" can never become false behind our - back. If that refusal is ever relaxed, THIS FUNCTION NEEDS A SEPARATE DURABLE FLAG — otherwise - deleting the agent resurrects it on the next list call, which is a delete that undoes itself. - ⚠ And `system` is STICKY through `clean_definition` (see its return) for the same reason: an - edit that stripped the marker would make the row deletable and re-open exactly that hole. - """ - if not is_statement_tenant(rt): - return None - existing = all_definitions(rt) or {} - if STATEMENTS_AGENT_ID in existing: - return None - raw = { - "name": "Monthly statements", - "kind": "plain", - # ⚠ A SCHEDULE THAT IS OFF, not an absent schedule: the cron is the CONFIGURATION T37 asks - # a person to open and read, and a blank one would make them invent it before they could - # judge it. - "schedule": {"cron": STATEMENTS_DEFAULT_CRON, "enabled": False}, - "trigger": {"key": "schedule"}, - "flow": {"actions": [{ - "id": "act_1", "kind": "send_statement", - # ⚠ TIER BLANK = every tier, which is the honest default: narrowing to "A-Urgent" would - # be us deciding who gets chased, and the blank is the value the config panel shows as - # "all of them" rather than an empty box. - "config": {"tier": "", "subject": "", "intro": "", "footer": ""}, - }]}, - } - defn, err = clean_definition(raw, None, username, rt=rt) - if err: - return None - defn["id"] = STATEMENTS_AGENT_ID - # Stamped AFTER the clean, because `clean_definition` reads this key from `prev` only — a - # payload may never assert it (see that function's note). - defn["system"] = SYSTEM_STATEMENTS - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - # ⚠ RE-CHECKED INSIDE THE MUTATION. Two concurrent list calls can both pass the read above; - # the store applies mutations under its own lock, so this is where "only once" is actually - # decided. Without it the second writer would overwrite a row a person may already have - # edited, silently resetting their schedule and template. - if STATEMENTS_AGENT_ID not in cur: - cur[STATEMENTS_AGENT_ID] = defn - return cur - - _store_update(rt, _up, flush="async") - return defn - - -def run_plain(rt, defn, username="automation", log=print, step=_no_step, rows=None): - """⭐ WAVE 24 / R6 — the runner for an automation with NO machine step: its flow IS the whole - automation. Returns the same 5-tuple every other runner does, so `run_now` needs no special - case and `apply_actions` still runs at the ONE call site that already exists. - - Before this existed, `RUNNERS.get("plain")` was None and pressing Run now committed - `error: unknown automation kind 'plain'` — on the kind every new automation now has. - - ⛔ AN UNBOUND FLOW ANSWERS `partial`, NEVER `ok`, and that is the load-bearing line here. - `apply_actions` returns immediately when there is no table, so a flow whose only action is - `create_record` into some other database does NOTHING — and a green dot over nothing is the - exact defect this module names in three other places. It reports the honest state and says - which fact is missing. - """ - # ⭐⭐ WAVE 35 · T36 — THE BATCH FLOW BRANCHES BEFORE THE TABLE CHECK, and the ORDER is the - # whole of it. A statements agent binds no `ut_*` table (its customers come from the Odoo AR - # worklist), so leaving this below the `if not table` return would answer "no database is bound - # yet" and the step would never run — a feature that is whole, gated and unreachable, which is - # this repo's most-repeated failure. See the section above `run_plain` for why it is not an - # `apply_actions` arm. - if is_statement_flow(defn): - return run_statements(rt, defn, username=username, log=log, step=step) - table = _flow_table(defn) - if not table: - return ("partial", - "no database is bound yet. A plain automation walks the records of its target " - "database, and this one names none (pick one on the trigger, or in Properties)", - {}, [], {}) - all_rows = (ut_get(rt, table) or {}).get("rows") or {} - # ⛔ THE TRIGGER'S RECORDS WIN OVER THE WHOLE TABLE, and getting this wrong is a day-one bug - # on the kind every new automation now has. `plain` has no machine step, so it has nothing of - # its own to call "the records this run touched" — and walking the WHOLE table would mean - # "When a record is CREATED in ut_leads -> set status = New" writes `New` onto every lead the - # first time one is added. The trigger knows exactly which rows fired; `run_now` threads them - # here. `rows=None` (manual, schedule, Run now) still means the whole table, which is what - # those genuinely mean. - ids = ([r for r in (rows or []) if str(r) in all_rows] if rows - else sorted(all_rows, key=_rid_num)) - - # An enrichment-only flow is the one exception to the ordinary manual-run rule above. Its - # saved View and quota are not merely how the action decides whether to spend; they ARE the - # set of records the automation was asked to process. Walking every row first meant a run - # bound to a 10-record "Pending" view announced (and needlessly visited) all 61 table rows. - # Besides being misleading, that shape made the runtime scale with unrelated historical rows. - # - # Keep this deliberately narrow: any flow with another action must still hand that action the - # whole manual/scheduled scope. Only one direct, enabled enrich action has no other record - # semantics to preserve, so it can begin at its selected records safely. - flow_actions = list((defn.get("flow") or {}).get("actions") or []) - enabled_direct = [a for a in flow_actions if isinstance(a, dict) and a.get("enabled", True)] - enrich_scope_note = "" - if rows is None and len(enabled_direct) == 1 \ - and enabled_direct[0].get("kind") == "enrich_instagram": - enrich_cfg = enabled_direct[0].get("config") or {} - profile_key = profile_field_key(ut_get(rt, table), enrich_cfg.get("profileField")) - if profile_key: - # ⛔⛔ THE SAME QUESTION MUST GET THE SAME ANSWER IN BOTH PLACES. - # `enrich_selection` is called TWICE per run — here, to decide which records the - # runner walks and announces, and again inside `_walk` to decide which the action - # SPENDS on. MEASURED LIVE 2026-08-09: this call omitted the not-found verdicts, so - # the run announced *"(1 of the 30 asked for)"* in its summary while its own note - # from the second call said *"0 of the 30"* — one run, two answers, both printed. - # ⚠ `gone` is read from the definition here rather than passed in, because this - # function has the definition and `_walk` does the same read; a third source of the - # same fact is how the two would drift again ([[one-evaluator-per-question]]). - selected, enrich_scope_note = enrich_selection( - rt, table, enrich_cfg, profile_key, - gone=dict(((defn or {}).get("state") or {}).get("enrichNotFound") or {})) - # ⛔ D-112 — AND THE CONSTANT IS THE FIX, NOT THE `if`. This guard was a `startswith` - # against the sentence SPELLED OUT, twelve hundred lines from the only place that - # produces it. Reword the message in `enrich_selection` — a perfectly ordinary edit, - # since it is a sentence a person reads — and this line silently stops matching, the - # run goes back to reporting `ok`, and the defect D-112 describes returns with nothing - # anywhere going red [[gate-pins-a-spelling-not-a-claim]], on the product side. - # ONE constant, two readers, so the wording is free to change and the behaviour is not. - if enrich_scope_note.startswith(ENRICH_VIEW_UNREADABLE): - return ("partial", f"nothing walked in {table}. {enrich_scope_note}", {}, [], {}) - ids = selected - step(f"Walking {len(ids)} record{'' if len(ids) == 1 else 's'} in {table}") - counts = {"records": len(ids)} - # ⭐⭐ WAVE 31 · T36 (D-143, owner ruling R6) — THE CAP IS THE TABLE'S, NOT THIS MODULE'S. - # - # ⛔ WHAT WAS WRONG, and it was a SECOND EVALUATOR rather than a wrong number. This line read - # `MAX_FLOW_RECORDS = MAX_UT_ROWS = 5000` — a constant mirrored from the EDITABLE substrate's - # ceiling — and applied it to every database alike. `ut_odoo_orders` is a CONNECTED source - # (32,826 rows live), and R6 is explicit that connected-source data has NO CAP: *"I thought we - # decided there is no cap in how many data from the API source … can be pulled into the app."* - # So a flow over it silently walked 5,000 of 32,826 — 15% of the records — and every total it - # produced understated the book while looking exactly like a complete run. - # - # ⭐ `core.user_tables.row_limit` IS THE ONE EVALUATOR for this question and it already answers - # the three cases (0 = read-through, None = connected and UNCAPPED, MAX_ROWS = editable). Using - # it here retires this module's private copy instead of correcting it — one question, one - # normaliser [[one-question-two-normalizers]]. `limit_report` is R6's SECOND sentence already - # expressed as data, so the sentence a person reads carries the cause and the recommended fix - # rather than just a number. - _cap = _flow_record_cap(table, rt) - if _cap is not None and len(ids) > _cap: - # The cap is DISCLOSED, never silent ([[no-unverifiable-aggregates]]): the summary names - # both numbers so "it only processed some of them" is readable rather than deducible — and - # since T36, WHY it applies and what to do about it. - total = len(ids) - ids = ids[:_cap] - return ("partial", - f"{total} records in {table}. This run walked the first {_cap}" - + _flow_cap_reason(table, rt), counts, ids, {}) - n = len(ids) - # ⭐⭐ 2026-08-07 (owner report) — SAY IT IN WORDS A PERSON CAN CHECK. - # Owner: *"how come it says that its 51 records walked in ut_beauty_influencer_leads (all)? - # wth is even ut_beauty_influencer_leads (all)??"* — and both halves of that name were ours: - # · `ut_beauty_influencer_leads` is the storage KEY. Every other surface in the product - # shows the database's LABEL ("Beauty influencer leads"); this one leaked the key. - # · `(all)` meant "every record, because a manual run has no trigger records to narrow to", - # which is unguessable from the word. It reads as a filter nobody chose. - # ⚠ THE NUMBER WAS ALWAYS HONEST and is unchanged: a `plain` automation walks its whole - # database, and a per-STEP narrowing (an enrich step's `fromView`) is a different count that - # this sentence never claimed to report. What was wrong was that nothing said so. - label = str((ut_get(rt, table) or {}).get("label") or "").strip() or table - scope = ("the records that fired the trigger" if rows - else ("the records selected for Instagram enrichment" - if len(enabled_direct) == 1 - and enabled_direct[0].get("kind") == "enrich_instagram" - else "every record. This run was started by hand, so nothing narrowed it")) - if enrich_scope_note: - scope += f" ({enrich_scope_note})" - # ⭐⭐ AND ON THE RUN, not only inside the summary sentence. `summary` is capped at 400 - # characters by `_commit_run`, and this note is the longest thing a run says — it names - # every skipped handle and what to do about them. MEASURED: a run that selects NOTHING - # (every candidate is a known-dead handle) never enters `_walk`, so the walk's own copy - # of this sentence is never produced. Without this line the explanation exists only in a - # string that is about to be truncated, on exactly the runs that look like the automation - # has stopped working. - counts[RUN_NOTES_KEY] = [enrich_scope_note] - return ("ok", f"{n} record{'' if n == 1 else 's'} walked in {label}: {scope}", - counts, ids, {}) - - -RUNNERS = {"plain": run_plain, "scrape_db": run_scrape_db, - "field_instagram": run_field_instagram, - # ⭐ WAVE 30 · T09 (D-129) — BOTH DISCOVERY KINDS MAP ONTO THE SAME FUNCTION. The kind is - # no longer chosen by which callable this dict holds; `run_discovery` reads it off the - # definition and looks the corpus up in `DISCOVERY_SPECS`. The two dicts are asserted - # key-for-key by the gate, because a kind in one and not the other is either an - # unroutable automation or a refused run. - "discover_instagram": run_discovery, - "discover_tiktok": run_discovery} - - -# --------------------------------------------------------------------------------------------- -# THE SOURCE REGISTRY (DEBT D-9's seam, wave-20 item 6b) -# --------------------------------------------------------------------------------------------- -# WHY THIS EXISTS BEFORE THERE IS A SECOND SOURCE. The vendor swap that produced it (HikerAPI → -# Bright Data) touched a dozen places: a tier name, two node ids, a config flag, a readiness -# boolean on the wire, four client strings and two gate sections. That is what a hard-coded -# vendor costs, and TikTok (D-9) would have paid it again from scratch. -# -# So a SOURCE is declared once — what it is called, which vendor answers it, and which of the -# three verbs it can do — and the surfaces read the declaration instead of naming a vendor: -# -# probe "is this configured?" -> the honest readiness bit the UI shows -# capture enrich a handle we know -> `pull_profile`-shaped -# discover find handles we do not -> the corpus query -# -# ⚠ A SOURCE DECLARES ONLY WHAT IT CAN ACTUALLY DO. `discover: None` is not a gap to fill in -# later; it is the honest statement that this source has no discovery route, and a surface that -# offers one anyway is offering a button that must refuse. -SOURCES = { - "instagram": { - # ⛔ `vendor` NAMES NO COMPANY (owner instruction 2026-08-09). It reaches operator-facing - # copy, and which provider serves a capability is a routing decision the product owns — - # `providers.py` still holds the real keys, chains and costs. - "key": "instagram", "label": "Instagram", "vendor": "Scraper", - "probe": bd_ready, - "capture": pull_profile, - "discover": bd_filter_start, - "captureKind": "field_instagram", - "discoverKind": "discover_instagram", - "tiers": TIERS, - }, - # ⭐⭐ WAVE 29 — D-9 LANDS, AND IT LANDED AS THE ONE ENTRY THIS REGISTRY PREDICTED IT WOULD. - # The canvas, the append tables, the caps and the honest-status contract were already in place; - # what TikTok added was a field map, a discovery runner and three dataset ids. - "tiktok": { - # `vendor` NAMES NO COMPANY (owner instruction 2026-08-09) — it reaches operator-facing - # copy, and which provider serves a capability is a routing decision `providers.py` owns. - "key": "tiktok", "label": "TikTok", "vendor": "Scraper", - "probe": bd_ready, - # ⭐ WAVE 30 · T08 — CAPTURE IS WIRED, and it is a LAZY reference rather than the function - # object every other row here holds. `connectors_tt` imports THIS module at module level, - # so naming `connectors_tt.pull_profile_tt` at import time is a circular import — which is - # why every existing site does `import connectors_tt` inside the function body. A - # zero-argument callable keeps the registry's shape (a reader gets a callable, not a - # string) without moving the import. - # ⚠ AND SETTING IT SWITCHES NOTHING ON. `SOURCES` is never subscripted anywhere; its one - # reader is `source_status()`, which reads `key/label/vendor/probe/discover`. An action - # kind is gated in three other places entirely (`ACTION_CATALOG`, `clean_actions`, - # `apply_actions`). This row is documentation that must not lie, not wiring. - "capture": lambda *a, **k: __import__("connectors_tt").pull_profile_tt(*a, **k), - "discover": bd_filter_start, - "captureKind": "enrich_tiktok", - "discoverKind": "discover_tiktok", - # No tier ladder: R6/R7 retired the free rung from enrichment entirely, and TikTok never - # had one. An empty tuple is the honest statement, not a gap to fill in later. - "tiers": (), - }, -} - - -def source_status(): - """`[{key,label,vendor,ready,canDiscover}]` — what the surface says about each source. - - ⚠ A BOOLEAN, NEVER THE KEY (the wire rule the `hikerReady` bit already followed): a surface - needs exactly one bit to say "the paid rung is not configured" honestly, and shipping the - credential to a browser would put a billable secret in every user's devtools. - """ - # ⛔ `vendor` IS NOT ON THE WIRE (2026-08-06). It stays in `SOURCES` as the record of which - # supplier this rung actually uses — that history is worth keeping in the code — but a - # customer has no use for it, and a server-supplied string rendered by a generic component - # is precisely the half a client-file scan cannot see. - return [{"key": s["key"], "label": s["label"], - "ready": bool(s["probe"]()), "canDiscover": bool(s.get("discover"))} - for s in SOURCES.values()] - - -# --------------------------------------------------------------------------------------------- -# THE CANVAS GRAPH (R5) — topology on the SERVER, pixels on the client -# --------------------------------------------------------------------------------------------- -# WHY THE TOPOLOGY LIVES HERE. The canvas draws what an automation DOES; the runners above are -# what it does. Two descriptions of one thing in two languages drift, and the drift is invisible — -# a canvas that still shows a step the engine stopped running looks completely fine. So the node -# list is derived from the DEFINITION by the same module that executes it, rides on the payload -# the way `CRON_PRESETS` and the kind list already do, and is asserted in `verify_automation.py`. -# The client owns layout (pixels, pan/zoom, hit targets) and nothing else. -# -# ⚠ BRANCHES RENDER, THEY DO NOT EXECUTE (R5, accepted at grill; DEBT D-7). The `capture` node is -# a real branch in BEHAVIOUR — the paid rung answers or the anonymous ladder does — but the engine -# walks it linearly. Nothing here should be read as a general branching runtime. - -#: A node's dot vocabulary. Deliberately WIDER than the run-level `STATES`: a node can be -#: `blocked` or `skipped` in ways a whole run cannot, and folding those into `partial` would throw -#: away the only information the dot is there to carry. -NODE_STATES = ("idle", "ok", "partial", "error", "blocked", "skipped") - -#: node id -> the switch it flips. A node ABSENT from this map has no switch, and that is a -#: deliberate answer rather than an omission: a "Fetch the page" step that can be turned off is -#: not an automation with a disabled step, it is a broken automation with a lie on it. The ones -#: here are all REAL — each changes what the next run does: -#: schedule the trigger fires on its cron, or only by hand -#: postMetrics likes/comments per post are bought, or the engagement series does not grow — -#: it costs a vendor record PER POST rather than per profile (measured: the -#: profile row carries post identity and no engagement) -#: commentMetrics the Comments dataset is bought, which can bill many rows per post -#: write DRY RUN — read everything, compute the counts, write nothing anywhere -#: -#: ⭐⭐ WAVE 28 / R5+R6 (contract C3) — `paid` AND `fallback` ARE GONE, and what replaced them is -#: the point of the ruling: the money switches used to be "which rung do we try" (a question about -#: our plumbing), and they are now "what do you want captured" (a question about the user's data). -#: Profile is always captured and has no switch — an automation that fetches nothing is not an -#: automation with a step turned off, it is a broken one with a lie on it. -#: ⚠ `trigger: "schedule"` IS NOT PART OF THAT COLLAPSE. It is the trigger card's own on/off and -#: has its own branch in `toggle_node` (event triggers flip themselves, not the cron); C3's "the -#: three toggles" names the CAPTURE ladder it is reshaping. -NODE_TOGGLES = {"trigger": "schedule", "capture_posts": "postMetrics", - "capture_comments": "commentMetrics", "write": "write"} - - -def _cron_label(cron): - return next((p["label"] for p in CRON_PRESETS if p["cron"] == cron), cron or "") - - -def node_status(steps, nid): - """A node's dot — from the last run's MEASURED step outcome, or `idle`. - - ⛔ NEVER INFERRED FROM THE RUN'S OVERALL STATE, and this is a module-level function precisely - so a negative control can prove that. Every run stored before W19-C has no `steps` map at all, - and painting those nodes green because the run said `ok` would manufacture a measurement — the - same defect as the green dot over the empty posts table recorded in `run_field_instagram`. - No data ⇒ `idle`. An unrecognised word ⇒ `idle`, never passed through to the UI. - """ - v = str((steps or {}).get(nid) or "") - return v if v in NODE_STATES else "idle" - - -def graph(defn): - """One automation → `{nodes, edges}`, left to right. Pure over the definition.""" - defn = defn if isinstance(defn, dict) else {} - cfg = defn.get("config") or {} - sched = defn.get("schedule") or {} - runs = defn.get("runs") or [] - last = runs[0] if runs and isinstance(runs[0], dict) else {} - steps = last.get("steps") if isinstance(last.get("steps"), dict) else {} - - def node(nid, kind, title, subtitle, col, row=0, panel="", detail="", on=True): - return {"id": nid, "kind": kind, "title": title, "subtitle": subtitle, "detail": detail, - "col": col, "row": row, "panel": panel or nid, "enabled": bool(on), - "toggle": NODE_TOGGLES.get(nid, ""), "status": node_status(steps, nid)} - - on = bool(sched.get("enabled")) - trg = defn.get("trigger") or {} - if trg.get("key") in TRIGGER_KEYS and trg.get("key") not in ("manual", "schedule"): - # C3 (wave 22): the trigger node SAYS which trigger this flow has — the board and rail - # read the same node, so an event-triggered flow must not read "Schedule". The membership - # test is the KEY SET, not a hand-listed tuple: wave 23 added four triggers and the old - # tuple would have quietly rendered every one of them as "Schedule" (they are stored, so - # the else-branch was reachable) — green, compiling, and wrong on screen. - t_on = bool(trg.get("enabled", True)) and not trg.get("paused") - sub = TRIGGER_LABELS.get(trg["key"], trg["key"]) - watched = ", ".join(trg.get("fields") or []) or "any field" - # ⭐ WAVE 24 · laws 4/5 — `event_field` no longer names a FIELD (its condition is the - # whole of it) and `record_updated` no longer names a condition (its watched fields are). - # These two lines are the migration's visible half: a node still describing a `.field` - # the validator has stopped storing would be the surface and the store disagreeing. - # ⚠ W34-T47 — A COLON, NOT A FULL STOP. R6's sweep turned these four em dashes into - # sentence breaks, which is right for a REFUSAL and wrong here: these are compact node - # SUBTITLES ("ut_leads: view v1"), not prose, and a full stop mid-label reads as two - # truncated fields rather than one qualified name. The rule the sweep encodes is "a - # sentence that used a dash usually wants a period"; a LABEL usually wants a colon. - det = {"event_field": (f"{trg.get('table', '')}" - + (f": {_lane_sentence(trg.get('when'))}" - if trg.get("when") else "")), - "record_updated": f"{trg.get('table', '')}: {watched}", - "record_created": str(trg.get("table") or ""), - "enters_view": f"{trg.get('table', '')}: view {trg.get('viewId') or 'unset'}", - "form_submitted": str(trg.get("table") or ""), - "webhook": "POST the hook URL to fire it", - # ⭐ WAVE 30 · T05 — BOTH corpus triggers, and the value is identical because the - # question is: a discovery trigger's detail line IS its filter. Without the TikTok - # key this fell to `.get(..., "")` and the trigger node rendered with a blank - # subtitle, which reads as "nothing configured" on a fully configured automation. - "ig_profile_match": (_predicate_sentence(cfg.get("predicates"), - cfg.get("operator"))[:80] - if cfg.get("predicates") - else "No filters yet. Nothing to search for"), - "tiktok_profile_match": (_predicate_sentence(cfg.get("predicates"), - cfg.get("operator"))[:80] - if cfg.get("predicates") - else "No filters yet. Nothing to search for"), - "email": str(trg.get("query") or "")}.get(trg["key"], "") - if not trg.get("configured", True): - det = "Finish setting this trigger up before it can fire" - if trg.get("paused"): - det = str(defn.get("statusNote") or "paused") - nodes = [node("trigger", "trigger", sub, - "On" if t_on else "Off", 0, panel="trigger", on=t_on, detail=det)] - else: - # ⭐ 2026-08-07 (owner ruling) — THE CARD NAMES WHAT ACTUALLY FIRES IT. With the cron off - # this node is a MANUAL trigger, and titling it "Schedule" was the screen disagreeing with - # the user's own pick — the same class of defect as the Database picker that could not be - # reached. ⚠ The SUBTITLE is untouched ("Manual only" / the cron label): it is what the - # disabled-schedule gate asserts, and it was never the wrong half. - # ⚠ `panel="schedule"` STAYS whichever way it reads. Picking Manual says how this fires - # TODAY, not that it may never be scheduled — the cron has to stay one click away, and - # this is the only node that offers it. - nodes = [node("trigger", "trigger", "Schedule" if on else "Manual", - _cron_label(sched.get("cron")) if on else "Manual only", 0, - panel="schedule", on=on, - detail="" if on else "It still runs when you press Run now")] - edges = [] - - if defn.get("kind") == "field_instagram": - # ⭐⭐ WAVE 28 / CONTRACT C3 — FOUR NODES, THREE OF THEM SWITCHES OVER WHAT IS CAPTURED. - # This branch used to draw a FORK: `Capture` splitting into `Exact counts` (paid) and - # `Estimated counts` (the free anonymous ladder), rejoining at `Post engagement`. R5 - # deleted the ladder, so the fork had one arm; keeping it would have drawn a decision the - # engine no longer makes, with a switch (`fallback`) flipping a config key the cleaners - # now discard. A canvas that offers a choice the runtime ignores is worse than no canvas. - # ⚠ `Profile set` STILL CARRIES NO SWITCH, and now that is the whole ruling rather than an - # implementation detail: the profile is always captured (R6), Posts and Comments are the - # opt-ins, and both are OFF until asked for. - metrics_on = bool(cfg.get("postMetrics")) - comments_on = bool(cfg.get("commentMetrics")) - max_posts = cfg.get("maxPosts") or DEFAULT_POSTS_PER_PULL - nodes += [ - node("source", "source", "Profile set", cfg.get("targetTable") or "No database", 1, - panel="source", - detail=f"URL column: {cfg.get('urlField')}" if cfg.get("urlField") - else "URL column: the one the field is bound to"), - # ⚠ ITS OWN NODE BECAUSE IT IS ITS OWN BILL. The profile row carries post IDENTITY - # and no engagement (measured), so likes/comments are a SECOND vendor call per post. - # A switch that multiplies a run's cost by the post count deserves to be visible on - # the canvas rather than buried in a config panel. - node("capture_posts", "capture", "Post data", - "Likes and comments per post" if metrics_on else "Off", - 2, panel="capture", on=metrics_on, - detail=(f"Up to {max_posts} posts per profile" if metrics_on else - "ut_ig_post_snapshots only grows while this is on")), - node("capture_comments", "capture", "Comment data", - "Comments on those posts" if comments_on else "Off", - 3, panel="capture", on=comments_on, - detail=("The full comments dataset. Many rows per post" if comments_on else - "Comments already embedded in a paid post row are kept either way")), - node("write", "write", "Write", - cfg.get("targetTable") or "No database", 4, panel="write", - on=not cfg.get("dryRun"), - detail="+ ut_ig_snapshots · ut_ig_posts · ut_ig_post_snapshots"), - ] - edges = [{"from": "trigger", "to": "source", "label": ""}, - {"from": "source", "to": "capture_posts", "label": ""}, - {"from": "capture_posts", "to": "capture_comments", "label": ""}, - {"from": "capture_comments", "to": "write", "label": ""}] - elif defn.get("kind") in DISCOVERY_KINDS: - # ⭐⭐ WAVE 30 · T05 — WIDENED FROM `== "discover_instagram"`, AND THIS IS THE FAILURE T04 - # UNMASKS. With the seed fixed but this arm still Instagram-only, a stored TikTok search - # fell past every arm to the no-bare-`else` promise below and returned THE TRIGGER NODE - # ALONE — so no node carried `panel="find"`, and `AutomationFind` (which mounts on - # `panelKey === "find"`) could never appear. The person would have picked the trigger - # successfully and then found nowhere to type a filter: a second, stranger bug, arriving - # as the reward for fixing the first one. - _platform, _default_table, _, _ = discovery_facts(defn.get("kind")) - limit = int(cfg.get("recordsLimit") or 0) - est = discover_estimate(limit) - pending = str((defn.get("state") or {}).get("pendingSnapshot") or "") - nodes += [ - node("find", "source", f"Find {_platform} profiles", - _predicate_sentence(cfg.get("predicates"), cfg.get("operator"))[:60], 1, - panel="find", - detail=(f"Up to {limit} profiles · about ${est['usd']}" - if bd_ready() - else "Profile search is not set up yet")), - node("collect", "capture", "Collect results", - "Searching…" if pending else "Takes about 20 minutes", 2, - panel="find", - detail=""), - node("write", "write", "Save results", - cfg.get("targetTable") or _default_table, 3, panel="write", - on=not cfg.get("dryRun"), - detail=""), - ] - edges = [{"from": "trigger", "to": "find", "label": ""}, - {"from": "find", "to": "collect", "label": ""}, - {"from": "collect", "to": "write", "label": ""}] - elif defn.get("kind") == "scrape_db": - url = str(cfg.get("url") or "") - host = (urlparse(url).hostname or url or "No URL") if url else "No URL" - which = ("Structured data (JSON-LD)" if cfg.get("extract") == "jsonld" - else f"HTML table #{int(cfg.get('tableIndex') or 0)}") - mapped = len(cfg.get("fieldMap") or {}) - nodes += [ - node("fetch", "source", "Fetch page", host, 1, panel="source", detail=url[:90]), - node("extract", "capture", "Extract", which, 2, panel="columns", - detail=f"{mapped} column{'' if mapped == 1 else 's'} mapped · key: " - f"{cfg.get('keyField') or ', '}"), - node("write", "write", "Write", cfg.get("targetLabel") or "Scraped table", 3, - panel="write", on=not cfg.get("dryRun"), - detail=cfg.get("targetTable") or "a new database"), - ] - edges = [{"from": "trigger", "to": "fetch", "label": ""}, - {"from": "fetch", "to": "extract", "label": ""}, - {"from": "extract", "to": "write", "label": ""}] - # ⛔ WAVE 24 — THERE IS NO BARE `else` HERE ANY MORE, and its absence is the point. - # It used to be `scrape_db`'s branch, which meant a kind this function had never been taught - # about inherited a scrape's three machine nodes: `Fetch page / No URL`, `Extract / HTML - # table #0`, `Write / Scraped table`, over a definition with no URL and no field map. A - # surface that draws nothing is obviously incomplete; one that draws a fetch step that does - # not exist is confidently wrong, and it is wrong on the canvas, the builder and the board at - # once because all three read this. - # - # ⭐ SO: `plain` (R6) — and any future kind, until somebody writes its arm — ANSWERS THE - # TRIGGER NODE AND NOTHING ELSE: `{"nodes": [], "edges": []}`. Exactly one node, - # guaranteed. The Builder's node filter and `stages_for` (hence the Board) both depend on - # that promise, so it is stated rather than left to be inferred from the control flow. - return {"nodes": nodes, "edges": edges} - - -def toggle_node(rt, auto_id, node_id): - """Flip ONE node's switch. Returns `(defn, error)`. - - The node → config mapping lives on the SERVER for the reason the whole graph does: a client - that knew which field each node writes would be a second copy of that knowledge, free to - drift. The canvas just says "this node was clicked". - """ - prev = all_definitions(rt).get(str(auto_id)) - if prev is None: - return None, "no such automation" - target = next((n for n in graph(prev)["nodes"] if n["id"] == str(node_id)), None) - if target is None: - return None, f"there is no {node_id!r} step in this automation" - if not target.get("toggle"): - return None, (f"the {target['title']} step cannot be turned off. It is what this " - f"automation is") - cfg, sched = dict(prev.get("config") or {}), dict(prev.get("schedule") or {}) - which = target["toggle"] - trg_key = str((prev.get("trigger") or {}).get("key") or "") - if which == "schedule" and trg_key and trg_key not in TRIGGER_SCHEDULE_KEYS: - # The trigger node's switch flips THE TRIGGER when the flow has an event one — flipping - # the cron under a node labelled "When a field changes" would be a switch that lies. - # ⭐ WAVE 24 — DERIVED from `TRIGGER_SCHEDULE_KEYS` instead of the hand-listed tuple that - # was here, which omitted `record_updated`, `enters_view` and `form_submitted` and so - # told exactly that lie for all three. The set names the SMALL side (schedule-driven), so - # a trigger added to `TRIGGER_KEYS` defaults to flipping itself, which is the safe half. - trg = dict(prev.get("trigger") or {}) - trg["enabled"] = not (bool(trg.get("enabled", True)) and not trg.get("paused")) - trg["paused"] = False # a deliberate flip clears an auto-pause: human wins - return patch(rt, auto_id, {"trigger": trg}) - if which == "schedule": - sched["enabled"] = not sched.get("enabled") - elif which == "postMetrics": - cfg["postMetrics"] = not cfg.get("postMetrics") - elif which == "commentMetrics": - cfg["commentMetrics"] = not cfg.get("commentMetrics") - elif which == "write": - cfg["dryRun"] = not cfg.get("dryRun") - return patch(rt, auto_id, {"config": cfg, "schedule": sched}) - - -# --------------------------------------------------------------------------------------------- -# THE BOARD IS DELETED (wave 27 item 12, owner ruling R3) — what is left, and why it stayed -# --------------------------------------------------------------------------------------------- -# Wave 22 made the automation detail a kanban board: columns were the flow's stages, cards were -# records of the target database, and each record's position lived in a single-select STAGE FIELD -# the engine created on the customer's own database. Wave 26 deleted the board's built-in -# terminals; wave 27 deletes the rest — `board()`, `move_card()`, `stages_for()`, -# `ensure_stage_field()`, the four card builders, the lane vocabulary on the wire, the review -# branch of the action walk, and the endings that sent a terminal card back round. -# -# ⛔ THE REASON IS A DATA REASON, NOT A UI ONE, and it is the sentence to keep: the board wrote -# MACHINE COLUMNS INTO A TENANT'S OWN TABLE — a stage select, an `_at` stamp and a `_cycles` -# counter per automation — to render a view of state the RUN LOG already holds. The customer paid -# for that in columns they did not ask for, in a store commit on every scheduled run, and in a -# permission wall (`humanMoves`/`arrive`) that existed only to stop them editing the columns we -# had added. `migrate_ig_tables` now drops those columns wherever an automation created them. -# -# ⛔⛔ `notify_review` IS GONE FROM HERE AND ITS CONSUMER IS STILL ALIVE — DEBT D-101, and this -# paragraph is its TOMBSTONE, placed where the next reader of the review machinery will look. -# -# `notify_review` was the producer of the `automation_review` notification: a card reaching a -# review stage told a human to come and look. It had ZERO callers even before R3 (wave-27 session -# C found it and correctly left it alone as another lane's subject), and R3's deletion took the -# function with the rest of the board. `verify_automation`'s W23-B section asserts its absence by -# name, so it cannot come back by accident. -# -# ⚠ THE SWEEP FOR ITS READERS, RUN 2026-08-12 (W31-T37) AND REPORTED RATHER THAN ASSUMED EMPTY: -# · SERVER — `notify_review` appears in exactly ONE file, `verify_automation.py`, inside the -# list of names asserted GONE. **No `.py` file anywhere produces an `automation_review` -# notification**; `core.alerts.notify` defaults `topic='automation'`, which is a different, -# live shape (the run-outcome alert). -# · CLIENT — the branch is FULLY ALIVE and is NOT in this fence: -# `web/src/alerts/alertsModel.ts` (`AUTOMATION_REVIEW_KIND`, `isAutomationReview`, the -# `autoId` guard) and `web/src/alerts/AlertsPane.tsx` (two call sites), plus legs in -# `web/verify_alerts.py`. All four are session B's files. -# ⇒ **The producer is deleted; the consumer guards an event that can never arrive.** D-101's own -# exit condition says these are one defect seen from two ends and must be closed in ONE change — -# so this half is the tombstone plus the sweep, and the client half is routed to B rather than -# reached across a fence. If a future wave gives review stages a real producer, THIS is the note -# that says the client already knows how to render it. -# -# ⚠ WHAT SURVIVES, AND IT IS DELIBERATE IN EACH CASE: -# * `lane_match` + `LANE_OPS` + the condition trees below — never board-only. They are the -# evaluator for ACTION conditions, trigger conditions and `find_records`, and the board was -# one of four callers. The name is the last thing the board left behind here. -# * `retire_automation_stage_fields` / `retire_automation_board_state` / `_without_retired_board` -# — the MIGRATION. It must outlive the thing it retires, or a definition written before the -# retirement walks into a runtime that no longer understands it. -# * `ai_decide` + `review_audit` — R3 keeps review "as an AI decision without lanes"; they are -# parked with no caller and say so at their own definitions. - -LANE_OPS = ("=", "!=", ">", ">=", "<", "<=", "includes", "not_includes", - "is_empty", "is_not_empty") -LANE_NULLARY_OPS = ("is_empty", "is_not_empty") - -# ───────────────────────────────────────────────────────────────────────────────────────────── -# WAVE 23 · C4 — CONDITION TREES. `Cond = leaf | {all: [Cond…]} | {any: [Cond…]}` -# -# A leaf is exactly what wave 22 called a lane condition (`{field, op, value?}`), which is why -# this generalisation needed no migration: a stored leaf IS a valid tree, `cond_match` dispatches -# on shape, and nothing at rest is rewritten until the owner saves that automation. (The doc -# calls this "legacy single-condition lanes auto-wrap as {all:[leaf]} on read" — wrapping is the -# same function applied one level up, so the cheaper honest version is to evaluate the leaf where -# it lies and never touch the bytes.) -# -# ⛔ DEPTH IS BOUNDED AND THE BOUND IS ENFORCED AT WRITE TIME, not at evaluation time. An -# unbounded tree is an unbounded evaluation on a hook that runs inside somebody's keystroke, and -# "the server got slow" is the failure nobody traces back to a filter somebody nested 40 deep. -# ⛔ REFUSE-NEVER-COERCE all the way down (`clean_predicates`' discipline): an empty group, an -# unknown comparison, a valueless compare are all REFUSED with the reason named. A group that -# quietly dropped its unanswerable leaf would WIDEN — the tri-state scar the filter engine -# carries ([[cg-filter-engine-sql-port]]), reproduced here where nothing would report it. -MAX_COND_DEPTH = 3 -MAX_COND_CHILDREN = 12 -COND_GROUP_KEYS = ("all", "any") - - -def clean_cond(raw, depth=0, where=""): - """Validate one condition TREE. Returns `(cond|None, error)`; `(None, None)` means "no - condition", which is legal everywhere a condition is optional (the catch-all lane, an - unconditioned trigger, an action group that always runs).""" - if raw in (None, "", {}): - return None, None - at = f" on {where}" if where else "" - if not isinstance(raw, dict): - return None, f"the condition{at} must be an object" - conj = [k for k in COND_GROUP_KEYS if k in raw] - if len(conj) > 1: - return None, (f"the condition{at} sets both 'all' and 'any'. A group is one or the " - f"other") - if conj: - key = conj[0] - if depth + 1 > MAX_COND_DEPTH: - return None, (f"conditions nest at most {MAX_COND_DEPTH} levels deep. " - f"the group{at} is deeper") - kids_raw = raw.get(key) - if not isinstance(kids_raw, list) or not kids_raw: - return None, f"the '{key}' group{at} needs at least one condition inside it" - if len(kids_raw) > MAX_COND_CHILDREN: - return None, (f"a group holds at most {MAX_COND_CHILDREN} conditions. " - f"the '{key}' group{at} has {len(kids_raw)}") - kids = [] - for child in kids_raw: - c, err = clean_cond(child, depth + 1, where) - if err: - return None, err - if c is None: - return None, (f"an empty condition sits inside the '{key}' group{at}. " - f"finish it or remove it") - kids.append(c) - return {key: kids}, None - field = _s(raw.get("field"), 80).strip() - op = _s(raw.get("op") or raw.get("operator"), 20).strip() - if not field: - # ⚠ WAVE 26 — the owner met this sentence repeatedly and it told them nothing they could - # act on. It states a fact about the stored tree; it never said what to DO, and the usual - # cause is not a typo but an empty field picker (the flow had no walking record to offer - # columns from). The fix belongs in the message. - return None, (f"the condition{at} names no field. Pick a column, or remove the " - f"condition") - if op not in LANE_OPS: - return None, f"{op or 'that comparison'!r} is not one of: " + ", ".join(LANE_OPS) - cond = {"field": field, "op": op} - if op not in LANE_NULLARY_OPS: - v = raw.get("value") - if v is None or (isinstance(v, str) and not v.strip()): - return None, f"give a value to compare {field!r} against{at}" - cond["value"] = v.strip() if isinstance(v, str) else v - return cond, None - - -def cond_fields(cond): - """Every field name a tree reads — the set a caller must have on the row image before the - answer means anything.""" - if not isinstance(cond, dict): - return set() - for key in COND_GROUP_KEYS: - if key in cond: - out = set() - for child in cond.get(key) or []: - out |= cond_fields(child) - return out - f = cond.get("field") - return {f} if f else set() -#: Lane labels a flow already uses for its fixed stages. A lane literally called "Review" would -#: collide with the stage the label is a choice FOR, and the board could no longer tell a -#: routed card from a gated one. -# ⚠ `tracked` / `declined` STAY RESERVED even though R6 deleted the stages that used them: the -# labels still sit in boards stored before this wave, and a user lane named "Tracked" beside a -# legacy stamped cell would read as the same lane while behaving as a different one. -def _lane_num(v): - try: - return float(str(v).replace(",", "")) - except (TypeError, ValueError): - return None - - -def lane_match(cond, row): - """Does `row` satisfy this condition TREE? None matches everything (the catch-all). - - Named for the lane that first needed it; it is now C4's whole evaluator, and every wave-22 - caller (`route_record`, `_settle_eval`, `_seed_event_state`) gained tree support by keeping - this name rather than growing a second entry point that could disagree with it. - - ⛔ REFUSE-NEVER-COERCE, per record: an ordering comparison whose either side does not parse - as a number answers False — the record simply does not enter the lane — never "treat blank - as 0", which is the `toNum(null)=0` widening the 2026-08-03 owner wave was about. A blank - cell is an unknown, and an unknown cannot be less than 10,000. - """ - if cond is None: - return True - if isinstance(cond, dict): - if "all" in cond: - return all(lane_match(c, row) for c in cond.get("all") or []) - if "any" in cond: - return any(lane_match(c, row) for c in cond.get("any") or []) - raw = (row or {}).get(cond.get("field")) - op = cond.get("op") - if op == "is_empty": - return raw in (None, "") - if op == "is_not_empty": - return raw not in (None, "") - want = cond.get("value") - if op in (">", ">=", "<", "<="): - a, b = _lane_num(raw), _lane_num(want) - if a is None or b is None: - return False - return {"<": a < b, "<=": a <= b, ">": a > b, ">=": a >= b}[op] - have_s, want_s = str(raw if raw is not None else ""), str(want if want is not None else "") - if op == "includes": - return want_s.lower() in have_s.lower() - if op == "not_includes": - return want_s.lower() not in have_s.lower() - a, b = _lane_num(raw), _lane_num(want) - same = (a is not None and b is not None and a == b) or have_s == want_s - return same if op == "=" else (not same) if op == "!=" else False - - -# ───────────────────────────────────────────────────────────────────────────────────────────── -# WAVE 23 · C4 — ACTIONS (owner ruling R3). The half of the Airtable builder that DOES things. -# -# `flow.actions` is an ordered list walked per RECORD, after the flow's machine steps have run. -# A `group` holds nested actions behind a condition — Airtable's "conditional action group", -# which the owner explicitly asked to be NESTABLE (R3 supersedes the wave-22 research brief's -# rec-8 "no nesting" verdict; that recommendation was about keeping a CANVAS legible, and this -# builder is a column, not a canvas). -# -# ⛔ THE CATALOG IS SERVER-OWNED AND INCLUDES WHAT WE HAVE NOT BUILT. `ACTION_CATALOG` carries a -# `ready` flag per kind, so the "+ Add advanced logic or action" menu paints Send email / Slack / -# Run script / Generate with AI faded with a reason instead of omitting them — the same honesty -# rule the trigger list follows, and the same enforcement: `clean_actions` REFUSES an unready -# kind with a sentence, so the faded state is a wall and not a styling choice. -MAX_ACTIONS = 20 # per automation, counting nested ones -MAX_GROUP_DEPTH = 2 # a group inside a group inside a group is a flowchart, not a flow -MAX_BRANCHES = 6 # per If / then -MAX_ACTION_VALUES = 20 # cells one update/create action may write -FIND_LIMIT_MAX = 100 -#: ⭐⭐ W31-T38 / E-4 — HOW MANY BROWSER JOBS ONE RUN MAY SUBMIT, and it belongs HERE because this -#: is the only layer that knows a record WALK is happening. `web_agent.MAX_STEPS` bounds steps per -#: JOB; nothing bounded jobs per RUN, and the web arm submits one job PER RECORD. A `web_read` on a -#: flow over the Customer grid is therefore thousands of ~10 s submissions against HF's -#: 6-concurrent cap — a run whose wall-clock is `records x 10 s` and a burst the `/hf-jobs` rules -#: exist to prevent. Session E raised it and could not fix it: the seam sees one step and cannot -#: know a walk is happening. -#: ⚠ 50 IS A STATED CHOICE, NOT A MEASUREMENT: at E's measured ~9-32 s per job that is roughly -#: 8-27 minutes of serial browsing, which is a long automation run and not a runaway one. It is -#: DISCLOSED when it binds (R6's second sentence — see the web arm), never silently applied. -#: ⛔ THE REAL FIX IS BATCHING, NOT A BIGGER NUMBER: `web_agent.run_plan(steps, ctx)` already takes -#: a list and the cost is per JOB, so a flow's web steps collected into ONE call would make this -#: ceiling nearly unreachable. That is E's own PENDING row; this bound is what stops the damage -#: until it lands. -MAX_WEB_JOBS_PER_RUN = 50 -#: ⭐⭐ W33-T58 (D-191) — HOW MANY WEB STEPS MAY SHARE ONE JOB. The comment above finally got its -#: batching, so the ceiling above now counts JOBS rather than pages and a record's consecutive web -#: steps cost one cold start between them instead of ~9 s each. -#: ⚠ 20 IS THE RUNNER'S OWN `MAX_STEPS` (`jobs/web_agent_job.py`), not a number chosen here: a plan -#: longer than that is refused by the job with `kind="too_many_steps"`, which would turn a -#: performance improvement into a whole record's worth of failed steps. Kept in step by -#: `verify_web_agent.py`, which reads both. -MAX_WEB_STEPS_PER_JOB = 20 -#: ⭐⭐ W33-T56 — HOW MANY STEPS ONE `ai_agent` ACTION MAY COMPOSE FOR ITSELF. -#: ⛔ A CEILING, NOT A TARGET, and it is the only bound between a sentence and a browser session. -#: The five `web_*` kinds are each ONE action a person wrote down; this one is a description that -#: the model turns into a journey at run time, so the number of real browser actions it performs is -#: decided by a paragraph rather than by the flow. Twelve is enough for "log in, search, open the -#: third result, read the price" and short of anything that reads as a program. -#: ⚠ Bounded by `MAX_WEB_STEPS_PER_JOB` as well, since the composed steps ride ONE job. -AI_AGENT_MAX_STEPS = 12 - - -#: The test seam for `_ai_agent_plan` — `None` in every shipped path. `verify_automation.py` sets it -#: so the fuzzy step is proven end to end with NO API key and NO spend, exactly as -#: `routes_automation._DRAFT_CHAT` does for the drafting door. -_AI_AGENT_CHAT = [None] - - -def _ai_agent_plan(cfg, row, log=print, st=None, user=""): - """A description + one record -> `(steps, "")` for `run_plan`, or `(None, sentence)`. - - ⭐⭐ W33-T56. This is the whole of the fuzzy step: the instruction and the record's own values - go to the model, and CONCRETE `web_*` steps come back — the same vocabulary a person could have - written by hand, so everything downstream (the seam, the job, the checkpoint, the per-step - verdicts) is unchanged and none of it has to know an assistant was involved. - - ⛔ IT MAY COMPOSE ONLY BROWSER STEPS. `ai_review.draft_flow` is handed a catalog filtered to - `WEB_KINDS`, so the enum in the tool schema cannot express `create_record` or a connector call. - A fuzzy sentence therefore cannot be talked into writing the tenant's data: the worst a bad - instruction can do is waste one browser session. That is a property of the SCHEMA, not of the - prompt, which is the only version of it worth relying on. - ⛔ AND IT IS BOUNDED. `maxSteps` is clamped at save time and re-applied here, because the - number of real browser actions is otherwise decided by a paragraph. - ⚠ `interpolate` FIRST, so the model is told about THIS record — `{{Website}}` is a different - page for every row, and a journey composed against the template would be one guess repeated. - """ - import ai_review # noqa: PLC0415 - url = interpolate(str(cfg.get("url") or ""), row) - instruction = interpolate(str(cfg.get("instruction") or ""), row) - cap = max(1, min(int(cfg.get("maxSteps") or AI_AGENT_MAX_STEPS), MAX_WEB_STEPS_PER_JOB)) - web_only = [r for r in ACTION_CATALOG if r.get("kind") in WEB_KINDS and r.get("ready")] - draft, why, _prov = ai_review.draft_flow( - prompt=(f"Starting page: {url}\n\nDo this: {instruction}\n\n" - f"Answer with at most {cap} steps. The FIRST step must open the starting page."), - catalog=web_only, - required={k: v for k, v in ACTION_REQUIRED.items() if k in WEB_KINDS}, - triggers=[], tables=[], - # ⭐ W35 · C7 (`NOTE E-16`) — threaded IN from `_walk` rather than resolved here: this - # function is pure over `(cfg, row)` by design and giving it a store handle of its own - # would be a second way to reach the tenant. - st=st, user=user, - chat=_AI_AGENT_CHAT[0]) - if why or not draft: - return None, (why or "the assistant produced no steps for that instruction") - steps, seen_url = [], False - for i, a in enumerate(draft.get("actions") or [], 1): - kind = str(a.get("kind") or "") - if kind not in WEB_KINDS: - # Belt and braces behind the enum: a rung that ignores its own schema is stopped here - # rather than reaching the browser. - return None, f"the assistant asked for a step this action cannot perform ({kind})" - c = a.get("config") or {} - step = {"kind": kind, "id": f"ai{i}", - "url": str(c.get("url") or "") or (url if not seen_url else ""), - "selector": str(c.get("selector") or ""), - "attr": str(c.get("attr") or "text"), - "all": bool(c.get("all")), - "waitFor": str(c.get("waitFor") or "") or None, - "timeoutMs": int(cfg.get("timeoutMs") or WEB_READ_TIMEOUT_MS)} - if c.get("value"): - step["value"] = str(c.get("value")) - if c.get("hint"): - step["hint"] = str(c.get("hint")) - if cfg.get("dryRun"): - step["dryRun"] = True - seen_url = seen_url or bool(step["url"]) - steps.append(step) - if len(steps) >= cap: - break - if not steps: - return None, "the assistant produced no steps for that instruction" - # ⛔ THE SEAM REFUSES A JOURNEY WHOSE FIRST STEP CARRIES NO ADDRESS — there is no page to act - # on yet — and the action's own `url` is exactly the answer. Supplying it here beats letting - # the whole job be refused for a sentence the model happened not to repeat. - if not steps[0].get("url"): - steps[0]["url"] = url - return steps, "" - - -def _web_missing(kind, cfg): - """Which of this kind's REQUIRED config keys are blank — the human phrases, in table order. - - ⚠ PER KIND, because they do not need the same things: `web_goto` needs a url and no selector; - `web_fill` needs a value nobody else takes; only `web_read` needs a column to write into. One - shared three-field test would have blocked every `web_goto` ever configured for want of a - selector it does not use. - ⭐ Lifted out of `apply_actions` at W33-T58 so the BATCH BUILDER and the per-step arm ask the - same question of the same table. Two copies of "is this step configured" would let a batch - include a step the arm then refuses — or worse, exclude one it would have run. - """ - return [n for n, key in WEB_REQUIRED.get(kind, ()) if not str((cfg or {}).get(key) or "").strip()] -#: ⭐⭐ W31 QA — THE FIVE WEB KINDS THE ENGINE DISPATCHES, and this is the THIRD list that names -#: them (`web_agent.RUNNABLE_KINDS` is the seam's, `jobs/web_agent_job.py::RUNNABLE_KINDS` is the -#: runner's). ⛔ IT IS DELIBERATELY NOT AN IMPORT: `_web_agent()` resolves the seam LAZILY so an -#: absent module cannot break the engine, and a module-level `from web_agent import RUNNABLE_KINDS` -#: would throw that property away for a tuple of five strings. The three lists are held in step by -#: `verify_web_agent.py::section_one_kind_only`, which reds if any pair disagrees — a per-kind -#: allow-list in three places that CAN disagree is the defect, whichever way it points. -WEB_KINDS = ("web_read", "web_goto", "web_fill", "web_click", "web_repair") -#: What each kind cannot run WITHOUT. Absence is never refused at SAVE (a step is addable before it -#: is configured — see `_clean_action_config`'s web arm); it is refused at RUN with a sentence -#: naming what is missing. `url` is required only by `web_goto`: every other kind acts on the page -#: the flow is already on, and the runner supplies its own refusal when it genuinely needs one. -WEB_REQUIRED = { - "web_read": (("a URL", "url"), ("a CSS selector", "selector"), - ("a column to write into", "field")), - "web_goto": (("a URL", "url"),), - "web_click": (("a CSS selector", "selector"),), - "web_repair": (("a CSS selector", "selector"),), - "web_fill": (("a CSS selector", "selector"), ("a value to type", "value")), - # ⭐⭐ W33-T56 — the fuzzy step needs a page to start on and a job to do, and nothing else. - # ⚠ NO `selector` AND NO `field`: working out the selector IS the step, and where to put the - # answer is optional (a journey may only need to have been performed). - "ai_agent": (("a URL", "url"), ("a description of what to do", "instruction")), -} -#: ⭐⭐ WAVE 32 · T45 (owner item 10) — WHAT EACH ACTION KIND CANNOT RUN WITHOUT, for every kind. -#: -#: `WEB_REQUIRED` above already WAS this table for five kinds, complete with the human phrase each -#: refusal says out loud, so this is its widening rather than a second opinion — spread in, never -#: re-typed, or a wave that adds a web key would teach the run refusal and not the label. -#: -#: ⛔ MOST KINDS ARE ABSENT, AND THE ABSENCES ARE THE INTERESTING PART. `create_record`, -#: `update_record`, `find_records` and `group` are REFUSED AT SAVE when their config is incomplete -#: (`_clean_action_config`: *"the create record action names no database"*, *"…writes no values"*, -#: *"a branch with no actions inside it does nothing"*), so a stored one is configured by -#: construction and a row here would be a second, weaker copy of a wall that already holds. -#: -#: ⛔⛔ AND `enrich_instagram` / `enrich_tiktok` ARE DELIBERATELY ABSENT DESPITE BEING THE OBVIOUS -#: CANDIDATE. Their `profileField` may be empty on purpose: the C3 profile FLAG on the target -#: database resolves the binding at run time (`profile_field_key`), so an empty one is a working -#: action on any flagged database and marking it Unconfigured would put a red label on the -#: commonest correct configuration there is. The genuinely unbound case still fails closed at RUN -#: with its own sentence — which is the honest division of labour: this table holds what can be -#: answered from the ACTION ALONE, and anything needing the target database's schema stays a -#: run-time refusal rather than becoming a store read on the automations LIST path (W30-T12 took -#: that read off this route; putting it back to draw a label would undo the wave before this one). -ACTION_REQUIRED = dict(WEB_REQUIRED) -#: ⭐ W31-T38 (C5) — the web step's own clamps. `20000` matches the default E's seam documents; -#: the ceiling exists because this blocks the record walk (~9-32 s of cold start ALREADY), and an -#: automation that can be configured to wait ten minutes per record is a stalled run, not a slow one. -WEB_READ_TIMEOUT_MS = 20000 -WEB_READ_TIMEOUT_MAX_MS = 120000 -#: ⭐ WAVE 24 · C-ACT — THE MENU'S GROUP ORDER IS THE SERVER'S. The client sorts by `groupOrder` -#: and never by a literal list of group names: a client-side ordering is a second copy of this -#: table, and the way it fails is that a group added here renders last, or not at all, with -#: nothing red anywhere. -ACTION_GROUP_ORDER = {"Web action": 1, "Database": 2, "Connected": 3, "Advanced logic": 4} -ACTION_CATALOG = [ - # ── 1. WEB ACTION (owner rulings R1-R5) — DECLARED HERE, NOT YET BUILT (DEBT D-51). ──────── - # ⛔ NOT A TEASE. `clean_actions` refuses an unready kind with a sentence, so the faded row - # is a WALL, not a styling choice — and each `detail` says the true reason rather than a - # placeholder, because "coming soon" on five rows is how a menu stops being believed. - # - # ⚠ WAVE 25 RETARGETED THE ONE STRING THAT NAMED A WAVE, and the reason generalises: W24 - # scheduled this build for wave 25 and wave 25's six owner items do not include it, so - # "building next (wave 25)" became false the moment this wave shipped — a menu that dates its - # own promises has to be re-read by whoever misses the date, and a stale date is worse than - # none because it reads as a commitment somebody already broke. The replacement names the - # missing CAPABILITY, like its four siblings always did; D-51 carries the schedule. - # ⭐⭐ WAVE 31 · T38 + QA (C5) — ALL FIVE ROWS ARE READY, AND THE RULING IS WHY. - # - # ⛔ THIS REVERSES T38's ORIGINAL `how:`, ON THE OWNER'S OWN WORDS. T38 shipped `web_read` - # alone and held the other four at `ready:False`, citing PRD R10 ("the ones that WRITE to a - # third party do not flip without R5's approval gate existing"). **R5 was REVOKED the same - # day, and the revocation is recorded as an amendment on this wave's board** - # (`TICKETS.md:1418`, `mailbox/E.md:278`, `web_agent.py:103`) — owner, verbatim: - # *"make sure we unblock all web actions, we don't need approval step first wtf, I never ask - # for that."* Session C declined E's ask against the superseded half of R10, so four actions - # the owner asked for shipped unusable, and `verify_web_agent` was RED on correct-by-ruling - # code for the whole wave. Confirmed at QA against four independent recordings of the quote. - # ⚠ THE FADED ROW IS A WALL, NOT A STYLE: `clean_actions` refuses an unready kind with a - # sentence, so this flag is what makes the action storable at all — which is exactly why a - # row left `False` against a ruling is a feature that does not exist. - # ⚠ THE `detail` STRINGS CHANGED WITH THE FLAGS, deliberately. They said *"Needs the browser - # job"* and *"Needs the recorder that captures a selector"* — both false since E's runner - # landed. A menu that dates its own promises is this module's own recorded complaint one - # comment up; a row that advertises a missing prerequisite it already has is the same defect. - # ⭐⭐ WAVE 33 · W33-T56 (owner item 7, ruling R3) — THE FUZZY STEP. - # ⛔ IT ADDS; IT REPLACES NOTHING. R3 is explicit: *"No existing `web_*` kind is removed — - # dropping a kind from the catalog 400s every stored automation using it, forever"* (D-65). - # This is the step for a journey somebody can DESCRIBE but not spell as a selector; the five - # kinds below stay exactly as they are for the journeys they can already express, and a person - # who knows the selector should still use `web_read`, which costs no model call. - # ⭐⭐ WAVE 34 · W34-T44 / R18 — SIX ROWS BECAME ONE, AND THE OTHER FIVE DID NOT LEAVE THE - # CATALOG. Owner, verbatim: *"Remove Do this on a page / Open a page / Fill a field / Click - # something / Read from the page / Repair a broken step completely. One action, Web agent."* - # - # ⛔⛔ WHY THEY ARE HIDDEN AND NOT DELETED, AND IT IS NOT D-65 THIS TIME — IT IS WORSE. - # `_ai_agent_plan` builds the model's tool schema as - # `[r for r in ACTION_CATALOG if r["kind"] in WEB_KINDS and r["ready"]]` - # so DELETING these five rows empties that list, and the ONE action the ruling keeps would be - # left able to compose nothing at all. Deleting the six would have deleted the survivor, in - # silence, and `ai_agent`'s own tests would still pass because they stub the chat rung - # (`_AI_AGENT_CHAT`). D-65's usual argument (a deleted kind 400s every stored automation - # forever) applies as well and is the smaller half. - # - # ⭐ SO THE LINE MOVED FROM "IS IT IN THE CATALOG" TO "IS IT ON THE MENU". `menu: False` is - # withheld by `action_catalog()` from the picker, while `clean_actions` (which reads this - # constant, not the wire) still validates the kind, the runner still runs it, and a stored - # automation built before today keeps working and stays editable. - # ⚠ THE LABELS CHANGED TOO, and that is R18's "appear NOWHERE" clause taken literally: a - # hidden row's label is still rendered on a STORED step's card, so leaving the six captions in - # place would have kept them on screen for exactly the people who already use them. They now - # name the mechanism instead, and no old caption survives as a substring in any case. - {"kind": "ai_agent", "label": "Web agent", "group": "Web action", "ready": True, - "detail": "Describe a job on a website in words; the agent works out the steps, runs them in " - "a browser and reports what it actually did"}, - {"kind": "web_goto", "label": "Web agent step (navigate)", "group": "Web action", - "ready": True, "menu": False, - "detail": "Opens a page in a browser job and reports the title it landed on"}, - {"kind": "web_fill", "label": "Web agent step (type)", "group": "Web action", - "ready": True, "menu": False, - "detail": "Types a value into a field. Mark it secret and the value is masked in the log"}, - {"kind": "web_click", "label": "Web agent step (click)", "group": "Web action", - "ready": True, "menu": False, - "detail": "Clicks the element a selector names, and reports where it landed"}, - # ⚠ `"kind": "web_read"` AND `"ready": True` STAY ON ONE LINE. `verify_wiring`'s C5 row matches - # `"kind": "web_read".*?"ready": True` without DOTALL, so wrapping this row the way its four - # siblings are wrapped turns that cross-fence assertion red — on a formatting change, with the - # mount and the flag both intact. The row is A's file and its CLAIM is right (a catalog kind - # the client can add must be one the runner will execute); the layout is what it happens to - # depend on, so the layout is preserved here rather than the assertion weakened there. - {"kind": "web_read", "ready": True, "label": "Web agent step (extract)", - "group": "Web action", "menu": False, - "detail": "Reads one value off a live page in a browser job. Expect ~10-30 s per step"}, - {"kind": "web_repair", "label": "Web agent step (relocate)", "group": "Web action", - "ready": True, "menu": False, - "detail": "Follows a label to its control when a selector has gone stale, and proposes one"}, - # ── 2. DATABASE ────────────────────────────────────────────────────────────────────────── - {"kind": "update_record", "label": "Update record", "group": "Database", "ready": True, - "detail": "Write values onto the record walking the flow"}, - {"kind": "create_record", "label": "Create record", "group": "Database", "ready": True, - "detail": "Add a row to another database"}, - {"kind": "find_records", "label": "Find records", "group": "Database", "ready": True, - "detail": "Look rows up by condition; the run log opens them"}, - # ── 3. CONNECTED ───────────────────────────────────────────────────────────────────────── - # ⭐ WAVE 25 · C4 (owner rulings R3/R4) — THE ENRICH ACTION, and it REPLACES a whole KIND. - # `field_instagram` was an automation you created to fill one column; this is a step any flow - # can take, which is the shape it should always have had — enriching a profile is something - # you do TO a record, not a species of automation. - # ⭐ WAVE 27 · C4 (item 33) — `connector` NESTS THIS ROW UNDER "Scraper" in the action menu, - # exactly as `TRIGGER_CONNECTOR` already nests the trigger picker. See `ACTION_CONNECTOR`. - {"kind": "enrich_instagram", "label": "Enrich Instagram profile", "group": "Connected", - "ready": True, "connector": "scraper", - "detail": "Fill this record's Instagram columns from its profile, and add a point to its " - "history"}, - # ⭐⭐ WAVE 30 · T08 / CONTRACT C3 — THE SECOND NETWORK, AND THIS ROW IS THE LAST SWITCH THAT - # LANDS, deliberately. `apply_actions._walk`'s kind dispatch has NO terminal `else` (measured - # from the AST, not read): an unknown kind is walked, counted, reports the run `ok`, and - # writes nothing. So a catalog entry ahead of its runner arm would ship an action that is - # addable, clickable, storable and silently inert — strictly worse than the state before it, - # where `clean_actions` refuses the kind with a sentence. - # ⚠ `connector: "scraper"` NESTS IT UNDER THE SAME BUCKET AS INSTAGRAM (R3), which is the same - # value T07 gave the trigger — one word, both menus. - {"kind": "enrich_tiktok", "label": "Enrich TikTok profile", "group": "Connected", - "ready": True, "connector": "scraper", - "detail": "Fill this record's TikTok columns from its profile, and add a point to its " - "history"}, - {"kind": "send_email", "label": "Send email", "group": "Connected", "ready": False, - "detail": "Needs a send scope on the Gmail connection"}, - # ⭐⭐ WAVE 35 · T35 / CONTRACT C8 / OWNER RULING R10 — STATEMENTS BECOME AN AGENT STEP. - # - # Owner item 14: move Statements out of Settings and into the agent automation, "templatic and - # easily toggleable". R10 is the half that decides the shape: the step ASSEMBLES and PARKS a - # batch in a review stage, and **nothing sends without a human click**. So this row's promise - # is deliberately "prepares", not "sends" — the label a person reads must not describe an act - # the step does not perform. - # - # ⛔ ADDED BESIDE `send_email`, NEVER BY REPURPOSING IT (the ticket's own trap, D-295): a - # stored automation naming a kind that no longer exists is refused FOREVER rather than dropped - # with a reason, so adding a kind is cheap and re-pointing one is not. - # - # ⛔ `ready: True` IS WHAT MAKES IT STORABLE — `clean_actions` refuses any row whose `ready` is - # false — and the note at `enrich_tiktok` above is the reason the RUNNER ARM lands in the same - # wave rather than after it: `_walk` has no terminal `else`, so a catalog row ahead of its arm - # is addable, storable and SILENTLY INERT, which is worse than not existing. T35 lands an arm - # that reports it is not configured; T36 makes it park a real batch. - # - # ⚠ TENANT-GATED, not `ready: False`: `TENANT_GATED_ACTIONS` withholds this row from every - # tenant but #0, because the send client behind it is env-credentialed and is tenant #0's. - {"kind": "send_statement", "label": "Prepare customer statements", "group": "Connected", - "ready": True, - "detail": "Assemble this month's statements and park them for review. Nothing is sent until " - "somebody opens the batch and clicks Send"}, - # ⭐⭐ WAVE 36 · W36-T39 — D-277 CLOSED THE WAY THAT ROW ITSELF RECOMMENDED: *"an - # `ACTION_CATALOG` row with `menu: false` — one line, reusing the door R18 built this same wave - # to keep the five web kinds readable but unofferable. (a) looks right; it is E's file."* - # - # ⛔ THE DEFECT WAS A KIND IN NO CATALOG ON EITHER SIDE. `routes_automation._field_agent_rows` - # emits `flow.actions[0].kind == "ai_enrich"` for every AI-enrichment column in the tenant, and - # that string appeared ZERO times here and ZERO times in `aios-web/web/src/automation/`. The - # builder resolves a stored step's caption with `catalog.find(c => c.kind === a.kind)` and falls - # back to `a.kind`, so a field agent opened in the Agents module showed a RAW TOKEN. - # - # ⚠ `menu: False`, NEVER `ready: False`. `ready: False` renders as "coming soon" — a promise — - # and `clean_actions` refuses the kind outright, which would 400 the synthetic row the moment - # anything validated it. `menu: False` is the exact shape R18 built: withheld from the picker, - # still resolvable as a caption, still valid. - # ⛔ AND IT IS NOT ADDABLE BY HAND ON PURPOSE. An enrichment belongs to a COLUMN; the automation - # canvas is not where one is created, which is why `patch_automation` already refuses a - # `field:` id with "change its prompt, model or schedule on the column itself". - # ⭐⭐ AND A THIRD INSTANCE, FOUND BY W36-T39's OWN GATE ON ITS FIRST RUN. `_odoo_sync_row` - # emits `flow.actions[0].kind == "odoo_sync"` for the connector's synthetic schedule agent, and - # that kind was in no catalog either — the same raw token on the same screen as D-277, one row - # down. Two known instances were enough to justify the check; the check then produced a third - # nobody had booked, which is the difference between a gate and a regression test. - {"kind": "odoo_sync", "label": "Sync from Odoo", "group": "Connected", - "ready": True, "menu": False, - "detail": "Pull the connected Odoo databases on a schedule. Configured on the connector, not " - "here"}, - {"kind": "ai_enrich", "label": "Enrich this column with AI", "group": "Connected", - "ready": True, "menu": False, - "detail": "Fill an AI column for the records this flow walks. Configured on the column, not " - "here"}, - {"kind": "slack", "label": "Send Slack message", "group": "Connected", "ready": False, - "detail": "Needs the Slack connector"}, - # ── 4. ADVANCED LOGIC ──────────────────────────────────────────────────────────────────── - # `group` is relabelled "If / then" (C-ACT): "Conditional logic" described the mechanism, - # and R8 made it a FORK with lettered branches, which is a thing people already have a name - # for. The KIND is untouched — renaming it would orphan every stored action for a caption. - {"kind": "group", "label": "If / then", "group": "Advanced logic", "ready": True, - "detail": "Send the record down one of several branches, by condition"}, - {"kind": "repeating_group", "label": "Repeating group", "group": "Advanced logic", - "ready": False, "detail": "Run the same actions on every item in a list"}, - {"kind": "run_script", "label": "Run script", "group": "Advanced logic", "ready": False, - "detail": "Not built. A sandbox is its own decision"}, - {"kind": "generate_ai", "label": "Generate with AI", "group": "Advanced logic", - "ready": False, "detail": "Needs an AI action implementation"}, -] - - -#: ⭐ WAVE 27 · C4 (owner item 33) — WHICH CONNECTOR AN ACTION BELONGS TO, so the action menu can -#: nest exactly as the trigger picker already does (`reference/Airtable Automation 10.png`: an -#: "Integrations" header, one row per connector, a chevron into that connector's own submenu). -#: -#: ⚠ SAME SHAPE AS `TRIGGER_CONNECTOR`, DELIBERATELY, and the same warning applies: these are -#: GROUPING HANDLES for the picker, not `/connectors/directory` slugs. A client must group by the -#: key and render the label, never join it against the directory. -#: ⛔ AND THE LABEL IS NOT DECLARED HERE AT ALL — it is LOOKED UP in `TRIGGER_CONNECTOR` by key. -#: A "Scraper" nest in the trigger menu and a "Scrapers" nest in the action menu would be one -#: connector wearing two names on two screens somebody sees within a second of each other, and a -#: second literal is how that happens. This tuple says only WHICH connectors an action may name; -#: what they are CALLED has exactly one source. -ACTION_CONNECTORS = ("scraper",) - - -def _connector_meta(key): - """`{key, label}` for a connector key, from the one place either picker declares it.""" - key = str(key or "") - if key not in ACTION_CONNECTORS: - return None - return next((dict(v) for v in TRIGGER_CONNECTOR.values() if v.get("key") == key), None) - - -#: ⭐⭐ WAVE 35 · T35 / CONTRACT C8 — THE TENANT PREDICATE `routes_statements._royal_only` USES, -#: LIFTED SO THERE IS EXACTLY ONE COPY OF IT. C8's words are "the predicate is imported, never -#: re-expressed", and this is the direction that import can run: `automation_engine` imports no -#: route module and no FastAPI (checked), and breaking that to reach `_royal_only` would drag -#: `deps` + `routes_admin` into the engine. So the ENGINE holds the test and the ROUTE calls it. -#: -#: ⛔⛔ THIS IS NOT `odoo_relational.is_royal`, AND THE DIFFERENCE IS A SEND. That one asks "is this -#: tenant ENTITLED to Odoo databases" over `RI_SLUGS = ("", "royal-imports")` — it answers TRUE for -#: the EMPTY slug and lower-cases its input. This one is `_royal_only`'s exact test: the runtime's -#: own `key`, matched exactly. A tenant whose key never got set would pass `is_royal("")` and then -#: queue statements THROUGH TENANT #0'S ENV-CREDENTIALED SEND CLIENT, i.e. email another company's -#: customers over Royal Imports' name. Entitlement to READ is not authority to SEND, and the two -#: questions keep their two predicates on purpose. Do not "unify" them. -ROYAL_TENANT_KEY = "royal-imports" - - -def is_statement_tenant(rt): - """Exactly `routes_statements._royal_only`'s test, as a boolean over the RUNTIME. - - Keyed on the runtime, never on a request field: a tenant is a property of the SESSION, so a - payload cannot argue its way into another company's sender. - """ - return getattr(rt, "key", None) == ROYAL_TENANT_KEY - - -#: Kinds only SOME tenants may see or store, as `{kind: predicate(rt) -> bool}`. -#: -#: ⛔ FAIL-CLOSED ON `rt=None`, AND THAT IS THE WHOLE SAFETY ARGUMENT. Every reader below treats an -#: absent runtime as "not allowed", so a call site that forgets to pass one makes the action VANISH -#: rather than become universal. The opposite default would mean any future caller of -#: `action_catalog()` or `clean_actions()` silently offers tenant #0's send door to every tenant — -#: an omission that is invisible in review and loud only in production [[default-must-pass-its-own-guard]]. -TENANT_GATED_ACTIONS = {"send_statement": is_statement_tenant} - -#: The dunning buckets a statements step may filter on. ⚠ ONE LITERAL, TWO READERS: this and -#: `routes_statements.statements()`'s `"tiers"` field are the same four strings, and a step -#: configured against a tier the sender's worklist does not produce would filter to nothing and -#: report success. `routes_statements` imports this rather than repeating it. -STATEMENT_TIERS = ("A-Urgent", "B-Active", "C-Light", "Monitor") - - -def _tenant_may_use(kind, rt): - """May this runtime see/store this action kind? Ungated kinds are always yes.""" - gate = TENANT_GATED_ACTIONS.get(str(kind or "")) - return True if gate is None else bool(rt is not None and gate(rt)) - - -def catalog_kinds(): - """Every action kind this module knows about — the LABEL vocabulary. - - ⭐ D-277's WHOLE LESSON IN ONE SENTENCE: the client resolves a stored step's caption out of the - catalog and falls back to the raw kind token, so a kind the server can EMIT and the catalog - does not carry is a token on somebody's screen. This is the set that must cover every kind any - server path can put into a `flow.actions` entry, whether or not a person may add it. - """ - return frozenset(str(row.get("kind") or "") for row in ACTION_CATALOG) - - -def configurable_kinds(): - """The kinds a PERSON may add from the picker, and must therefore be able to configure. - - ⭐⭐ W36-T39 — THE CONTRACT BETWEEN THE TWO TREES, DERIVED AND NEVER LISTED. `ready` alone is - the wrong set: the five `web_*` kinds are ready and `menu: False` (R18), and `ai_enrich` is - ready and `menu: False` (D-277) — all six are real, runnable, captioned, and unofferable. What - a client must be able to CONFIGURE is exactly what a person can ADD. - - ⛔ DERIVED FROM THE CATALOG, so a new row joins the contract by existing. A hand-kept list - would be a third copy of the vocabulary, and the two copies this ticket exists to reconcile - were already one too many. - """ - return frozenset(str(row.get("kind") or "") for row in ACTION_CATALOG - if row.get("ready") and row.get("menu", True) is not False) - - -def action_catalog(rt=None): - """The catalog as the wire carries it — a copy, because a caller that mutated the module - constant would change every later reader's answer. - - ⭐⭐ WAVE 35 · T35 (C8): `rt` filters TENANT-GATED rows out entirely — not `ready: False`, not - `menu: False`, but ABSENT. A tenant that may not send statements should not learn that the - capability exists, and `ready: False` renders as "coming soon", which is a promise we are not - making to them. ⚠ Called with no `rt` the gated rows are withheld (fail-closed): see - `TENANT_GATED_ACTIONS`. - - ⭐ WAVE 24 (C-ACT): each row is stamped with its `groupOrder`, DERIVED from - `ACTION_GROUP_ORDER` rather than hand-written per row, so a group cannot be given two - different orders by two rows that claim to be in it. A group nobody has ordered sorts LAST - (not first) — a new group appearing above "Web action" because its order defaulted to 0 is - the failure that would look deliberate. - - ⭐ WAVE 27 (C4): a row naming a `connector` is stamped with the full `{key, label}` the client - nests on — RESOLVED here rather than written out per row, for the reason `groupOrder` is: two - rows in one nest cannot disagree about what that nest is called. A row naming an unknown - connector loses the stamp instead of inventing a nest with a raw slug for a title. - - ⭐⭐ WAVE 34 · W34-T44 / R18: `menu` is stamped on EVERY row, never left absent. A row the - picker must not offer carries `menu: False` and STILL RIDES THE WIRE, because the client - resolves a STORED step's label out of this same list (`AutomationBuilder`: - `catalog.find(c => c.kind === a.kind)` then `row?.label || a.kind`) — withholding the row - entirely would make an existing web step render its raw kind token at somebody, which this - module forbids in those words elsewhere. - ⚠ STAMPED RATHER THAN LEFT TO DEFAULT: absent-means-true is a rule two codebases have to - remember the same way, and a client filter is one `!== false` away from meaning the opposite. - An explicit boolean on every row cannot be read two ways. - """ - last = max(ACTION_GROUP_ORDER.values()) + 1 - out = [] - for a in ACTION_CATALOG: - if not _tenant_may_use(a.get("kind"), rt): - continue - row = {**a, "groupOrder": ACTION_GROUP_ORDER.get(a.get("group"), last), - "menu": bool(a.get("menu", True))} - conn = _connector_meta(a.get("connector")) - if conn: - row["connector"] = dict(conn) - else: - row.pop("connector", None) - out.append(row) - return out - - -def action_needs(action): - """⭐⭐ WAVE 32 · T45 (owner item 10) — what THIS action is still missing, in the words a person - reads. `[]` means Configured. - - Pure over `(kind, config)` — no runtime, no store read. That is what lets `_wire` stamp every - action on the automations LIST without putting the `user_tables` document back on a route - W30-T12 just took it off. - """ - action = action or {} - cfg = action.get("config") if isinstance(action.get("config"), dict) else {} - return [name for name, key in ACTION_REQUIRED.get(str(action.get("kind") or ""), ()) - if not str(cfg.get(key) or "").strip()] - - -def _walk_actions(actions): - """Every action in a flow, INCLUDING the ones nested inside If / then branches. - - ⛔ A FLAT `for a in actions` MISSES HALF A FLOW. `group.config.branches[].actions` is where a - conditional puts its real work, and a configured-check that only saw the top level would - report a flow ready to run while the step inside branch B had never been filled in — which is - the exact class of defect this ticket exists to surface, hiding inside the ticket's own fix. - """ - for action in actions or []: - if not isinstance(action, dict): - continue - yield action - for branch in ((action.get("config") or {}).get("branches") or []): - if isinstance(branch, dict): - yield from _walk_actions(branch.get("actions")) - - -def unconfigured_actions(defn): - """Every ENABLED action of this automation that cannot run, as `[{id, kind, label, needs}]`. - - ⚠ DISABLED ACTIONS ARE SKIPPED, and that is the point of being able to disable one: a step - somebody switched off is not a step blocking the run. `apply_actions` already ignores them. - """ - out = [] - for action in _walk_actions(((defn or {}).get("flow") or {}).get("actions")): - if not action.get("enabled", True): - continue - needs = action_needs(action) - if needs: - out.append({"id": str(action.get("id") or ""), "kind": str(action.get("kind") or ""), - "label": _action_label(action), "needs": needs}) - return out - - -def run_refusal(defn): - """⛔ WAVE 32 · T45 — the sentence a run is refused with, or `""`. - - Owner item 10: *"running an automation with ANY unconfigured action is REFUSED with a message - naming which action"*. Naming it is half the requirement and the half that is easy to drop — a - bare *"an action is not configured"* on a twelve-step flow is a hunt, not a message. - - ⛔ IT LIVES ON THE ENGINE, NOT ON THE ROUTE, BECAUSE THE ROUTE IS NOT THE ONLY DOOR. The tick - runs automations on a schedule, the webhook door runs them, and a check mounted in - `POST /automations/{id}/run` alone would refuse the button and let the cron sail past it — - [[seal-the-transport-not-the-rung]], and D-112's own shape (an action bound to a deleted view - walked zero records and reported `ok`). `run_now` refuses for every caller; the route asks - first only so the person clicking gets a 400 with the sentence instead of a silent no-op. - - ⚠ THIS IS A BEHAVIOUR CHANGE FOR STORED AUTOMATIONS AND IT IS THE RULING. A flow with one - unconfigured web step used to run, skip that step with a note, and report `ok`; it now does not - run at all. That is what "blocks the run" means, and the alternative — running everything else - and reporting success — is precisely what the owner is asking to stop. - """ - missing = unconfigured_actions(defn) - if not missing: - return "" - return "; ".join(f"{m['label']} still needs " + ", ".join(m["needs"]) for m in missing) - - -def _action_label(act): - for row in ACTION_CATALOG: - if row["kind"] == act.get("kind"): - return row["label"] - return str(act.get("kind") or "Action") - - -def clean_actions(raw, depth=0, _seen=None, _count=None, notes=None, rt=None): - """Validate `flow.actions`. Returns `(actions, error)` — refuses, never coerces. - - ⭐⭐ WAVE 35 · T35 (C8) — `rt` IS THE TENANT WALL ON THE STORE SIDE, and it is a separate wall - from `action_catalog(rt)`'s. Withholding a row from the MENU stops it being offered; it does - not stop a hand-written body naming the kind, and the picker is not a security boundary - [[opening-a-route-widens-every-field]]. ⚠ Keyword-only in effect and defaulting to None, so - every existing caller is untouched by construction — the same shape `notes` used, for the - reason this docstring already gives about a signature change failing at RUN, not at import. - ⛔ `rt=None` REFUSES a gated kind rather than allowing it (`TENANT_GATED_ACTIONS`). - - Ids are STABLE: a caller's well-formed `id` is kept, so selecting an action in the builder - survives a Save. A missing or colliding one is minted `act_`; minting on every clean would - move the selection under the person editing it. - - ⭐⭐ D-75 — `notes` IS THE DISCLOSURE CHANNEL, AND IT IS OPT-IN. Pass a list and this function - appends one plain sentence per thing it SILENTLY CHANGED: a `create_record` condition dropped - (see the note at that branch), and any config key an arm's allowlist did not keep. - - ⛔ WHY IT IS AN OUT-PARAMETER RATHER THAN A THIRD RETURN VALUE. D-75's own exit says *"this is - a signature change across its callers"* and that is exactly what makes the obvious fix - dangerous mid-wave: `clean_actions` is called from `clean_flow`, from `_is_untouched_ig_seed` - and recursively from its own group arm, and a caller that unpacked two values from a - three-tuple fails at RUN, not at import. An optional list defaults to `None`, every existing - caller is untouched by construction, and the one door that wants to tell somebody opts in. - ⚠ IT REPORTS, IT NEVER REFUSES. Every drop here is deliberate and D-65 is the reason — refusing - a stored automation's condition would 400 it forever, with no way to edit it out, because the - editor cannot save the automation it needs to fix. A drop is recoverable; a locked door is not. - What was missing was never the refusal, it was somebody being told. - """ - if raw in (None, ""): - return [], None - if not isinstance(raw, list): - return None, "actions must be a list" - seen = _seen if _seen is not None else set() - count = _count if _count is not None else [0] - out = [] - for entry in raw: - if not isinstance(entry, dict): - return None, "each action must be an object with a kind" - kind = _s(entry.get("kind"), 30).strip() - row = next((r for r in ACTION_CATALOG if r["kind"] == kind), None) - if row is None: - return None, (f"{kind or 'that action'!r} is not one of: " - + ", ".join(a["kind"] for a in ACTION_CATALOG)) - if not row["ready"]: - return None, (f"{row['label']!r} is on the menu but not built yet. " - f"{row['detail'][0].lower()}{row['detail'][1:]}") - # ⭐⭐ W35-T35 (C8) — THE TENANT WALL. Refused, not dropped, and this is the one place in - # this function where a refusal is right: D-65's "drop, never refuse" protects a stored - # automation from becoming permanently unsavable, and NO tenant this can refuse has ever - # been able to store one — the kind is withheld from their catalog, so there is no legacy - # body to strand. Dropping it silently would instead let a flow save, look saved, and never - # do the step the person configured. - if not _tenant_may_use(kind, rt): - return None, (f"{row['label']!r} is not available in this workspace. It sends as " - f"Royal Imports, using Royal Imports' own mail credentials") - count[0] += 1 - if count[0] > MAX_ACTIONS: - return None, f"an automation runs at most {MAX_ACTIONS} actions" - aid = _s(entry.get("id"), 40).strip() - if not re.fullmatch(r"act_[a-z0-9_]{1,32}", aid or "") or aid in seen: - n = 1 - while f"act_{n}" in seen: - n += 1 - aid = f"act_{n}" - seen.add(aid) - # ⭐ WAVE 26 · C4 / owner ruling R9 — A `create_record` CARRIES NO CONDITION, EVER. - # - # Owner, correcting their own earlier answer mid-grill: *"if it is Create Record, I don't - # think you can even add Conditions at all. That's not how the Create Record works."* - # Airtable agrees, and so does the shape: every other action operates ON the record the - # flow is walking, so "run this only when …" is a question about something - # that exists. Create record MAKES one. There is nothing to test yet, which is why the - # picker on this panel was empty and why every condition saved against it named no field — - # the owner's *"the condition on action act_1 names no field"*. The condition belongs on - # the trigger, or on a conditional group wrapping the action. - # - # ⛔ DROPPED, NOT REFUSED, AND THE DIFFERENCE IS D-65. Refusing would 400 every stored - # automation that already carries one — forever, with no way to edit it out, because the - # editor cannot save the automation it needs to fix. A drop is recoverable; a refusal is a - # locked door. - # ⚠ Dropped SILENTLY, and that is defensible only because the panel goes with it: wave 26 - # removes the condition editor from this action kind entirely (C4), so there is no control - # whose value could appear to be ignored. If a `create_record` condition editor ever comes - # back, this needs a disclosure channel — `clean_actions` has none today. - if kind == "create_record": - # D-75: the drop is unchanged; what is new is that a caller can be TOLD about it. - if notes is not None and entry.get("when"): - notes.append(f"{aid}: the condition on a Create-record step was dropped. A " - f"Create record has no record to test yet. Put the condition on the " - f"trigger, or on an If wrapping this step.") - when = None - else: - when, cerr = clean_cond(entry.get("when"), where=f"action {aid}") - if cerr: - return None, cerr - cfg_raw = entry.get("config") if isinstance(entry.get("config"), dict) else {} - cfg, cerr = _clean_action_config(kind, cfg_raw, depth, seen, count, notes=notes, rt=rt) - if cerr: - return None, cerr - # ⛔ D-75 — THE ALLOWLIST'S OWN DROPS, DIFFED HERE RATHER THAN REPORTED BY EACH ARM. Every - # arm builds a fresh `out = {...}`, so none of them knows what it did not keep; the caller - # of the arm does, because it holds both dicts. One diff, and a new arm is covered the day - # it is written rather than the day somebody remembers to instrument it. - if notes is not None and isinstance(cfg, dict): - gone = [k for k in cfg_raw if k not in cfg] - if gone: - notes.append(f"{aid}: {', '.join(sorted(gone))} " - f"{'is' if len(gone) == 1 else 'are'} not a setting a " - f"{KIND_LABELS.get(kind, kind)!r} step uses, so " - f"{'it was' if len(gone) == 1 else 'they were'} not saved.") - out.append({"id": aid, "kind": kind, - "enabled": bool(entry.get("enabled", True)), - "when": when, "config": cfg}) - return out, None - - -def _clean_action_config(kind, cfg, depth, seen, count, notes=None, rt=None): - """One action's `config`, per kind. Returns `(config, error)`. - - ⚠ `rt` is threaded ONLY so the `group` arm can hand it back to `clean_actions` for the branch - recursion (W35-T35 / C8). Without it a tenant-gated kind is refused at the top level and - ACCEPTED inside an If, which is the half of a flow `_walk_actions`' own docstring says people - cannot see. - """ - if kind == "group": - # ⭐ WAVE 24 · C-FORK (owner ruling R8) — a group is a FORK now: `{branches: [...]}`, - # each `{id, label, cond, actions}`. It was one condition with one action list, i.e. an - # if with no else, so expressing "otherwise" meant a second group carrying the negation - # by hand — two conditions a later edit could put out of step with each other. - if depth + 1 > MAX_GROUP_DEPTH: - return None, (f"conditional groups nest at most {MAX_GROUP_DEPTH} deep. " - f"past that a flow is a flowchart and belongs on the board") - raw_branches = cfg.get("branches") - if not isinstance(raw_branches, list) or not raw_branches: - # ⛔ MIGRATION, NOT A REFUSAL. The shipped shape is exactly one implicit branch, and - # the live automations carry it — refusing it would 400 every Save of a flow that - # was legal yesterday. Read-side, so nothing has to be rewritten in the store. - raw_branches = [{"id": "b1", "label": "A", "cond": cfg.get("cond"), - "actions": cfg.get("actions")}] - if len(raw_branches) > MAX_BRANCHES: - return None, (f"an If / then has at most {MAX_BRANCHES} branches. Past that the " - f"record is being routed, which is what the board's lanes are for") - out_branches, bseen = [], set() - for i, br in enumerate(raw_branches): - if not isinstance(br, dict): - return None, "each branch of an If / then is an object" - cond, cerr = clean_cond(br.get("cond"), where="the branch") - if cerr: - return None, cerr - if cond is None and i != len(raw_branches) - 1: - # ⛔ ONE Otherwise, and it is LAST. `_walk` takes the first branch that matches - # and a null condition matches everything, so a catch-all above another branch - # would make every branch below it dead code — silently, and only for the - # records that reached it. Refused with the reason rather than reordered: this - # module does not quietly rewrite what somebody built. - return None, ("only the LAST branch of an If / then can be the Otherwise leg. " - "a catch-all above another branch makes the ones below it " - "unreachable") - # D-75: threaded into the branch too — an unconfigured step inside an If is - # exactly the one a person cannot see, which is `_walk_actions`' own argument. - kids, kerr = clean_actions(br.get("actions"), depth + 1, seen, count, - notes=notes, rt=rt) - if kerr: - return None, kerr - if not kids: - return None, "a branch with no actions inside it does nothing" - bid = _s(br.get("id"), 40).strip() - if not re.fullmatch(r"b[a-z0-9_]{0,32}", bid or "") or bid in bseen: - bid = f"b{i + 1}" - bseen.add(bid) - label = " ".join(_s(br.get("label"), 40).split()) or ( - "Otherwise" if cond is None else _branch_letter(i)) - out_branches.append({"id": bid, "label": label, "cond": cond, "actions": kids}) - return {"branches": out_branches}, None - if kind in ("update_record", "create_record"): - values = cfg.get("values") - if not isinstance(values, dict) or not values: - return None, f"the {kind.replace('_', ' ')} action writes no values" - if len(values) > MAX_ACTION_VALUES: - return None, f"one action writes at most {MAX_ACTION_VALUES} cells" - clean_vals = {} - for k, v in values.items(): - key = _s(k, 80).strip() - if not key: - return None, "a value is written to a field with no name" - if v is not None and not isinstance(v, (str, int, float, bool)): - return None, (f"the value for {key!r} must be text or a number. " - f"an automation writes cells, not objects") - clean_vals[key] = _s(v, 500) if isinstance(v, str) else v - out = {"values": clean_vals} - # ⭐ WAVE 24 — an OPTIONAL self-given name, so a SEEDED action can say what it is - # ("Create an Instagram record", C-TRIG law 3) instead of wearing the catalog's generic - # "Create record". Follows the `review` arm below, which has carried `config.label` since - # wave 23 — one precedent, not a new mechanism, and it lives inside the untyped `config` - # bag so no shared client type has to change for it. - lbl = " ".join(_s(cfg.get("label"), 60).split()) - if lbl: - out["label"] = lbl - if kind == "create_record": - table = _s(cfg.get("table"), 60).strip() - if not table: - return None, "the create record action names no database" - if not table.startswith(UT_PREFIX): - return None, (f"actions write to blank databases (ut_*) this wave. " - f"{table!r} is not one") - out["table"] = table - # ⭐ WAVE 25 · C5 (owner ruling R1a) — UNIQUE ON. Without it this action APPENDS - # forever (`_commit_action_writes` mints `max+1`), so ANY scheduled flow carrying a - # Create record duplicates a row per run — silently, and worse every day. - # - # ⛔ `""` IS THE STORED DEFAULT AND IT MEANS TODAY'S APPEND. Not a migration, not a - # guess: a stored automation must not change meaning because this key arrived. The - # upsert is a thing somebody turns on, the same shape `clean_ending` gave `terminal`. - # - # ⛔ AND IT MUST NAME A FIELD THIS ACTION ACTUALLY WRITES. Upserting on a key the - # action never sets means every incoming row carries a BLANK key — `upsert_rows` - # counts those as `skipped` and writes nothing at all. That is a control that reads - # as "no duplicates" and delivers "no rows", which is the worse failure by a distance. - # Same shape as `scrape_db`'s "the key field must be one of the mapped fields" - # (`clean_config`, above) — one precedent, not a second mechanism. - unique = _s(cfg.get("uniqueOn"), 80).strip() - if unique and unique not in clean_vals: - return None, (f"this action does not write {unique!r}, so it cannot keep records " - f"unique on it. It writes: " + ", ".join(sorted(clean_vals))) - out["uniqueOn"] = unique - return out, None - if kind == "send_statement": - # ⭐⭐ WAVE 35 · T35 / R10 — WHAT A STATEMENTS STEP IS CONFIGURED WITH: a customer filter - # and a template. Both are the SENDER's own vocabulary (`routes_statements.send` passes - # `templates: {subject, intro, footer}` straight to `collections_send.queue_statement`), - # so this arm names no field the send door does not already take. - # - # ⚠ EVERY TEMPLATE FIELD IS OPTIONAL AND EMPTY MEANS "THE DEFAULT", which is exactly how - # the send door already reads them (`t.get("subject") or cs.DEFAULT_SUBJECT`). Storing our - # own copy of the default instead would freeze today's wording into every stored agent and - # silently stop tracking `collections_send`'s. - out = {} - tier = _s(cfg.get("tier"), 40).strip() - # ⛔ REFUSED, NOT COERCED, AND THE ANSWER LISTS THE REAL ONES — the same shape the connector - # cadence uses. A tier nobody sends to would filter the batch to zero customers and report - # a successful run: a silent no-op is the worst outcome available here, because the person - # believes their customers were invoiced. - # ⚠ `""` IS LEGAL and means EVERY tier. It is the stored default, so an agent saved before - # anybody picks a filter does not change meaning the day this key arrives (D-65's rule). - if tier and tier not in STATEMENT_TIERS: - return None, (f"{tier!r} is not a collection tier. Pick one of: " - + ", ".join(STATEMENT_TIERS) + ", or leave it blank for all of them") - out["tier"] = tier - for key, cap in (("subject", 200), ("intro", 4000), ("footer", 4000)): - out[key] = _s(cfg.get(key), cap) - lbl = " ".join(_s(cfg.get("label"), 60).split()) - if lbl: - out["label"] = lbl - return out, None - if kind in ENRICH_KINDS: - # ⭐ WAVE 30 · T08 — BOTH networks share this branch. See `ENRICH_KINDS`: the selection, the - # cooldown, the limit clamps and their reasons are network-agnostic, and a second copy for - # TikTok would drift until one network accepted a limit the other refused. - # ⭐ WAVE 25 · C4. The switches are `field_instagram`'s, unchanged in meaning — the paid - # rung, its fallback, the per-post engagement buy and the dry run — because they are the - # things that decide what a run COSTS and R4 moves the capability, not the controls. - # - # ⚠ `profileField` MAY BE EMPTY, and that is the A3 stored-inert pattern rather than - # laxness: the C3 profile FLAG resolves it at run time, so the action can be added to a - # flow before anybody has flagged the column. An unresolvable binding fails CLOSED with a - # sentence at run (`profile_field_key`), which is the honest half — refusing the save - # would make the action unaddable on a database that has not been flagged yet. - # C5: ONE validator, shared with `clean_config`'s machine-kind branch. Two independent - # clamps on the same key is how the action and the legacy kind come to disagree about - # what "10 posts" means. - # - # ⚠ `submitted=False` HERE, AND IT IS THE OPPOSITE OF THE OTHER CALL SITE. The difference - # is not the value, it is what "present" MEANS on each path. `clean_config` gets a PATCH, - # so a key being there is a person having typed it. This function gets the whole action - # list re-posted on every save (`_pin_unique`'s note says so in as many words), so a - # `maxPosts` here is usually just the client echoing back what was already stored — and - # every action written before this wave stored the old default of 24. Refusing would - # therefore 400 those automations forever on a number nobody chose, which is D-65. - # The ceiling is still taught, in the place where a person can act on it: the panel's - # input is bounded at MAX_POSTS_PER_PULL (C5 tells B not to offer more). - max_posts, perr = clean_max_posts(cfg.get("maxPosts"), submitted=False) - if perr: - return None, perr - # ⭐ 2026-08-07 (owner ruling) — THE SELECTION, and every key here is OPTIONAL with a - # working default. That is D-65's lesson applied ahead of time rather than after: an enrich - # action stored before this wave carries none of these, and it must keep saving and keep - # running. Absent `fromView` = the whole database; absent `limit` = DEFAULT_ENRICH_LIMIT; - # `skipRecent` absent = OFF, because a cooldown nobody asked for silently stops enriching. - # ⚠ `limit` is CLAMPED here and again in `enrich_selection`. Not belt-and-braces: this door - # sees a person's typed number and can refuse politely, while the selection sees whatever - # is STORED — including configs written before the cap existed. Clamping only here would - # let an old 5000 through at run time. - try: - e_limit = int(cfg.get("limit") or DEFAULT_ENRICH_LIMIT) - except (TypeError, ValueError): - return None, "how many records to enrich has to be a number" - if e_limit < 1: - return None, "an enrich step has to be allowed at least one record" - try: - e_days = int(cfg.get("skipRecentDays") or DEFAULT_ENRICH_COOLDOWN_DAYS) - except (TypeError, ValueError): - return None, "the number of days has to be a number" - groups, gerr = clean_post_groups(cfg.get("postGroups"), max_posts) - if gerr: - return None, gerr - return {"profileField": _s(cfg.get("profileField"), 80).strip(), - "fromView": _s(cfg.get("fromView"), 80).strip(), - "sortField": _s(cfg.get("sortField"), 60).strip() or DEFAULT_ENRICH_SORT, - # Anything that is not "asc" is newest-first, so a typo cannot invert the order a - # person is spending against — it lands on the documented default instead. - "sortDir": "asc" if str(cfg.get("sortDir") or "").strip().lower() == "asc" - else "desc", - "limit": min(e_limit, MAX_ENRICH_PER_RUN), - "skipRecent": bool(cfg.get("skipRecent")), - "skipRecentDays": max(1, e_days), - # ⛔ `tier`/`noFallback` are ACCEPTED AND IGNORED here for the same reason as in - # `clean_config` (R5 / C2): a stored action carrying them still saves and simply - # loses them, because refusing a retired key 400s every definition written before - # this wave (D-65). - # ⛔ THE FLAG THAT MULTIPLIES THE BILL BY THE POST COUNT. Off unless asked for. - # ⭐ WAVE 30 · T10 — THE TIKTOK FORCE-OFF IS GONE, because the capability it was - # standing in for now exists. T08 shipped `... and kind != "enrich_tiktok"` here - # deliberately: the key was ACCEPTED (never a 400, D-65's lesson) and answered - # honestly with `False`, because storing a `True` nothing acts on is a money switch - # claiming a capability nobody built. Both networks now read the SAME two keys. - "postMetrics": bool(cfg.get("postMetrics")), - # The separate Comments dataset is more granular than a post read and stays off - # until the user intentionally enables it. - "commentMetrics": bool(cfg.get("commentMetrics")), - "dryRun": bool(cfg.get("dryRun")), - # ⭐ 2026-08-09 (owner) — "last 12 reels / 12 videos", per GROUP. Absent = OFF, so - # every action stored before today keeps saving and keeps running unchanged. - "postGroups": groups, - "maxPosts": max_posts}, None - if kind == "find_records": - table = _s(cfg.get("table"), 60).strip() - if not table: - return None, "the find records action names no database" - cond, cerr = clean_cond(cfg.get("cond"), where="the find") - if cerr: - return None, cerr - try: - limit = int(cfg.get("limit") or 25) - except (TypeError, ValueError): - return None, "the find records limit must be a whole number" - if limit < 1 or limit > FIND_LIMIT_MAX: - return None, f"the find records limit is between 1 and {FIND_LIMIT_MAX}" - return {"table": table, "cond": cond, "limit": limit}, None - if kind == "ai_agent": - # ⭐⭐ W33-T56 — THE FUZZY STEP'S CONFIG. Same posture as the web arm below in every - # respect that matters: an ALLOWLIST, a shape test on the url, a clamped timeout, and an - # EMPTY config stored rather than refused so the step can be added before it is configured - # (the wave-23 illegal-default-seed defect, and D-79's scar). - # ⚠ IT DELIBERATELY DOES NOT TAKE A `selector`. Working the selector out is the step; a - # selector box here would be a control that contradicts the action's own reason to exist. - _u = _s(cfg.get("url"), 500).strip() - if _u and not (_u.startswith("http://") or _u.startswith("https://") - or _u.startswith("{{")): - return None, "a web address must start with http:// or https://" - try: - _t = int(cfg.get("timeoutMs") or WEB_READ_TIMEOUT_MS) - except (TypeError, ValueError): - return None, "the web step timeout must be a whole number of milliseconds" - try: - _max = int(cfg.get("maxSteps") or AI_AGENT_MAX_STEPS) - except (TypeError, ValueError): - return None, "the step ceiling must be a whole number" - out = {"url": _u, - # ⚠ LONGER THAN A SELECTOR AND SHORTER THAN A PROMPT. This is a sentence or two - # describing a job, and the model is given the page's own vocabulary at run time — - # a 4,000-character instruction is a sign somebody is writing a program in prose. - "instruction": _s(cfg.get("instruction"), 600), - "field": _s(cfg.get("field"), 80).strip(), - "timeoutMs": max(1000, min(_t, WEB_READ_TIMEOUT_MAX_MS)), - # ⛔ THE CEILING IS THE POINT OF THE CLAMP, not the default. Every step this action - # composes is a real browser action inside one job, and a fuzzy instruction is - # exactly the input that produces twenty of them. Bounded here so a run cannot be - # talked into an unbounded journey by a sentence. - "maxSteps": max(1, min(_max, AI_AGENT_MAX_STEPS))} - if cfg.get("dryRun"): - out["dryRun"] = True - return out, None - if kind in WEB_KINDS: - # ⭐⭐ WAVE 31 · T38 (C5 / R10) — AND THIS ARM IS THE HALF THAT WOULD HAVE SHIPPED MISSING. - # - # ⭐⭐ W31 QA WIDENED IT FROM `web_read` TO ALL FIVE KINDS, AND THAT IS THE SAME DEFECT THIS - # ARM'S OWN COMMENT DESCRIBES, ARRIVING A SECOND TIME. When the four write kinds flipped to - # `ready:True` on the owner's revocation of R5, they became storable — and with this arm - # still testing `kind == "web_read"` they fell through to `return {}, None` below, keeping - # their KIND and losing `url` / `selector` / `value` / `hint` **silently, with no error - # anywhere**. Addable, storable, runnable, and forever unconfigured. The paragraph below - # was written about exactly this, one kind earlier; widening the test is what makes it - # true of the feature rather than of one row in it. - # - # ⛔ FOUND BY THE GATE, NOT BY REVIEW, AND IT IS THE TICKET'S OWN FAILURE MODE. Flipping - # `ready:True` and mounting the dispatch arm is not enough: with no branch here the - # function falls through to `return {}, None` below, so a stored `web_read` action keeps - # its KIND and loses its `url`, `selector` and `field` **silently, with no error anywhere**. - # The action would be addable, storable, runnable — and would fetch `""` forever. That is - # precisely the *"addable and inert"* outcome E's hand-off warned about, arriving through a - # different door than the one being watched, and the same silent-drop class as `patch`'s - # hand-maintained key list one screen up. - # - # ⚠ THE URL IS NOT `guard`ed HERE, on purpose. `guard` does DNS, and this is a validator - # that runs on every save of every automation — a save must not block on name resolution, - # and a template like `https://{Domain}/p` is not resolvable until a record supplies the - # value. The rail is enforced where the fetch happens (the job, and `web_agent`'s own - # refusal), which is the only place the FINAL url exists. What is enforced here is the - # shape: http(s) or an interpolation that could become one. - # ⛔⛔ AN EMPTY CONFIG IS STORED, NOT REFUSED — THE A3 STORED-INERT PATTERN, and the first - # draft of this arm got it wrong in a way only a gate could see. Requiring `url` / - # `selector` / `field` here makes the action UNADDABLE: `AutomationBuilder::seedFor` mints - # a new step from a seed and persists it immediately, so a validator that refuses an empty - # seed yields a red banner and no card — the wave-23 illegal-default-seed defect, caught by - # `verify_automation_ui`'s catalog-derived seed check the moment `web_read` became ready. - # ⇒ The same posture the enrich arm already takes with `profileField`: a step may be added - # to a flow before it is configured, and it FAILS CLOSED AT RUN with a sentence naming what - # is missing. What is refused here is a value somebody actually TYPED and got wrong — never - # the absence of one they have not typed yet. - url = _s(cfg.get("url"), 500).strip() - # ⚠ `{{` IS THE TEMPLATE FORM — `interpolate`'s syntax is `{{field_key}}`, NOT `{field}`. - # A url that STARTS with a placeholder is legitimate (the record supplies the host), so the - # shape test admits it; anything else must be an absolute http(s) address. - if url and not (url.startswith("http://") or url.startswith("https://") - or url.startswith("{{")): - return None, "a web address must start with http:// or https://" - selector = _s(cfg.get("selector"), 200).strip() - field = _s(cfg.get("field"), 80).strip() - try: - timeout = int(cfg.get("timeoutMs") or WEB_READ_TIMEOUT_MS) - except (TypeError, ValueError): - return None, "the web step timeout must be a whole number of milliseconds" - timeout = max(1000, min(timeout, WEB_READ_TIMEOUT_MAX_MS)) - out = {"url": url, "selector": selector, "field": field, - "attr": _s(cfg.get("attr"), 40).strip() or "text", - "all": bool(cfg.get("all")), "timeoutMs": timeout} - wait = _s(cfg.get("waitFor"), 200).strip() - if wait: - out["waitFor"] = wait - # ⭐ THE FOUR WRITE KINDS' OWN KEYS. Carried for EVERY web kind rather than switched on - # `kind`, because the cost of carrying an unused key is nothing and the cost of dropping a - # used one is a step that runs forever against a value the user typed and cannot see. - # `value` is the text `web_fill` types; `secret` masks it in the run log; `hint` is the - # human label `web_repair` follows when a selector has gone stale; `dryRun` rehearses a - # write without performing it (E measured the field reading back empty afterwards). - value = _s(cfg.get("value"), 500) - if value: - out["value"] = value - hint = _s(cfg.get("hint"), 200).strip() - if hint: - out["hint"] = hint - if cfg.get("secret"): - out["secret"] = True - if cfg.get("dryRun"): - out["dryRun"] = True - return out, None - return {}, None - - -def _branch_letter(i): - """0 -> 'A', 1 -> 'B', … (R8's lettering). Past 'Z' it doubles rather than wrapping, so a - label is never reused — `MAX_BRANCHES` makes that unreachable, and a silent collision is - worse than an ugly name.""" - return chr(65 + i % 26) * (1 + i // 26) - - -def group_branches(act): - """The branches of a group action, MIGRATING the pre-wave-24 `{cond, actions}` shape. - - ⭐ ONE READER for the fork's shape, because the alternative is what this module keeps paying - for: `clean_actions`, `walk_actions` and the runner would each need to know that a stored - group might carry either shape, and the one that forgot would silently skip a live - automation's actions. A definition cleaned since wave 24 always carries `branches`; one read - straight out of the store (the runner reads definitions, not payloads) may not. - """ - cfg = (act or {}).get("config") or {} - brs = cfg.get("branches") - if isinstance(brs, list) and brs: - return [b for b in brs if isinstance(b, dict)] - return [{"id": "b1", "label": "A", "cond": cfg.get("cond"), - "actions": cfg.get("actions") or []}] - - -def walk_actions(actions): - """Every action in the tree, depth-first, groups included. One walker, so "how many actions - does this flow have" and "which review stages exist" cannot answer differently. - - ⚠ WAVE 24: a group's children live under its BRANCHES now. This walker feeds the action - COUNT, `review_actions` (hence the board's stages) and `compose_sentence` — so a version - that still read `config.actions` would report a fork's contents as zero actions, hide every - review inside a branch from the board, and let `MAX_ACTIONS` be exceeded without noticing. - """ - for act in actions or []: - yield act - if act.get("kind") == "group": - for br in group_branches(act): - for kid in walk_actions(br.get("actions")): - yield kid - - -def interpolate(text, row): - """`{{field_key}}` → the record's value. Unknown keys resolve to EMPTY, deliberately: an - action that wrote the literal `{{status}}` into a cell because somebody typo'd the key would - put template syntax in front of a customer, and a blank is the visible failure.""" - if not isinstance(text, str) or "{{" not in text: - return text - def _sub(m): - v = (row or {}).get(m.group(1).strip()) - return "" if v is None else str(v) - return re.sub(r"\{\{([^}]{1,80})\}\}", _sub, text) - - -# ──────────────────────────────────────────────────────────────────────────────────────────���── -# WAVE 23 · C4/C5/C6 — THE ACTION RUNNER. One record at a time, actions in declared order. -# -# ⛔ ONE COALESCED WRITE PER TABLE PER RUN. The runner accumulates patches in memory and commits -# them once at the end (the 256-commits/hr budget every writer in this module respects). A -# per-action `rt.update` would be correct and would also spend the tenant's whole commit budget -# on one busy flow. -# ⛔ A REVIEW SUSPENDS THAT RECORD AND NOTHING ELSE. When a card reaches a review action the -# runner stamps its stage and stops walking THAT record — later actions belong to the branch a -# human (or the AI) has not chosen yet. Other records keep going; a gate is not a global pause. -# ⛔ EVERY WRITE HERE IS MACHINE-ORIGIN and goes through this module's own writers, never the -# human doors — which is exactly what keeps A2(2)'s loop prevention structural: an action's write -# cannot fire an event trigger, its own or a sibling's. - -def _act_row_patch(patches, table, rid, values): - patches.setdefault(table, {}).setdefault(str(rid), {}).update(values) - - -def profile_field_key(table, named="", source=PROFILE_SOURCE_IG): - """C3/C4: WHICH COLUMN on this database carries the profile handle. '' when none does. - - Resolution order, and the missing branch is the important one: - 1. the column the action NAMES — the person picked it, so it is not a guess; - 2. otherwise the column carrying C3's flag `profile: {source: "instagram"}`; - 3. otherwise **NOTHING**, and the run says so. - - ⛔ THERE IS DELIBERATELY NO FALL-BACK TO `_auto_url_field`. That helper finds "the first `url` - column" and is exactly right for `field_instagram`, whose whole configuration was a URL column - — but here it would silently bind the enrich step to whatever URL column happens to be first, - on a database where nobody has said which column is a profile. The failure would be a run that - reports success having enriched from the wrong column, which is worse than one that refuses: - a wrong number is harder to notice than a missing one, and it would be written into the - permanent history as if it had been measured. - - ⭐ WAVE 30 · T08 — `source` PARAMETERISES STEP 2 ONLY, defaulting to Instagram so every - existing caller is byte-for-byte unchanged. Step 1 (the column the person NAMED) is already - network-agnostic: a named column is a decision, not a guess, and second-guessing it against a - flag would refuse a binding somebody made on purpose. - ⛔ AND THE TWO SOURCES MUST NOT BE MERGED INTO "any profile flag". `ut_tt_profile.handle` - carries `profile: {source: "tiktok"}` and `ut_ig_profile.handle` carries `"instagram"`; a - resolver that accepted either would let a TikTok enrich step bind to an Instagram column and - spend money asking a TikTok dataset about an Instagram handle — a wrong number written into a - permanent history, which is the exact failure the no-fallback rule above exists to prevent. - """ - fields = (table or {}).get("fields") or [] - want = str(named or "").strip() - if want: - return want if any(str(f.get("key")) == want for f in fields) else "" - for f in fields: - p = f.get("profile") - if isinstance(p, dict) and str(p.get("source") or "") == str(source): - return str(f.get("key") or "") - return "" - - -def _flow_table(defn): - """The database a flow's actions and ending operate on: the automation's own target, or the - table its event trigger watches when the flow has no target of its own. - - ⭐ WAVE 30 · T05 — the discovery arm covers BOTH kinds. It read `== "discover_instagram"`, so a - TikTok discovery with no stored `targetTable` fell to the trigger's table — which a corpus - search does not have — and resolved to `""`. `_presets_after_write` asks this function where to - spawn the preset columns, so an empty answer there is a database that never appears. - """ - cfg = (defn or {}).get("config") or {} - if defn.get("kind") in DISCOVERY_KINDS: - return cfg.get("targetTable") or discovery_facts(defn.get("kind"))[1] - return cfg.get("targetTable") or ((defn.get("trigger") or {}).get("table") or "") - - -#: ⭐ 2026-08-07 (owner ruling) — WHAT ONE ENRICH STEP MAY SPEND IN A SINGLE RUN. -#: `limit` is the number the panel offers; this is the ceiling the module enforces whatever the -#: panel sends, and it is DISCLOSED when it bites rather than applied as a silent `[:N]` -#: ([[no-unverifiable-aggregates]]). 100 profiles is ~4 minutes of vendor pacing on its own. -MAX_ENRICH_PER_RUN = 100 -DEFAULT_ENRICH_LIMIT = 25 -DEFAULT_ENRICH_COOLDOWN_DAYS = 30 -#: Which column the selection orders by when nobody has said. `first_found` newest-first is the -#: owner's own example ("top N enrichment sorted by date") and the useful default: it enriches what -#: discovery just found rather than re-walking the oldest leads in the book. -DEFAULT_ENRICH_SORT = "first_found" - -#: ⭐⭐ 2026-08-09 (DEBT D-103) — WHERE A RUN'S PER-RECORD REASONS TRAVEL. -#: A run's `counts` are integers and `_commit_run` drops everything non-numeric, so the sentence -#: a vendor gave us had nowhere to ride and was thrown away at three separate layers. This key is -#: a deliberate passenger IN the counts dict, popped by `run_now` before the counts are stored. -#: ⛔ A RESERVED KEY, NOT A COUNT: it must never be rendered as one, which is why it starts with -#: an underscore — `COUNT_LABELS` on the client is an allow-list and cannot pick it up by -#: accident, and `_commit_run`'s numeric filter is the second net under it. -RUN_NOTES_KEY = "_notes" -#: ⭐⭐ WAVE 28 / OWNER RULING R9 — `NOT_FOUND_RETRY_DAYS` IS RETIRED. A handle a VENDOR SAID DOES -#: NOT EXIST is now a TOMBSTONE: never auto-retried, at any interval. -#: ⛔ THE 30-DAY BACKOFF WAS THE DEFENSIBLE ANSWER AND IT WAS STILL WRONG, which is the sentence -#: worth keeping. Its argument was that a handle can be renamed back or a suspension lifted, so -#: the verdict should expire. But nothing about US changes on day 31 — the only new information -#: is a guess that the world moved — so the timer buys one paid vendor call per dead handle per -#: month, forever, on a row nobody is looking at, and reports it as "1 blocked". A tenant with a -#: hundred stale handles pays a hundred times a month to be told the same thing. -#: ⭐ WHAT RE-ARMS IT IS A HUMAN, and there are two doors: -#: 1. the KEY is `(platform, handle)` — so correcting a typo re-arms IMMEDIATELY, with no -#: wiring at all, because the corrected handle simply is not the one we recorded; and -#: 2. `clear_gone()` — called when the profile CELL is written, so re-typing the SAME handle -#: (a person saying "try it again") also re-arms. -#: ⚠ The run keeps NAMING the skipped handles every time, not only on the run that discovered -#: them: a tombstone the owner cannot see is just a disappearance. -#: A queued profile snapshot the vendor never finishes is dropped after this, with a note. An -#: unbounded pending list is the forever-loop this whole change exists to remove, wearing the -#: opposite mask ([[gate-answers-the-wrong-question]]). -PENDING_PROFILE_MAX_HOURS = 24 - - -def _order_key(value): - """Order one CELL. Numbers numerically, everything else as text. - - ⚠ Cells are stored as STRINGS, so `"20872"` and `"9"` compare in the wrong order as text — - which on a `followers` sort is not a cosmetic wrong order, it is the wrong 25 accounts getting - paid for. Numbers are tried first and text is the fallback, never the other way round. - ⚠ Dates need no special case: R3 made these columns `date` and they store `YYYY-MM-DD`, where - lexicographic order IS chronological order. If that format ever changes, this comment is the - thing that was relied on. - """ - s = str(value if value is not None else "").strip() - try: - return (0, float(s.replace(",", "")), "") - except ValueError: - return (1, 0.0, s.lower()) - - -def _days_since(day, today=None): - """Whole days between a `YYYY-MM-DD` stamp and today, or None when it cannot be read. - - ⛔ None means UNKNOWN and every caller must treat it as "not measured", never as 0 or as a - large number — the cooldown below reads an unreadable stamp as "never enriched", which spends - money rather than skipping. That is the right way round: a skipped record is invisible, and a - stamp we cannot parse is our bug, not a reason to silently stop enriching somebody's data. - """ - import datetime as _dt - s = str(day or "").strip()[:10] - if not s: - return None - try: - d = _dt.date.fromisoformat(s) - except ValueError: - return None - return ((today or _dt.date.today()) - d).days - - -def _hours_since(stamp): - """Whole hours since a full ISO stamp, or None when it cannot be read. - - ⚠ ISO, NOT `YYYY-MM-DD` — `_days_since` above answers a question in DAYS about a `date` - column and truncates to 10 characters. A pending snapshot's age is measured in hours and its - stamp carries a time, so reusing that function would read every entry as "queued at - midnight". Same reason its None means UNKNOWN: an unreadable stamp must not age a paid, - outstanding snapshot out of the queue. - """ - import datetime as _dt - s = str(stamp or "").strip() - if not s: - return None - try: - t = _dt.datetime.fromisoformat(s.replace("Z", "+00:00")) - except ValueError: - return None - now = _dt.datetime.now(_dt.timezone.utc) - if t.tzinfo is None: - t = t.replace(tzinfo=_dt.timezone.utc) - return max(0, int((now - t).total_seconds() // 3600)) - - -def _gone_key(row, handle, platform=None): - """The identity a "this account does not exist" verdict is remembered under. - - `(platform, handle)` — wave 26 R4's dedup key, not the handle alone, because `@x` on - Instagram and `@x` on TikTok are two accounts. Blank platform means Instagram, which is what - every row on a preset profile table is and what `preset_cells` stamps. - - ⭐ WAVE 30 · T08 — `platform` OVERRIDES THE ROW, and the override is what makes the key honest - on a table the row cannot speak for. The fallback above reads the row's own `platform` cell, - which the TikTok runner writes — but only AFTER the first successful pull. A hand-typed row on - an arbitrary database that a TikTok step is bound to by NAME (`profile_field_key` step 1, - which is deliberately network-blind) carries no such cell, so a TikTok "this account does not - exist" verdict would have been filed under `instagram:`. A tombstone has no expiry - (W28/R9), so that would permanently suppress an Instagram read of a DIFFERENT person's - account with the same handle — silently, and without ever spending the money that would have - shown it was wrong. The caller knows which network it asked; the row does not. - """ - plat = str(platform or (row or {}).get("platform") or PLATFORM_INSTAGRAM).strip().lower() - return f"{plat}:{str(handle or '').strip().lstrip('@').lower()}" - - -def clear_gone(rt, table_key, handle, row=None): - """R9's re-arm door: forget the "this account does not exist" verdict for ONE handle. - - Returns the number of automations whose memory was changed — 0 is the ordinary answer and is - not an error, because most cell edits are not on a dead handle. - - ⭐⭐ WHY THIS IS A PUBLIC FUNCTION IN THE ENGINE RATHER THAN A LOOKUP IN THE ROW WRITER. - The verdict lives on the AUTOMATION (`state.enrichNotFound`), not on the row — it has to, - because it is a fact about what a run learned and paid for. But the thing that re-arms it is a - ROW event, and the row writer must not need to know the shape of an automation's state to - trigger it. So the seam is one call with the three things the writer already has, and every - walk of `all_definitions` stays on this side of the fence. - ⚠ IT MATCHES THE SAME WAY THE SKIP DOES. `_gone_key` is the one implementation of "which - handle is this", so a verdict can never be recorded under a key this cannot find - ([[one-evaluator-per-question]]). - - ⛔ WITHOUT ITS CALLER THIS IS INERT, AND THAT IS RECORDED RATHER THAN ASSUMED. R9's first - door — correcting a typo — works with no wiring at all, because the key IS the handle and a - different handle is simply not the one we recorded. This second door only opens when the row - write path calls it, which lives in `routes_tables.patch_row` (another session's fence). - Until that line lands, re-typing the SAME dead handle stays skipped - ([[flag-shipped-without-its-writer]] — named on purpose, so it is not discovered later). - """ - key = _gone_key(row or {}, handle) - if not str(handle or "").strip(): - return 0 - changed = 0 - for auto_id, defn in (all_definitions(rt) or {}).items(): - if str((defn.get("config") or {}).get("targetTable") or "") != str(table_key): - continue - known = dict(((defn.get("state") or {}).get("enrichNotFound")) or {}) - if key not in known: - continue - known.pop(key, None) - # ⚠ `or None` — an empty dict must clear the key rather than store `{}`, which is what - # every other state writer here does and what keeps a definition from growing a graveyard - # of empty maps. - set_state(rt, str(auto_id), {"enrichNotFound": known or None}) - changed += 1 - return changed - - -def _prime_enrich_batch(chosen, rows, profile_key, table, enrich, prefetch, - step=_no_step, log=print): - """Buy the selection's profiles in ONE vendor call per chunk. Returns how many resolved. - - ⭐⭐ 2026-08-09 — THE FIX FOR "WHY IS THIS ONE RECORD PER CALL". `bd_scrape` has always taken a - LIST of URLs and its own docstring names the failure mode ("…turning a 20-profile run into an - hour"); the only profile caller passed a single-element list from inside the per-record walk. - MEASURED on nurilab: a 25-record walk still running at 67 minutes, 25 billed snapshots, 25 - vendor emails. This runs once, where the whole chosen set and the row bodies are both in hand. - - ⛔ IT IS A FAST PATH AND MUST STAY ONE. Everything it fails to resolve is simply absent from - `prefetch`, and the per-record rung then behaves exactly as it does today — including its - corpus fallback and its Apify second opinion. A batch that cannot make the run WRONG is a - batch that can be shipped on a paid path; that is why the vendor call is wrapped and a failure - returns 0 instead of raising. - - ⛔ A DEFERRED CHUNK IS FANNED OUT PER DESTINATION, NOT PER SNAPSHOT. One snapshot id now - serves many records, so each pending task keeps its OWN `influencer`/`table`/`rowId` and they - share the id — `_pending_profile_tasks` is explicit that a profile cannot be resolved from the - handle alone ("two databases may both hold `@x`"). The handle is ALSO written into `prefetch` - behind `DEFERRED_MARK` so the walk does not buy the same snapshot a second time. - """ - want, dests, handles = [], {}, [] - for rid in (chosen or []): - row = rows.get(str(rid)) or {} - h = str(row.get(profile_key, "") or "").strip().lstrip("@").lower() - if not h or h in prefetch: - continue - want.append((h, str(rid))) - if h not in dests: - handles.append(h) - dests.setdefault(h, []).append(str(rid)) - if not handles: - return 0 - step(f"Reading {len(handles)} profile{'' if len(handles) == 1 else 's'} from the source") - deferrals = [] - try: - nodes, note = bd_profiles_batch(handles, deferred=deferrals) - except Exception as exc: # noqa: BLE001 — never fatal, see docstring - log(f"[aios-auto] profile batch failed, falling back to per-record reads: " - f"{type(exc).__name__}: {exc}") - return 0 - prefetch.update(nodes) - if note: - log(f"[aios-auto] profile batch note: {_s(note, 300)}") - for d in deferrals: - sid = str(d.get("snapshotId") or "") - if not sid: - continue - for u in (d.get("urls") or []): - h = str(ig_handle(str(u)) or "").strip().lstrip("@").lower() - if not h or h in nodes: - continue - prefetch[h] = {DEFERRED_MARK: sid} - for rid in dests.get(h, []): - enrich["pendingProfiles"].append( - {"snapshotId": sid, "datasetId": str(d.get("datasetId") or ""), - "urls": [str(u)], "kind": "profile", "influencer": h, - "table": table, "rowId": rid, "requestedAt": _iso()}) - return len(nodes) - - -def enrich_selection(rt, table_key, cfg, profile_key, today=None, gone=None, platform=None): - """⭐ 2026-08-07 (owner ruling) — WHICH records this enrich step spends on, in order. - - Returns `(ordered_row_ids, note)`. The note is the honest account of what the selection did - and is surfaced in the run summary; `""` means there is nothing worth saying. - - ⛔ **`limit` IS A QUOTA OF WORK DONE, NOT A WINDOW OF ROWS EXAMINED**, and that is the owner's - ruling in as many words: *"if a user choose to enrich 30 and from that sorted list of 30, 20 is - enriched last 30 days, then it goes to next list like this"*. So the walk CONTINUES past every - skipped record until 30 have actually been enriched or the list runs out. The naive reading — - take the top 30, then filter — would have billed for 10 and reported success, and the number in - the box would have meant something different every run depending on how much of the top of the - list happened to be fresh. A quota is predictable; a filtered window is not. - - The order of operations, each step narrowing the one above: - 1. the optional saved VIEW — resolved through `view_filter`, the SAME resolver `enters_view` - and `seed_rows` use. A view that has been deleted is a PROBLEM, never an empty tree: an - empty tree matches everything, so degrading would turn "enrich my shortlist" into "enrich - the entire database", at vendor prices. - 2. the SORT — the owner's "top N sorted by date", blanks always last in both directions - (a blank is unknown, not smallest). - 3. the COOLDOWN — skip anything enriched within N days, when the toggle is on. - 4. the QUOTA — stop at `limit`, itself clamped to `MAX_ENRICH_PER_RUN`. - - ⚠ A record with a BLANK handle is skipped and never counted against the quota — there is - nothing to enrich and it must not consume a slot somebody paid for. - """ - t = ut_get(rt, str(table_key or "")) - if t is None: - return [], f"{table_key!r} is not a database in this workspace" - rows = dict(t.get("rows") or {}) - notes = [] - - view_id = str(cfg.get("fromView") or "").strip() - if view_id: - tree, fields, problem = view_filter(rt, table_key, view_id) - if problem: - # ⛔ REFUSE, do not widen. See the docstring — this is the branch where a quiet - # fallback costs real money. - return [], f"{ENRICH_VIEW_UNREADABLE}. {problem}" - # ⭐⭐ 2026-08-07 (owner report) — `filter_eval.matches`, NOT `lane_match`. THIS LINE WAS - # A THIRD IMPLEMENTATION OF "does this row match", AND IT SPOKE THE WRONG LANGUAGE. - # - # `view_filter` returns a SAVED VIEW's filter tree — `{"nodes": [{colId, op, value}], - # "conj"}` in the GRID's dialect, whose operators are `contains/eq/neq/gt/gte/lt/lte/ - # isEmpty/isNotEmpty/between/within`. `lane_match` reads an AUTOMATION LANE condition — - # `{field, op, value}` with `=/!=/>/includes/is_empty/…` — and dispatches on - # `COND_GROUP_KEYS`. Handed a view tree it found no group key, read `raw.get("field")`, - # got `""`, and answered **False for every row**: MEASURED on the owner's own automation, - # a view matching exactly one record selected NOTHING, reported NO error, and the run - # committed `ok`. *"I just chose the View 'Enrichment test' where there is only 1 manual - # record... how come it says 51 records walked?"* - # - # ⛔ AND THE RIGHT EVALUATOR WAS ALREADY IN THE FILE. `_row_gate`'s `enters_view` branch - # resolves the SAME tree through `harness.filter_eval.matches(tree, row, fields)` and has - # always been correct. `_seed_event_state`'s own docstring states the law this line broke: - # *"Two implementations of 'does this row match' is how a seed disagrees with the edge it - # is supposed to arm."* There were three. Now there are two callers of one function. - # ⚠ `fields` is no longer discarded — `matches` needs the column TYPES to compare a date - # as a date and a number as a number, which is the half `lane_match` could not have had. - import harness.filter_eval as filter_eval - rows = {rid: r for rid, r in rows.items() if filter_eval.matches(tree, r, fields)} - - sort_field = str(cfg.get("sortField") or DEFAULT_ENRICH_SORT).strip() or DEFAULT_ENRICH_SORT - newest_first = str(cfg.get("sortDir") or "desc").strip().lower() != "asc" - # ⭐⭐ 2026-08-07 (owner ruling) — A MISSING DATE MEANS **NOW**, NOT "UNKNOWN". - # - # Owner: *"treat a missing first found as now, and manual entry should go first, instead of - # treated as last. I want to see my inayma manual entry works with automation enrichment."* - # And they are right about the semantics, not just the preference: on these tables the ONLY - # rows without a `first_found` are ones a PERSON typed, because the discovery runner stamps it - # on every row it writes. So a blank is not missing data — it is a row that was first found - # today, by the person sitting in front of it, and the one they most want enriched. - # - # The old rule sorted blanks LAST in both directions, so a hand-typed handle sat at position - # 41 of 41 and fell outside a limit of 25 — the row the whole feature exists for was the one - # it never reached. - # - # ⛔ DATE COLUMNS ONLY, and the narrowing is the honest half. "Blank means now" is a fact about - # a TIMESTAMP; a blank `followers` is not "the most followers", it is genuinely unknown, and - # treating it as the maximum would spend the budget on the rows we know least about. So a - # non-date sort keeps the old rule: unknown sorts last, in both directions. - # - # ⚠ NOTE THIS FALLS OUT OF THE KEY RATHER THAN BEING A SECOND PASS — `(0, value)` for a real - # date and `(1, "")` for a blank, with `reverse` doing the rest. Newest-first puts blanks at - # the front (they are "now"); oldest-first puts them at the back (they are still "now"). One - # rule, both directions, no branch that can disagree with itself. - ftype = str(((next((f for f in (t.get("fields") or []) - if str(f.get("key")) == sort_field), None)) or {}).get("type") or "") - if ftype == "date": - ordered = sorted(rows.items(), - key=lambda x: (0, str(x[1].get(sort_field) or "").strip()) - if str(x[1].get(sort_field) or "").strip() else (1, ""), - reverse=newest_first) - else: - have = [(rid, r) for rid, r in rows.items() if str(r.get(sort_field) or "").strip()] - blank = [(rid, r) for rid, r in rows.items() if not str(r.get(sort_field) or "").strip()] - have.sort(key=lambda x: _order_key(x[1].get(sort_field)), reverse=newest_first) - blank.sort(key=lambda x: _rid_num(x[0])) - ordered = have + blank - - try: - quota = int(cfg.get("limit") or DEFAULT_ENRICH_LIMIT) - except (TypeError, ValueError): - quota = DEFAULT_ENRICH_LIMIT - quota = max(1, min(quota, MAX_ENRICH_PER_RUN)) - if cfg.get("limit") and quota != int(cfg.get("limit") or 0): - notes.append(f"the limit was capped at {MAX_ENRICH_PER_RUN} for one run") - - cooling = bool(cfg.get("skipRecent")) - try: - days = int(cfg.get("skipRecentDays") or DEFAULT_ENRICH_COOLDOWN_DAYS) - except (TypeError, ValueError): - days = DEFAULT_ENRICH_COOLDOWN_DAYS - days = max(1, days) - - picked, cooled, blank_handle = [], 0, 0 - # ⭐⭐ 2026-08-09 — HANDLES A VENDOR HAS ALREADY SAID DO NOT EXIST. - # - # ⛔ THE DEFECT THIS CLOSES IS STRUCTURAL, AND IT IS THE WORD "AGAIN" IN THE OWNER'S REPORT. - # A blocked read writes NO cells, so `enriched_at` stays unset, so `_days_since(None)` is - # None, so the cooldown above can never exclude the row — while `Followers is empty` keeps it - # in the Pending cohort by construction. MEASURED on nurilab: one dead handle, re-bought at - # 06:00 on three consecutive days, reported each time as an opaque "1 blocked". No vendor fix - # removes that loop; only a memory of the verdict does. - # - # ⚠ THEY ARE **NAMED**, NOT SILENTLY DROPPED. The whole point is that the owner can act — the - # note goes into the run every single time, not only on the run that discovered it, because a - # run that quietly reports "0 records walked" tomorrow puts them straight back at "wtf". - skipped_gone = [] - for rid, r in ordered: - if len(picked) >= quota: - break - raw_handle = str(r.get(profile_key) or "").strip() - if not raw_handle: - blank_handle += 1 - continue - # ⛔ R9 — NO EXPIRY. There is deliberately no date arithmetic here any more: a verdict is - # a verdict until a human edits the cell. An `at` stamp is still STORED (it is what the - # owner reads to know when we last paid to be told this), it is simply not a clock. - # ⚠ WAVE 30 · T08 — the SAME `platform` the runner will file a new verdict under, so the - # skip and the write cannot disagree about which account a tombstone belongs to. - if isinstance((gone or {}).get(_gone_key(r, raw_handle, platform)), dict): - skipped_gone.append(raw_handle) - continue - if cooling: - since = _days_since(r.get("enriched_at"), today=today) - if since is not None and since < days: - cooled += 1 - continue - picked.append(str(rid)) - - # ⛔ THE HONEST ACCOUNT. "10 enriched" reads as success whether the quota was 10 or 30, so the - # run says when it could NOT fill the quota and why — the same disclosure rule `cap_note` and - # `run_plain` follow. Silence here would make a shrinking selection invisible. - if cooled: - notes.append(f"{cooled} skipped as enriched in the last {days} days") - if skipped_gone: - shown = ", ".join(f"@{h}" for h in skipped_gone[:5]) - # ⚠ THE SENTENCE IS THE FEATURE. It must name the handles AND the two things a person can - # do, because nothing else will ever retry them — under R9 this note is the only path - # back from a tombstone, so a vaguer version would strand the row permanently. - notes.append(f"{len(skipped_gone)} skipped because Instagram has no such account " - f"({shown}{', …' if len(skipped_gone) > 5 else ''}). Delete the row or " - f"correct the handle; they are not retried automatically") - if blank_handle: - notes.append(f"{blank_handle} skipped with no handle") - if len(picked) < quota and (cooled or blank_handle or skipped_gone or ordered): - notes.append(f"{len(picked)} of the {quota} asked for. The list ran out") - return picked, "; ".join(notes) - - -def _has_action(actions, kind): - """Does this flow contain `kind` ANYWHERE, forks included? - - ⛔ FORKS ARE THE WHOLE REASON THIS IS A FUNCTION. A group's children live under - `config.branches[].actions` (C-FORK), so a flat scan of the top level answers False for an - enrich step somebody put inside an If — and the caller would then skip a schema top-up the - run genuinely needs. Same walk `mapTree` does on the client, and the same trap wave 24 - recorded when four hand-rolled walks all forgot to descend. - """ - for a in (actions or []): - if not isinstance(a, dict): - continue - if a.get("kind") == kind: - return True - for br in ((a.get("config") or {}).get("branches") or []): - if _has_action((br or {}).get("actions") or [], kind): - return True - return False - - -def _actions_of_kind(actions, kind): - """Every action of `kind`, including actions nested inside If branches.""" - out = [] - for action in actions or []: - if not isinstance(action, dict): - continue - if action.get("kind") == kind: - out.append(action) - for branch in ((action.get("config") or {}).get("branches") or []): - out.extend(_actions_of_kind((branch or {}).get("actions") or [], kind)) - return out - - -def _web_agent(): - """C5's seam, resolved LAZILY — `web_agent.run_step(step, ctx) -> (dict|None, str)`. - - ⚠ IMPORTED INSIDE THE CALL, like every `connectors_tt` site in this module, and here it also - buys a failure mode worth having: if `web_agent.py` is ever missing from a deployment, the - engine still imports and every OTHER action still runs — the web step alone reports a sentence. - A module-level import would turn one absent file into a dead automation module. - - ⛔ AND THE ABSENCE IS REPORTED, NEVER SWALLOWED (R6's second sentence). The shim below answers - the same `(None, sentence)` contract the real seam does, so the caller's `if why:` branch is - the only branch there has ever been. - """ - try: - import web_agent - return web_agent - except Exception as exc: # noqa: BLE001 - class _Absent: - @staticmethod - def run_step(_step, _ctx=None): - return None, ("The web-browsing agent is not available in this deployment " - f"({type(exc).__name__}). Nothing was read.") - return _Absent - - -def apply_actions(rt, defn, table_key, row_ids, username="automation", log=print, - step=_no_step): - """Walk `flow.actions` for each of `row_ids`, then apply the ENDING. Returns the run counts. - - ⚠ The ending runs even when there are NO actions, and that is not a detail: a discovery - automation's review stage comes from `stages_for`, not from actions, - so an early return on an empty action list would have silently disabled the owner's loop - feature on the flagship flow — the one kind most likely to want `auto_reset`. Caught in - review; the tests all happened to pass a non-empty action list, which is why it was green. - """ - defn, _retired = _without_retired_board(defn) - flow = (defn or {}).get("flow") or {} - actions = flow.get("actions") or [] - # ⛔ THE PINNED STEP IS A PICTURE OF THE ENGINE'S OWN WRITE, NOT A SECOND ONE. - # - # MEASURED 2026-08-06: a discovery run wrote its candidates through `ut_write_rows` (the - # "Save results" node) and THEN walked the flow over the rows it had just written — where - # the seeded `create_record` inserted every one of them AGAIN. Two rows became four, on - # every run, silently, since wave 24 seeded that action. Making step 1 permanent (owner - # ruling, same day) would have made the duplication permanent and unavoidable with it. - # - # ⚠ THE CARD STAYS. It is what the owner asked for and it is honest — "this search puts what - # it finds in a database" is exactly what the engine does. What it must not do is do it - # twice. Skipped HERE rather than not-seeded, so the builder still shows the step and the - # server still refuses to let it be removed. - if ig_action_pinned(defn, 0) and actions: - actions = actions[1:] - # ⭐ `webRead` / `webBlocked` JOIN HERE (W31-T38, C5): declared up front like every other - # counter, so a run that read nothing and a run with no web step are DIFFERENT numbers. R6's - # second sentence is a reporting rule, and a counter that only exists once it is non-zero - # cannot report a refusal. - counts = {"actionsRun": 0, "updated": 0, "created": 0, "found": 0, - "webRead": 0, "webBlocked": 0} - #: W31-T38 — the once-per-run markers the web arm uses, so an unconfigured step says so ONCE - #: rather than once per record. Same shape as the enrich accumulators' `notes`. - web_notes = [] - # ⭐ C4 — THE ENRICH ACCUMULATOR. Every append row the enrich action produces is collected - # across the WHOLE walk and written once at the end, for the reason the module header states - # and `run_field_instagram` learned the hard way: `upsert_rows` rebuilds the row dict on - # entry, so calling it per record is O(existing) per record — survivable at 5,000 rows and an - # automation that never finishes at `MAX_UT_IG_ROWS`. One flush, three upserts. - enrich = {"snaps": [], "posts": [], "psnaps": [], "comments": [], "pending": [], - # ⭐ 2026-08-09 — deferred PROFILE snapshots (paid, still building at the vendor) - # and handles a vendor has said do not exist. Both are collected across the whole - # walk and written once, for the same reason every other list here is. - "pendingProfiles": [], "gone": {}, - "profiles": 0, "ok": 0, "blocked": 0, "notes": [], "dry": False} - # ⭐⭐ WAVE 30 · T08 — THE TIKTOK ACCUMULATOR IS ITS OWN, and the separation is the design - # decision rather than a convenience. `_enrich_flush` is not network-agnostic: it calls - # `ensure_ig_graph`, addresses the four `IG_*_TABLE` constants and writes through to the - # platform Instagram master (`ig_master.append_run`). Routing TikTok rows through it would - # append a TikTok creator's followers into Instagram's pooled history, which the metric - # FIELDS read (W29-T34) — a wrong number in a permanent series, i.e. the same failure class - # as `preset_cells`' unconditional `PLATFORM_INSTAGRAM` stamp that `tt_preset_cells` exists - # to avoid. - # ⚠ AND THE KEYS IT FLUSHES ARE PREFIXED `tt…` FOR A MEASURED REASON: `apply_actions` merges - # both flushes into ONE `counts` dict, so a shared `enriched`/`enrichBlocked` key would let a - # flow carrying BOTH an Instagram and a TikTok step report one network's numbers under the - # other's name, last writer winning, with no error anywhere. - # ⭐ W30 · D-156 — `pending` is the TikTok twin of Instagram's, and it is a RUN-level list for - # the same reason theirs is: a media snapshot belongs to a HANDLE, not to a row, so nothing - # about it needs the per-record sink that `pendingProfiles` needs. - tt = {"snaps": [], "posts": [], "psnaps": [], "comments": [], "pending": [], - "profiles": 0, "ok": 0, "blocked": 0, "notes": [], "dry": False} - # The verdicts this automation already holds, read ONCE — `enrich_selection` consults them - # per record and re-reading the definition per row would be a store read per record. - known_gone = dict(((defn or {}).get("state") or {}).get("enrichNotFound") or {}) - # ⭐ 2026-08-07 — THE SELECTION, RESOLVED ONCE PER ACTION AND NOT ONCE PER RECORD. - # `enrich_selection` sorts and scans the whole table; doing that inside the per-record walk - # would be O(rows²) and, worse, would re-answer "which 25 records" every time a record walked - # through — so the quota could never be honoured. Keyed by ACTION id, because two enrich steps - # in one flow are two independent budgets. - enrich_plan = {} - # ⭐⭐ 2026-08-09 — `{handle: node}` FOR THE WHOLE SELECTION, BOUGHT IN ONE CALL PER CHUNK. - # The vendor read used to happen inside the per-record walk, one URL per `/v3/scrape` and a - # `PACE_SECONDS` floor between records, so a 25-record walk was 25 billed snapshots and was - # MEASURED still running at 67 minutes. `bd_scrape` always accepted a list; nothing ever - # handed it one. Run-scoped rather than per-action: two enrich steps over the same table are - # two budgets (`enrich_plan`) but the same profiles, and a handle already bought is not worth - # buying twice. - enrich_prefetch = {} - if not table_key: - return counts - table = str(table_key) - # ⭐⭐ 2026-08-07 (owner report) — THE PRESET COLUMNS MUST EXIST BEFORE THE CELLS ARE WRITTEN. - # - # ⛔ MEASURED on the owner's own row: a successful enrich reported `enriched: 1` and filled - # NINE cells out of a pull that carried far more, because `_act_row_patch` can only write a - # cell whose COLUMN is declared — `user_tables` filters an unknown key on every write door. - # So every preset the target table did not already happen to have was silently dropped, and - # the run still said ok. Owner: *"UNTIL I SEE THAT ALL OF INAYMA'S INSTAGRAM FIELDS GET - # FILLED"* — this is the half that was swallowing them. - # - # ⚠ WHY IT WAS MISSING RATHER THAN BROKEN: the columns are spawned by `_presets_after_write`, - # which runs when a **Create record** action is SAVED (W25/R10). A `plain` automation whose - # only step is `enrich_instagram` never saves one, so nothing had ever declared them — the - # feature worked on discovery tables purely because discovery creates them for its own reasons. - # - # ⚠ ONCE PER RUN, not once per record, and only when an enrich step is actually present: - # `ut_ensure` MERGES by key and returns without a write when nothing is new, so the steady - # state is one dict comparison. Guarded by the same walk that would use it, so a flow with no - # enrich step never touches the schema of the database it walks. - # ⛔ ONLY ON A TABLE THAT IS ALREADY BOUND, and the narrowing is the careful half. The preset - # set declares `handle` with `pinned` AND `profile: {source: 'instagram'}` — so topping up an - # ARBITRARY database would (a) reshape someone's schema with 36 columns because they pointed a - # step at it, and (b) add a SECOND profile column to a table that already flagged a different - # one, which `user_tables` forbids at both write doors. A table with no profile column is an - # UNBOUND enrich: it must stay unbound and say so, which is a different defect (D-79) with its - # own honest refusal, not something to paper over by inventing the binding. - _tbl_now = ut_get(rt, table) or {} - _enrich_actions = _actions_of_kind(actions, "enrich_instagram") - _bound = next((f for f in (_tbl_now.get("fields") or []) - if isinstance(f.get("profile"), dict)), None) - if _bound is None: - _named = next((str((a.get("config") or {}).get("profileField") or "").strip() - for a in _enrich_actions - if str((a.get("config") or {}).get("profileField") or "").strip()), "") - _bound = next((f for f in (_tbl_now.get("fields") or []) - if str(f.get("key") or "") == _named), None) - if _bound and _enrich_actions: - # ⚠ AND THE DECLARATIONS ARE STRIPPED FROM ANYTHING NEW. The binding already exists — - # `_bound` is it — so a preset arriving now must carry DATA, never a second identity: a - # table whose profile column is `ig_handle` would otherwise gain a rival `handle`. - topup = [({k: v for k, v in f.items() if k not in ("profile", "pinned")} - if f.get("key") != _bound.get("key") else dict(f)) - for f in PRESET_PROFILE_FIELDS] - try: - if any(not bool((a.get("config") or {}).get("dryRun")) for a in _enrich_actions): - ensure_ig_graph(rt, username, str(defn.get("id") or ""), - profile_table=table, profile_field=str(_bound.get("key") or "")) - else: - # Dry run keeps its original no-create contract; only the already-established - # Profile table is topped up, and no canonical related database is spawned. - ut_ensure(rt, _tbl_now.get("label") or table, topup, username, key=table, - flow_tag=str(defn.get("id") or ""), lock_fields=True) - except Exception as exc: # noqa: BLE001 - # A schema top-up that cannot run must not stop the enrichment: the cells that DO - # have columns still land, which is strictly better than the pull being thrown away. - log(f"[aios-auto] preset top-up on {table} failed: {type(exc).__name__}: {exc}") - # ⭐⭐ WAVE 30 · T08 — THE SAME TOP-UP FOR TIKTOK, AND THE SAME NARROWING: only on a table that - # is ALREADY BOUND. Separate from the block above because every constant in it is Instagram's - # (`PRESET_PROFILE_FIELDS`, `ensure_ig_graph`) and because the binding is resolved by a - # DIFFERENT question — `profile_field_key(..., source=PROFILE_SOURCE_TT)`, so a TikTok step - # cannot adopt an Instagram column and go on to ask a TikTok dataset about an Instagram handle. - # ⚠ It runs on a dry run too, exactly as the Instagram branch does: the target's own columns - # are topped up, and the related append database is NOT spawned (that is the flush's job, and - # it returns before writing anything on a dry run). - _tt_actions = _actions_of_kind(actions, "enrich_tiktok") - if _tt_actions: - _tt_named = next((str((a.get("config") or {}).get("profileField") or "").strip() - for a in _tt_actions - if str((a.get("config") or {}).get("profileField") or "").strip()), "") - _tt_bound = profile_field_key(_tbl_now, _tt_named, source=PROFILE_SOURCE_TT) - if _tt_bound: - try: - ut_ensure(rt, _tbl_now.get("label") or table, - _tt_profile_schema_for(_tt_bound), username, key=table, - flow_tag=str(defn.get("id") or ""), lock_fields=True) - except Exception as exc: # noqa: BLE001 - log(f"[aios-auto] TikTok preset top-up on {table} failed: " - f"{type(exc).__name__}: {exc}") - rows = dict((ut_get(rt, table) or {}).get("rows") or {}) - if not rows: - return counts - # ⛔ NOTHING HERE TRACKS WHERE A RECORD "GOT TO" ANY MORE (wave 27 item 12, ruling R3). The - # walk used to keep a `reached` map and stamp each record with the furthest action it made — - # a board column's worth of bookkeeping written into the tenant's own table on every run. - # The board is deleted, so the stamp had no reader, and a machine column nobody reads is a - # cell the customer has to look at and a store commit we pay for on every scheduled run. - # The RUN LOG is the operational record of what happened now, and it is the only one. - patches, creates = {}, {} - - #: ⭐⭐ W33-T58 · D-191 — this record's web results, keyed by ACTION ID, and the run's job - #: count. `web_done` is cleared per record (a batch is built from one row's interpolated - #: values and means nothing for the next one); `web_jobs` counts JOBS across the whole run, - #: which is what `MAX_WEB_JOBS_PER_RUN` has always been trying to bound. - #: ⚠ A key present with `None` means "this step was in a batch that refused" — distinct from - #: absent, which means "no batch has covered it yet". Collapsing the two would re-run the - #: whole batch once per step of it, which is the defect this fixes, inverted. - web_done: dict = {} - web_jobs = [0] - - def _web_step_dict(act, row): - """One action + one record → the step dict the seam takes. Interpolated HERE, as before. - - ⚠ `interpolate` on the CALLER's side, exactly as the update/create arms do it, so - `{{Field}}` works in a url, a selector or a typed value. The seam takes a plain dict and - does not know about rows. - """ - cfg = act.get("config") or {} - wait = cfg.get("waitFor") - step = {"kind": act.get("kind"), "id": str(act.get("id") or ""), - "url": interpolate(str(cfg.get("url") or ""), row), - "selector": interpolate(str(cfg.get("selector") or ""), row), - "attr": str(cfg.get("attr") or "text"), - "all": bool(cfg.get("all")), - "waitFor": interpolate(str(wait), row) if wait else None, - "timeoutMs": int(cfg.get("timeoutMs") or 20000)} - # ⚠ ADDED ONLY WHEN PRESENT: the seam distinguishes a key that is absent from one that is - # empty, and an empty `value` on a `web_read` would be a typed blank. - if cfg.get("value"): - step["value"] = interpolate(str(cfg.get("value")), row) - if cfg.get("hint"): - step["hint"] = interpolate(str(cfg.get("hint")), row) - if cfg.get("secret"): - step["secret"] = True - if cfg.get("dryRun"): - step["dryRun"] = True - return step - - def _run_web_batch(acts, start, row, rid): - """Run the longest safe run of consecutive web steps from `acts[start]` in ONE job. - - ⛔ **CONSECUTIVE, AND ONLY WHILE NOTHING IN THE BATCH DEPENDS ON THE BATCH.** `run_plan` - sends every step to one browser at once, so a step whose url/selector/value interpolates a - column an EARLIER step in the same batch writes would be interpolated against the value - that column had BEFORE the batch ran. That is a wrong answer rather than a slow one, so the - batch is cut immediately before any such step and the remainder becomes the next batch. - The single-step case is then exactly the old behaviour, which is what makes this safe to - land on a live flow. - ⚠ A step that is disabled, filtered out by its `when`, or unconfigured ENDS the batch - rather than being skipped inside it: each of those is a reason this record does not run - that step, and the loop's own arms already report them one at a time with their own - sentences. Ending here keeps exactly one place that decides what a blocked step says. - """ - batch, produced = [], set() - for act in acts[start:]: - kind = act.get("kind") - if kind not in WEB_KINDS or not act.get("enabled", True): - break - if not lane_match(act.get("when"), row): - break - cfg = act.get("config") or {} - if _web_missing(kind, cfg): - break - # The dependency cut. `interpolate` reads `{{Name}}`; a step naming a column an - # earlier step in THIS batch writes has to wait for the next job. - refs = " ".join(str(cfg.get(k) or "") for k in ("url", "selector", "value", "hint")) - if any(("{{" + f) in refs or ("{{ " + f) in refs for f in produced): - break - batch.append(act) - if str(cfg.get("field") or ""): - produced.add(str(cfg.get("field"))) - if len(batch) >= MAX_WEB_STEPS_PER_JOB: - break - if not batch: - return - steps = [_web_step_dict(a, row) for a in batch] - # ⛔ THE SEAM REFUSES A JOURNEY WHOSE FIRST STEP CARRIES NO ADDRESS — there is no page to - # act on yet — and a refusal is for the WHOLE plan. Batching a url-less first step would - # therefore take its followers down with it, where one-job-per-step only lost that step. - # A batch that cannot start is cut to one, which is exactly the old behaviour. - if not str(steps[0].get("url") or "").strip(): - batch, steps = batch[:1], steps[:1] - web_jobs[0] += 1 - - def _block(a, why_one): - web_done[str(a.get("id") or "")] = None - counts["webBlocked"] += 1 - if why_one: - log(f"[aios-auto] {a.get('kind')}: {why_one}") - - # ⛔ THE SEAM'S OWN BOUNDARY IS ON `run_step`, NOT ON `run_plan` — `run_step` wraps its - # call in `try/except` and calls that "the LAST boundary". Calling `run_plan` directly - # steps around it, and this code runs inside a record walk where an escaping exception - # ends the whole run. So the boundary moves here with the call. - try: - rows_out, why = _web_agent().run_plan( - steps, {"tenant": str(defn.get("tenant") or ""), - "automationId": str(defn.get("id") or ""), - "runId": str(defn.get("id") or ""), "log": log}) - except Exception as exc: # noqa: BLE001 — the LAST boundary - why, rows_out = (f"The web steps failed unexpectedly ({type(exc).__name__}: " - f"{str(exc).splitlines()[0][:200]}). Nothing was read."), None - if why: - log(f"[aios-auto] web: {why}") - for a in batch: - _block(a, "") - return - # ⚠ MATCHED BY ID, NEVER BY POSITION. The job returns a row for a step that FAILED and for - # every step after it that was never attempted, so the list can be shorter than, or - # misaligned with, the plan — and a positional read would hand step 3's caller step 2's - # answer, writing a wrong value into a real column, which is worse than the missing one it - # replaced. `_clean_step` mints an id for every step and the runner echoes it back, so the - # id is carried the whole way and is the only thing worth matching on. - by_id = {str(r.get("id") or ""): r for r in (rows_out or []) if isinstance(r, dict)} - for a in batch: - hit = by_id.get(str(a.get("id") or "")) - # ⛔ `ok` IS CHECKED HERE BECAUSE `run_step` USED TO CHECK IT. It turned a row with - # `ok:false` into a sentence and returned no result; reading `hit` without that test - # would take a failed step's empty `value` and write it over a real cell. - if not hit or not hit.get("ok"): - _block(a, str((hit or {}).get("error") or "") - or "The browser job returned nothing for this step.") - continue - web_done[str(a.get("id") or "")] = hit - - def _walk(acts, row, rid, depth=0): - """Walk the actions in order for ONE record. - - ⚠ THE RETURN VALUE IS VESTIGIAL and is deliberately kept as `False`. It used to mean - "this record was SUSPENDED by a review gate" — the one branch that could stop a walk - early. With review retired nothing suspends anything, so every record walks its whole - flow; the signature stays so a future gate-style action has an obvious place to say so. - """ - for idx, act in enumerate(acts): - if not act.get("enabled", True): - continue - if not lane_match(act.get("when"), row): - continue - kind = act.get("kind") - cfg = act.get("config") or {} - counts["actionsRun"] += 1 - if kind == "group": - # ⭐ WAVE 24 · C-FORK — FIRST MATCHING BRANCH WINS, and only that one runs. - # Declaration order is priority order, the same rule `route_record` applies to - # the board's lanes, and the Otherwise leg (`cond: null`) is simply the branch - # nothing above it beat — `lane_match(None, row)` is True, and `clean_actions` - # has already guaranteed a null condition can only be LAST. - for br in group_branches(act): - if lane_match(br.get("cond"), row): - if _walk(br.get("actions") or [], row, rid, depth + 1): - return True - break - elif kind == "update_record": - vals = {k: interpolate(v, row) for k, v in (cfg.get("values") or {}).items()} - row.update(vals) # later actions see the write, as they must - _act_row_patch(patches, table, rid, vals) - counts["updated"] += 1 - elif kind == "create_record": - target = str(cfg.get("table") or "") - vals = {k: interpolate(v, row) for k, v in (cfg.get("values") or {}).items()} - # C5: keyed by (table, uniqueOn) rather than by table alone, because two actions - # may legitimately write to ONE database on different keys — collapsing them onto - # the table would silently apply one action's uniqueness rule to the other's rows. - creates.setdefault((target, str(cfg.get("uniqueOn") or "")), []).append(vals) - counts["created"] += 1 - elif kind == "send_statement": - # ⭐⭐ WAVE 35 · T35 / R10 — THE ARM LANDS WITH THE CATALOG ROW, ON PURPOSE. - # - # `_walk` has no terminal `else` (see the note on `enrich_tiktok` in the catalog): - # an unknown kind is walked, COUNTED, reports the run `ok` and writes nothing. So a - # catalog row whose arm arrives in a later ticket is addable, storable and silently - # inert — a step a person configured, that reports success and does nothing. This - # arm exists so that window never opens. - # - # ⛔ T35 DOES NOT SEND AND DOES NOT PARK. R10's review stage is W35-T36; until it - # lands this says so out loud and counts the record as blocked, which is the same - # shape `ai_agent` uses for a step it cannot perform. It must never fall through to - # "ok". - # ⛔ AND IT NEVER SENDS FROM HERE, in this wave or any later one. The send door is - # `routes_statements`, behind SAFE_MODE + `admin_gate` + the tenant gate, reached by - # a human click on the review batch. This arm's whole job is to PREPARE. - if "send_statement_pending" not in web_notes: - web_notes.append("send_statement_pending") - log("[aios-auto] send_statement: statements are assembled for review, not " - "sent. The review batch is not configured yet, so nothing was prepared " - "and nothing was sent.") - counts["webBlocked"] += 1 - continue - elif kind == "odoo_sync": - # Same refusal, same reason as the arm below: a catalog row with no arm is walked, - # counted, and reports success having done nothing. The connector owns this sync. - if "odoo_sync_not_run_here" not in web_notes: - web_notes.append("odoo_sync_not_run_here") - log("[aios-auto] odoo_sync: the Odoo pull runs on the connector's own " - "schedule, not from this canvas. Nothing was done") - counts["webBlocked"] += 1 - continue - elif kind == "ai_enrich": - # ⛔⛔ A REFUSAL ARM, AND IT EXISTS FOR THE REASON THE `send_statement` ARM ABOVE - # STATES: `_walk` has no terminal `else`, so a catalog row whose arm is missing is - # walked, COUNTED, reports the run `ok` and writes nothing — a step somebody - # configured that succeeds at doing nothing. Adding the row (D-277) without this - # arm would have opened exactly that window. - # ⚠ AND THE REFUSAL IS THE TRUTH, not a stub. An AI column is filled by - # `ai_enrich`, driven from the column's own editor; the synthetic agent row this - # kind appears in is DERIVED from that column and is never stored, so nothing - # reaches here through the ordinary path. If something ever does, it must say so - # rather than report success. - if "ai_enrich_not_run_here" not in web_notes: - web_notes.append("ai_enrich_not_run_here") - log("[aios-auto] ai_enrich: an AI column is filled from the column's own " - "editor, not from this canvas. Nothing was done") - counts["webBlocked"] += 1 - continue - elif kind == "ai_agent": - # ⭐⭐ W33-T56 (owner item 7, ruling R3) — THE FUZZY STEP, AT RUN TIME. - # - # A description becomes concrete web steps HERE, against this record's own values, - # and then rides the ordinary seam. Composing at run time rather than at save time - # is the whole point: `{{Website}}` is a different page for every row, so a journey - # fixed at save time would be the same guess repeated. - # ⛔ IT COMPOSES ONLY `web_*` KINDS. The composer is handed the same catalog the - # AI-agent module uses, filtered to what a browser job can perform — so a fuzzy - # instruction cannot talk this action into writing a record or calling a connector. - # The blast radius of a bad sentence is one browser session, not the tenant. - # ⚠ AND IT REPORTS THE STEPS IT ACTUALLY TOOK, which is the ticket's own - # `done-when`. A step that composes a journey and reports only its final value is - # unauditable: nobody can tell a right answer from a lucky one. - _missing = _web_missing(kind, cfg) - if _missing: - _note = f"{kind}_unconfigured" - if _note not in web_notes: - web_notes.append(_note) - log(f"[aios-auto] {kind}: this step still needs " - + ", ".join(_missing) + ". Nothing was done") - counts["webBlocked"] += 1 - continue - if web_jobs[0] >= MAX_WEB_JOBS_PER_RUN: - if "web_cap" not in web_notes: - web_notes.append("web_cap") - log(f"[aios-auto] web: this run stopped after {MAX_WEB_JOBS_PER_RUN} " - f"browser jobs of about 10-30 seconds each.") - counts["webBlocked"] += 1 - continue - # W35 · C7: `st` + `user` so the model spend is attributed (`NOTE E-16`). - _plan, _why = _ai_agent_plan(cfg, row, log, st=rt, - user=str(defn.get("createdBy") or "")) - if _why: - # ⛔ NAMED, NEVER OPAQUE — the second half of the `done-when`. "The assistant - # could not work out how to do that" with the reason attached is actionable; - # a blank cell is not. - log(f"[aios-auto] ai_agent: {_why}") - counts["webBlocked"] += 1 - continue - web_jobs[0] += 1 - try: - _rows_out, _why2 = _web_agent().run_plan( - _plan, {"tenant": str(defn.get("tenant") or ""), - "automationId": str(defn.get("id") or ""), - "runId": str(defn.get("id") or ""), "log": log}) - except Exception as _exc: # noqa: BLE001 — the LAST boundary - _rows_out, _why2 = None, ( - f"the browser job failed unexpectedly ({type(_exc).__name__}: " - f"{str(_exc).splitlines()[0][:200]}). Nothing was done.") - if _why2: - log(f"[aios-auto] ai_agent: {_why2}") - counts["webBlocked"] += 1 - continue - # THE ACCOUNT OF WHAT IT DID — one line per step, in order, with each step's own - # verdict. This is what makes a fuzzy step auditable at all. - _done = [r for r in (_rows_out or []) if isinstance(r, dict)] - for _i, _r in enumerate(_done, 1): - log(f"[aios-auto] ai_agent step {_i}/{len(_plan)}: {_r.get('kind')} " - f"{'ok' if _r.get('ok') else 'FAILED'}" - + (f". {str(_r.get('error'))[:160]}" if not _r.get("ok") else "")) - _last = _done[-1] if _done else {} - if not _done or not _last.get("ok"): - log("[aios-auto] ai_agent: the journey did not finish. " - + str((_last or {}).get("error") - or "the browser job returned nothing for the last step")) - counts["webBlocked"] += 1 - continue # ⛔ NOTHING IS WRITTEN on an unfinished journey. - _target = str(cfg.get("field") or "") - if _target: - vals = {_target: _last.get("value")} - row.update(vals) - _act_row_patch(patches, table, rid, vals) - counts["webRead"] += len(_done) - elif kind in WEB_KINDS: - # ⭐⭐ WAVE 31 · T38 (C5) — THE WEB-BROWSING AGENT'S LIVE ARM, ALL FIVE KINDS. - # - # ⛔ WHY THIS ARM EXISTS SEPARATELY FROM THE RUNNER: session E built - # `web_agent.run_step` and **could not verify its own mounting**. An unmounted - # runner is a whole, correct, unreachable feature — the exact class five wave-29 - # features shipped as — so the mount and its `verify_wiring` row are C's, in one - # change. - # ⭐⭐ W31 QA WIDENED THIS ARM FROM `web_read` TO ALL FIVE KINDS on the owner's - # revocation of D-51/R5 (`TICKETS.md:1418`). ⚠ IT NEEDS NO PER-KIND HANDLING, and - # that is E's design rather than an omission: the runner normalises EVERY kind to - # set `result["value"]` (read → the text · goto → the title · click → the title it - # landed on · fill → the typed value, masked when secret · repair → the proposed - # selector), precisely so this one arm does not grow a switch that would be a - # second copy of the runner's table living in another lane's file. - # - # ⛔ IT BLOCKS FOR ~9-32 s (E measured it; `proto/web-agent-job.md` §4). That is - # tolerable HERE and nowhere else: this is the automation RUNNER, already a - # background walk. It must never be called from a route a person is waiting on. - # - # ⚠ `interpolate` ON THE CALLER'S SIDE, exactly as the update/create arms do it, so - # `{Field}` works in a url or a selector. E's seam takes a plain dict and does not - # know about rows. - # ⛔ FAIL CLOSED, ONCE, WITH THE REASON — the counterpart to the validator storing - # an unconfigured step (see `_clean_action_config`'s `web_read` arm). Reported once - # per RUN and not per record: the missing config is a property of the flow, so a - # 100-record walk would otherwise print the same sentence a hundred times and bury - # everything else. The same shape the enrich arm's `unbound` note uses. - # ⚠ PER KIND, because they do not need the same things: `web_goto` needs a url and - # no selector; `web_fill` needs a value nobody else takes; only `web_read` needs a - # column to write into. One shared three-field test would have blocked every - # `web_goto` ever configured for want of a selector it does not use. - _missing = _web_missing(kind, cfg) - if _missing: - # ⚠ THE NOTE KEY CARRIES THE KIND. It used to be the literal - # `"web_read_unconfigured"`, so a flow with an unconfigured `web_goto` AND an - # unconfigured `web_fill` would have reported the first and swallowed the - # second — once-per-RUN is the property, not once-per-FLOW. - _note = f"{kind}_unconfigured" - if _note not in web_notes: - web_notes.append(_note) - log(f"[aios-auto] {kind}: this step still needs " - + ", ".join(_missing) + ". Nothing was done") - counts["webBlocked"] += 1 - continue - # ⛔⛔ THE PER-RUN JOB CEILING (E-4). A browser job is ~9-32 s against HF's - # 6-concurrent cap — a flow over a few thousand rows would submit a few thousand - # jobs and run for days. Reported ONCE with its cause AND the fix, which is R6's - # second sentence: a limit that cannot be removed today must say why and what would - # remove it. `MAX_WEB_JOBS_PER_RUN` carries the reasoning. - # ⭐⭐ W33-T58 (D-191) — THE CEILING NOW COUNTS **JOBS**, NOT PAGES, because the - # two stopped being the same thing on the line below. A record whose three web - # steps batch into one job spends ONE of these, not three. - if web_jobs[0] >= MAX_WEB_JOBS_PER_RUN: - if "web_cap" not in web_notes: - web_notes.append("web_cap") - log(f"[aios-auto] web_read: this run stopped after " - f"{MAX_WEB_JOBS_PER_RUN} browser jobs of about 10-30 seconds each, " - f"which do not run in parallel. A record's consecutive web steps " - f"already share ONE job; to read more, narrow the flow's records.") - counts["webBlocked"] += 1 - continue - # ⭐⭐ W33-T58 · D-191 — ONE JOB FOR A RECORD'S CONSECUTIVE WEB STEPS. - # `run_plan(steps, ctx)` has always taken a list and nothing ever called it with - # more than one: the arm called `run_step`, which wraps `[step]`, so a flow with - # three web reads paid THREE ~9 s cold starts to do what one job does. The batch is - # built at WALK time rather than from the stored flow, because whether a step runs - # at all depends on this record (`enabled`, `when`, and whether it is configured). - if str(act.get("id") or "") not in web_done: - _run_web_batch(acts, idx, row, rid) - result = web_done.get(str(act.get("id") or "")) - if result is None: - continue # its batch refused; the reason was logged once, there - _target = str(cfg.get("field") or "") - if _target: - vals = {_target: (result or {}).get("value")} - row.update(vals) # later actions see the write, as they must - _act_row_patch(patches, table, rid, vals) - counts["webRead"] += 1 - elif kind == "enrich_instagram": - # ⭐ C4 (R3/R4). Reuses `pull_profile` and `capture_rows` — the SAME functions - # `run_field_instagram` calls, not a second implementation of either. - pkey = profile_field_key(ut_get(rt, table), cfg.get("profileField")) - if not pkey: - # FAIL CLOSED, ONCE, WITH THE REASON. Not per record: the binding is a - # property of the flow, so a 100-record run would otherwise put the same - # sentence in the log a hundred times and bury everything else. - if "unbound" not in enrich["notes"]: - enrich["notes"].append("unbound") - log("[aios-auto] enrich_instagram: no profile column on " - f"{table}. Name one on the action, or mark a text column as an " - "Instagram profile") - continue - # ⭐ 2026-08-07 (owner ruling) — IS THIS RECORD IN THIS RUN'S BUDGET? - # Resolved once (see `enrich_plan`) and then a membership test. A record outside - # the selection is NOT an error and NOT a skip worth logging per row — it is simply - # not this run's work, and the selection's own note already accounts for it. - aid = str(act.get("id") or "") - if aid not in enrich_plan: - chosen, sel_note = enrich_selection(rt, table, cfg, pkey, gone=known_gone) - enrich_plan[aid] = set(chosen) - if sel_note: - enrich["notes"].append(sel_note) - # ⭐ ONE VENDOR CALL PER CHUNK FOR THE WHOLE SELECTION, here and nowhere else: - # this is the only place the full chosen set and the row bodies are both in - # hand. Anything it resolves the per-record rung below reads from memory. - _prime_enrich_batch(chosen, rows, pkey, table, enrich, - enrich_prefetch, step, log) - if str(rid) not in enrich_plan[aid]: - continue - handle_raw = str(row.get(pkey, "") or "").strip() - if not handle_raw: - continue # nothing to enrich on this record, not an error - # ⚠ THE PACE FLOOR IS THE VENDOR'S, SO IT IS PAID ONLY WHEN THE VENDOR IS CALLED. - # A handle already in `enrich_prefetch` was bought by the batch above and is read - # from memory; sleeping 2.5 s before a dictionary lookup would hand most of the - # batching win straight back (25 records = ~62 s of pure waiting). - if enrich["profiles"] and str(handle_raw).strip().lstrip("@").lower() \ - not in enrich_prefetch: - time.sleep(PACE_SECONDS) # >=2 s between profiles (R7), as the runner does - enrich["profiles"] += 1 - # ⭐ THE DEFERRED-PROFILE SINK IS PER RECORD so the snapshot can be stamped with - # the row it belongs to: the collector writes preset cells back onto THAT record, - # and a run-wide list would have no way to say which handle each snapshot was for. - pend_prof = [] - # ⭐ W31-T39(c) — the DEFERRAL WATERMARK, taken before the pull. See the `partial` - # report below: a capability that was deferred is not a capability that failed, - # and `pull_profile` appends into these two sinks from inside the call. - _pend_before = len(enrich["pending"]) - res = pull_profile(handle_raw, - max_posts=cfg.get("maxPosts") or DEFAULT_POSTS_PER_PULL, - log=log, - post_metrics=bool(cfg.get("postMetrics")), - comment_metrics=bool(cfg.get("commentMetrics")), - pending_metrics=(enrich["pending"] - if not cfg.get("dryRun") else None), - pending_profile=(pend_prof - if not cfg.get("dryRun") else None), - prefetch=enrich_prefetch, - post_groups=cfg.get("postGroups")) - # ⭐ 2026-08-09 (owner: *"a Filter on our enrichment so we only get Last 12 reels / - # 12 videos etc … not just last 12 but by group also"*). `config.postGroups` is - # `[{"type": "video", "limit": 12}, …]`; a type nobody names is dropped. - # ⛔ DEFAULT OFF — with no `postGroups` the list is returned unchanged, so this - # costs nothing and changes nothing until somebody configures it. - # ⭐⭐ WAVE 30 · T16 — THE WINDOW IS NOW WIDENED BEFORE IT IS FILTERED, and the - # paragraph that used to sit here explaining why it could not be is gone with the - # reason. It said: *"it FILTERS the window we already bought … the views top-up - # runs INSIDE `pull_profile` against the posts it captured, so swapping a wider - # list in afterwards would hand back posts with no view counts."* Correct, and - # exactly why the widening went INTO `pull_profile` (`post_groups=`, above) rather - # than being bolted on out here — ahead of the top-up, not after it. - # ⚠ THIS LINE STAYS, and it is not redundant. A single-type ask is served by a - # native route that returns only that type; a MIXED ask has no such route, so the - # wide window comes back mixed and this is what keeps N of each. - if cfg.get("postGroups"): - res = {**res, "posts": select_post_groups(res.get("posts") or [], - cfg.get("postGroups"))} - for _p in pend_prof: - _p["table"], _p["rowId"], _p["requestedAt"] = table, str(rid), _iso() - enrich["pendingProfiles"].extend(pend_prof) - pulled = _iso() - if res["state"] in ("ok", "partial"): - enrich["ok"] += 1 - snap_row, idents, metric_rows, comment_rows = capture_rows(res, pulled) - enrich["snaps"].append(snap_row) - enrich["posts"].extend(idents) - enrich["psnaps"].extend(metric_rows) - enrich["comments"].extend(comment_rows) - # R3: LATEST onto the record itself. `row.update` too, so a LATER action in - # this same flow sees the enriched values — the rule `update_record` already - # follows, and without it "enrich, then route by follower count" would judge - # the record on the values it had before the pull. - # ⭐⭐ WAVE 31 · T39(c) — D-177(c): THE FIX APPLIED TO ONE PLATFORM AND NOT ITS - # TWIN, and that is the shape worth naming rather than the line. - # - # W30-T11 found this on TikTok the expensive way: a paid run enriched two - # profiles with `postMetrics` ON, wrote no posts, and said NOTHING — because - # `partial` takes THIS branch and only the `else` ever appended a note, so the - # reason `pull_profile` carries in `res["note"]` was discarded. Instagram has - # the identical branch and never got the fix; the sibling arm below has carried - # it since wave 30. One defect, two networks, one of them repaired — which is - # exactly how the last four TikTok/Instagram divergences were found. - # - # ⛔ ONLY WHEN IT WAS ASKED FOR, and ⛔ NOT WHEN THE BATCH DEFERRED — both - # conditions copied deliberately from the TikTok arm rather than re-reasoned. - # With `postMetrics` off, a `partial` is the normal answer for every profile - # and reporting it would put one useless line per record in the run entry; a - # DEFERRAL is a paid snapshot already filed for collection, and reporting it as - # a miss teaches a person to re-run and buy it twice. - # ⚠ Instagram's deferral sinks are `enrich["pending"]` (post metrics) and - # `pend_prof` (the profile snapshot) — the two lists `pull_profile` appends - # into — where TikTok reads `res["deferredMedia"]`. Different signal, same - # question: did this record's work get filed rather than lost? - if (cfg.get("postMetrics") and not (res.get("posts") or []) - and len(enrich["pending"]) == _pend_before and not pend_prof): - _why = _s(res.get("note"), 300) - if _why: - enrich["notes"].append(f"@{handle_raw}: {_why}") - cells = preset_cells(res, pulled) - if cfg.get("dryRun"): - enrich["dry"] = True - else: - row.update(cells) - _act_row_patch(patches, table, rid, cells) - else: - enrich["blocked"] += 1 - # ⭐⭐ D-103 — THE NOTE IS NAMED AND KEPT AT FULL LENGTH. - # It used to be `_s(note, 90)` with no handle attached: three runs blocked - # the same record and the store could not say which record, let alone why. - # 90 characters also truncated the one measured sentence mid-clause. This is - # the most valuable thing the run produces and it now reaches the run entry. - enrich["notes"].append( - f"@{handle_raw}: {_s(res.get('note'), 300) or res['state']}") - # A vendor STATING that the account does not exist is remembered, so the next - # run stops paying to be told the same thing. - if res.get("gone"): - enrich["gone"][_gone_key(row, handle_raw)] = { - "at": _iso(), "handle": handle_raw, - "note": _s(res.get("note"), 300)} - elif kind == "enrich_tiktok": - # ⭐⭐ WAVE 30 · T08 (carrying wave-29's dropped T05). THE SECOND NETWORK, and it - # is a sibling of the branch above rather than a flag inside it. What is genuinely - # shared is shared by CALL — `enrich_selection`, `enrich_plan`, the pace floor, - # the tombstone memory; what differs is the four things B-9 measured as - # Instagram-hardcoded, and each has a TikTok counterpart of its own name. - pkey = profile_field_key(ut_get(rt, table), cfg.get("profileField"), - source=PROFILE_SOURCE_TT) - if not pkey: - # D-79, on the network that did not exist when D-79 was written: FAIL CLOSED, - # ONCE, WITH THE REASON — and the marker is its OWN, so a flow carrying both - # steps cannot report the Instagram sentence about the TikTok one. - if "unbound" not in tt["notes"]: - tt["notes"].append("unbound") - log("[aios-auto] enrich_tiktok: no profile column on " - f"{table}. Name one on the action, or mark a text column as a " - "TikTok profile") - continue - aid = str(act.get("id") or "") - if aid not in enrich_plan: - # ⚠ ONE plan dict for both networks is correct and not an oversight: it is - # keyed by ACTION id, and an action has exactly one kind. Two steps over one - # table are two budgets whichever networks they read. - chosen, sel_note = enrich_selection(rt, table, cfg, pkey, gone=known_gone, - platform=PLATFORM_TIKTOK) - enrich_plan[aid] = set(chosen) - if sel_note: - tt["notes"].append(sel_note) - if str(rid) not in enrich_plan[aid]: - continue - handle_raw = str(row.get(pkey, "") or "").strip() - if not handle_raw: - continue # nothing to enrich on this record, not an error - # ⚠ THE PACE FLOOR IS THE VENDOR'S. There is no batch prefetch on this path yet - # (`pull_profile_tt` accepts one and nothing writes it — see the mailbox), so - # every profile after the first pays it. - if tt["profiles"]: - time.sleep(PACE_SECONDS) - tt["profiles"] += 1 - import connectors_tt as _tt_conn # lazy: connectors_tt imports this module - pend_prof = [] - res = _tt_conn.pull_profile_tt( - handle_raw, log=log, - pending_profile=(pend_prof if not cfg.get("dryRun") else None), - # ⭐ W30-T10. Both INCLUDE axes default OFF (W28/R5-R7), and the same - # `config` keys the Instagram step reads — one vocabulary, two networks. - max_posts=int(cfg.get("maxPosts") or 0), - post_metrics=bool(cfg.get("postMetrics")), - comment_metrics=bool(cfg.get("commentMetrics"))) - for _p in pend_prof: - _p["table"], _p["rowId"], _p["requestedAt"] = table, str(rid), _iso() - # The deferred-profile queue is the Instagram one BY DESIGN: it is a vendor - # snapshot id waiting to be collected, and `_pending_profile_tasks` keys tasks by - # table+row, not by network. Sharing it is what makes the tick finish a TikTok - # read it has already been charged for. - enrich["pendingProfiles"].extend(pend_prof) - # ⭐⭐ W30 · D-156 — A DEFERRED MEDIA BATCH IS FILED, NOT NARRATED. Before this, - # a posts or comments scrape the vendor took too long over was reported in the - # note and collected by NOTHING: paid for, and recoverable only by a human reading - # a sentence. `connectors_tt` has already filtered these to the media corpora, so - # a profile snapshot cannot arrive here; what the engine adds is the vocabulary the - # QUEUE speaks — the `kind` (from the dataset id, which is one-to-one on TikTok) - # and the HANDLE, which the transport never knew. - # ⚠ `dryRun` queues NOTHING: a dry run buys nothing, so an entry here could only - # be a fixture leaking, and filing it would make the tick collect a snapshot that - # was never paid for. - if not cfg.get("dryRun"): - for _d in (res.get("deferredMedia") or []): - _kind = tt_metric_kind(_d.get("datasetId")) - if not _kind: - continue # not a media corpus ⇒ not this queue's business - tt["pending"].append({**_d, "kind": _kind, "requestedAt": _iso(), - "influencer": str(handle_raw).strip() - .lstrip("@").lower()}) - pulled = _iso() - if res["state"] in ("ok", "partial"): - tt["ok"] += 1 - snap_row = tt_snapshot_row(res, pulled) - if snap_row: - tt["snaps"].append(snap_row) - tt_idents, tt_metrics, tt_comments = tt_capture_rows(res, pulled) - tt["posts"].extend(tt_idents) - tt["psnaps"].extend(tt_metrics) - tt["comments"].extend(tt_comments) - # ⭐⭐ WAVE 30 · T11 — A CAPABILITY THAT WAS ASKED FOR AND DID NOT ARRIVE IS - # REPORTED. This is R6's second sentence applied to a capability rather than to - # a row cap, and it was found the expensive way: the 09:14 UTC paid run on - # nurilab enriched two profiles with `postMetrics` ON, wrote no posts, and said - # NOTHING — `ok: true`, no note, no count — because `partial` takes the branch - # ABOVE and only the `else` ever appended a note. Every `partial` return in - # `pull_profile_tt` carries the reason in `note` (*"this account's row carried - # no post links"*, *"the post source returned nothing"*, or the vendor's own - # words), and all of them were being discarded. A person then sees two enriched - # rows, an empty posts database and no explanation anywhere. - # ⛔ ONLY WHEN IT WAS ASKED FOR, which is the difference between a report and - # noise: with `postMetrics` off, `pull_profile_tt` returns `partial` + *"post - # capture is off for this step"* for EVERY profile, and appending that would put - # one useless line per record into the run entry and let the summary quote it. - # ⛔ AND NOT WHEN THE BATCH DEFERRED — a deferral is not a failure to deliver, - # it is a paid snapshot already filed for collection - # (`ttEnrichMetricBatchesPending`, D-156), and reporting it as a miss would - # teach a person to re-run and buy it twice. - if (cfg.get("postMetrics") and not (res.get("posts") or []) - and not (res.get("deferredMedia") or [])): - _why = _s(res.get("note"), 300) - if _why: - tt["notes"].append(f"@{handle_raw} (TikTok): {_why}") - cells = tt_preset_cells(res, pulled) - if cfg.get("dryRun"): - tt["dry"] = True - else: - row.update(cells) - _act_row_patch(patches, table, rid, cells) - else: - tt["blocked"] += 1 - # ⚠ TAGGED, because `run_notes` now carries BOTH networks' per-record reasons - # (the two flushes are concatenated). The summary picks ONE note to quote, so an - # untagged TikTok line could be quoted under Instagram's sentence and vice versa - # — which would undo the whole point of giving each network its own sentence. - tt["notes"].append( - f"@{handle_raw} (TikTok): {_s(res.get('note'), 300) or res['state']}") - # ⛔ AND THERE IS DELIBERATELY NO TOMBSTONE WRITER HERE, which is the opposite - # of an oversight. On the Instagram side `res["gone"]` has exactly ONE source — - # Apify answering `ACCOUNT_GONE_NOTE` (`connectors_ig`); Bright Data has no - # not-found verdict at all, and `DEFAULT_CHAINS["tt_profile"]` is deliberately - # single-provider, so nothing on this chain can say "no account exists". A - # phrase match invented against unmeasured vendor output would file a - # PERMANENT verdict (W28/R9 — no expiry) on the strength of a guess. - # ⚠ The READ side is still network-scoped above (`platform=PLATFORM_TIKTOK`) - # and that half is load-bearing today: `known_gone` is shared, Apify DOES - # write `instagram:` verdicts, and without the scoping one of those - # would silently suppress a TikTok read of a different person's account. - elif kind == "find_records": - found = find_records(rt, cfg.get("table"), cfg.get("cond"), - int(cfg.get("limit") or 25)) - counts["found"] += len(found) - # ⛔ THE `review` BRANCH IS DELETED (wave 27 item 12, owner ruling R3), and it had - # already stopped being reachable one wave earlier — which is the part worth reading. - # `_without_retired_board` strips every `review` action out of a definition on the - # READ path, so `all_definitions` and this function have not seen one since the board - # was retired. What was left behind was a branch referencing `skey`, `stamp` and - # `ai_budget` — three names with NO DEFINITION anywhere in this module. It was not - # dead code that merely wasted space: it was a `NameError` held back by a migration - # rather than by a guard, and any change that let one stored `review` action through - # would have crashed the whole action walk for every record in that flow. - # ⚠ `ai_decide` and `review_audit` SURVIVE as library code with no caller — R3 keeps - # review "as an AI decision without lanes", and the cheap-first provider ladder behind - # it is real, working, measured work. They are PARKED, deliberately and in writing, - # not orphaned; see their own notes. - return False - - if actions: - for rid in list(row_ids or [])[:FLOOD_LIMIT]: - row = dict(rows.get(str(rid)) or {}) - if not row: - continue - # ⭐ D-191 — the web batch is built from THIS row's interpolated values, so it means - # nothing for the next one. Cleared here rather than inside `_walk`, which recurses - # into branches and would wipe a batch its own caller is still consuming. - web_done.clear() - _walk(actions, row, str(rid)) - # ⭐⭐ WAVE 30 · T08 — TWO FLUSHES, ONE `counts`, AND THE NOTES ARE CONCATENATED RATHER THAN - # OVERWRITTEN. `RUN_NOTES_KEY` is the one key both flushes legitimately produce, so a plain - # `counts.update(a); counts.update(b)` would drop every Instagram per-record reason the moment - # a flow also carried a TikTok step — silently, and precisely on the mixed flows where a - # person most needs to know which half failed. Every other key is prefixed and cannot collide. - _ig_out = _enrich_flush(rt, defn, username, enrich, log) - _tt_out = _tt_enrich_flush(rt, defn, username, tt, log) - _flush_notes = (list(_ig_out.pop(RUN_NOTES_KEY, None) or []) - + list(_tt_out.pop(RUN_NOTES_KEY, None) or [])) - counts.update(_ig_out) - counts.update(_tt_out) - if _flush_notes: - counts[RUN_NOTES_KEY] = _flush_notes - # ⭐ C5: the REALIZED numbers overwrite the walk's attempt count. `counts["created"]` was - # incremented once per create the flow decided to make; what landed is what the store says, - # and with `uniqueOn` on they are routinely different (a re-run of a scheduled flow matches - # every row it made last time — which is the whole point of the feature). - counts.update(_commit_action_writes(rt, table, patches, creates, username, log)) - return counts - - -def migrate_field_instagram(rt, defn): - """⭐ WAVE 25 · R4 — one stored `field_instagram` definition → a `plain` one carrying the - `enrich_instagram` action. Returns `(definition, changed)`. - - R4: "the ENRICH ACTION REPLACES the `field_instagram` KIND… migrate the one live - `field_instagram` automation to a plain flow carrying it; delete the kind and its label." - - ⛔ THE BINDING IS RESOLVED HERE, NOT LEFT TO THE FLAG, and this is the line the migration - turns on. `run_field_instagram` reads `cfg.urlField` **or falls back to `_auto_url_field`** — - so a live automation whose `urlField` is blank has been working off that fallback for months. - The enrich action deliberately has no such fallback (see `profile_field_key`), so migrating a - blank `urlField` verbatim would produce an automation that USED to work and now refuses. The - fallback is therefore evaluated ONCE, here, and the answer is written down as an explicit - binding — which is also the honest outcome: the column stops being implicit. - - ⚠ ONE BEHAVIOUR DOES CHANGE, AND IT IS THE POINT OF THE RULING RATHER THAN A REGRESSION. The - old kind wrote a STATUS STRING ("ok · 2026-08-06 · 12,400 followers") into `config.fieldKey`'s - column; the action writes the C1 PRESET CELLS instead. The status column is left in place and - simply stops being written — deleting somebody's column as part of a migration would be data - loss, and a stale cell beside a fresh `enriched_at` is readable for what it is. - - ⚠ PURE OVER THE DEFINITION apart from the one table READ. It writes nothing, so a caller can - run it to INSPECT what a migration would do — which is exactly how R4's "must be PROVEN - against the real stored definition" is meant to be satisfied. - """ - if (defn or {}).get("kind") != "field_instagram": - return defn, False - cfg = dict(defn.get("config") or {}) - target = str(cfg.get("targetTable") or "") - bound = str(cfg.get("urlField") or "").strip() - if not bound and target: - bound = str(_auto_url_field(ut_get(rt, target) or {}, cfg.get("fieldKey")) or "") - try: - max_posts = int(cfg.get("maxPosts") or DEFAULT_POSTS_PER_PULL) - except (TypeError, ValueError): - max_posts = 24 - act = {"id": "act_enrich", "kind": "enrich_instagram", "enabled": True, "when": None, - "config": {"profileField": bound, - "postMetrics": bool(cfg.get("postMetrics")), - "commentMetrics": bool(cfg.get("commentMetrics")), - "dryRun": bool(cfg.get("dryRun")), - "maxPosts": max(1, min(max_posts, 200))}} - flow = dict(defn.get("flow") or {}) - out = dict(defn) - out["kind"] = DEFAULT_KIND - # The enrich step goes FIRST — it is what the automation was, and every action the owner - # added afterwards was written expecting the capture to have happened. - out["flow"] = {"actions": [act] + list(flow.get("actions") or [])} - out["config"] = {"targetTable": target, - "targetLabel": str(cfg.get("targetLabel") or "")} - return out, True - - -def _enrich_flush(rt, defn, username, acc, log): - """C4: the enrich action's canonical IG tables, written ONCE for the whole run. - - Returns the counts the run reports. Empty when no enrich action ran, so a flow without one - pays nothing and its run history is unchanged (`run_now` merges only non-zero keys). - - ⛔ THE SAME WRITE PATH AS `run_field_instagram`, NOT A PARALLEL ONE: `ut_ensure` the three - tables, `upsert_rows` on their own keys with their own caps, one coalesced `rt.update`, then - the write-through to the platform master. Forking any of those would give the enrich action a - history that the metric fields could not see — and it is the LAST step, the write-through to - the platform master, that they read (W29-T34: `compute_metric_cells` → `ig_master.series_for`, - never this tenant's `ut_ig_snapshots`), so dropping that one line is the version of this fork - that would look harmless. - """ - if "unbound" in (acc.get("notes") or []): - # ⛔ DEBT D-79, SECOND HALF — THE SILENT FAILURE, AND IT WAS THE WORSE HALF. An enrich step - # on a database with no profile column `continue`s BEFORE `enrich["profiles"] += 1`, so - # this function used to return `{}`, `run_now` merged only non-zero keys, and the run - # committed **`ok` — "N records walked"** having written nothing at all. The sentence - # naming the fix went to `log()`, i.e. the server console, which no customer reads. That - # is "green over nothing" on the one path where somebody is waiting for data. - # ⚠ IT IS A COUNT, not a flag, so `run_now`'s existing non-zero merge carries it without a - # special case — and so the run entry itself records that this happened. - return {"enrichUnbound": 1} - # ⭐⭐ 2026-08-09 — THE NOTES SURVIVE A RUN THAT READ NOTHING, and that is not a detail. - # `if not acc["profiles"]: return {}` is exactly the branch a run takes when EVERY candidate - # was skipped as a known-dead handle — so the sentence explaining why the automation appears - # to do nothing would have been dropped on precisely the runs that most need it, and the - # owner would be back at "0 records walked, wtf". The selection note is produced before any - # profile is read and must outlive that early return. - if not acc.get("profiles"): - notes = list(acc.get("notes") or []) - return {RUN_NOTES_KEY: notes} if notes else {} - out = {"enriched": acc["ok"], "enrichBlocked": acc["blocked"]} - if acc.get("notes"): - out[RUN_NOTES_KEY] = list(acc["notes"]) - # ⭐ D-103's own prescription: "the block note survives on the RUN … not a status column, so - # no table gains a column it did not ask for". `RUN_NOTES_KEY` is that channel. - if not acc.get("dry"): - # The vendor's not-found verdicts, merged into engine state (never a tenant column and - # never a status string — W25/R4 retired those). Merged rather than replaced: a run that - # walked one record must not forget what earlier runs learned about the others. - if acc.get("gone"): - merged = dict(((defn or {}).get("state") or {}).get("enrichNotFound") or {}) - merged.update(acc["gone"]) - set_state(rt, str(defn.get("id") or ""), {"enrichNotFound": merged}) - pending_profiles = queue_pending_profile_snapshots( - rt, str(defn.get("id") or ""), acc.get("pendingProfiles") or []) - if pending_profiles: - out["enrichProfileBatchesPending"] = pending_profiles - queued = queue_pending_metric_snapshots(rt, str(defn.get("id") or ""), - acc.get("pending") or []) - if queued: - out["enrichMetricBatchesPending"] = queued - if acc.get("dry") or not (acc["snaps"] or acc["posts"] or acc["psnaps"] or acc["comments"]): - # A dry run resolves nothing and writes nothing — not even `ut_ensure`, which CREATES. - if acc.get("dry"): - out["enrichDryRun"] = acc["profiles"] - return out - tag = str(defn.get("id") or "") - profile_table = str(_flow_table(defn) or "") - graph = ensure_ig_graph(rt, username, tag, profile_table=profile_table) - snap_key, post_key, ps_key, comment_key = (graph[IG_SNAPSHOTS_TABLE], graph[IG_POSTS_TABLE], - graph[IG_POST_SNAPSHOTS_TABLE], graph[IG_COMMENTS_TABLE]) - missing = ut_missing(rt, snap_key, post_key, ps_key, comment_key) - snaps, c_snap = upsert_rows(dict((ut_get(rt, snap_key) or {}).get("rows") or {}), - acc["snaps"], "snapshot_key", cap=row_cap(snap_key)) - old_posts, collapsed_posts = dedupe_canonical_rows( - dict((ut_get(rt, post_key) or {}).get("rows") or {}), "shortcode", newest_by="measured_at") - posts, c_post = upsert_rows(old_posts, acc["posts"], "shortcode", cap=row_cap(post_key)) - c_post["duplicates"] += collapsed_posts - psnaps, c_ps = upsert_rows(dict((ut_get(rt, ps_key) or {}).get("rows") or {}), - acc["psnaps"], "post_snapshot_key", cap=row_cap(ps_key)) - old_comments, collapsed_comments = dedupe_canonical_rows( - dict((ut_get(rt, comment_key) or {}).get("rows") or {}), "comment_key") - comments, c_comments = upsert_rows(old_comments, acc["comments"], "comment_key", - cap=row_cap(comment_key)) - c_comments["duplicates"] += collapsed_comments - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - for k, rws in ((snap_key, snaps), (post_key, posts), (ps_key, psnaps), - (comment_key, comments)): - if cur.get(k) is not None: - cur[k]["rows"] = rws - _refresh_relations_inplace(cur, log=log) - return cur - - rt.update(UT_STORE_KEY, _up, flush="sync") # ONE coalesced update for all three tables - # LOUD, never silent (D-11): a full append table means the SERIES has stopped growing, which - # is the failure a chart cannot show you. - capped = c_snap["capped"] + c_post["capped"] + c_ps["capped"] + c_comments["capped"] - if capped: - out["enrichCapped"] = capped - log(f"[aios-auto] enrich_instagram: {capped} row(s) refused by a table's row cap") - if missing: - out["enrichMissingTables"] = len(missing) - log(f"[aios-auto] enrich_instagram: could not create {', '.join(missing)}") - out["enrichPoints"] = len(acc["snaps"]) - out["enrichPosts"] = c_post["inserted"] - out["enrichComments"] = c_comments["inserted"] - # C6/R2's write-through to the PLATFORM MASTER. Three postures, never conflated — the same - # contract `run_field_instagram` follows, because a pooled history with silent holes is worse - # than none. - try: - import ig_master - m_status, m_note = ig_master.append_run(getattr(rt, "key", ""), acc["snaps"], - acc["posts"], acc["psnaps"]) - if m_status == "error": - out["enrichMasterFailed"] = 1 - log(f"[aios-auto] enrich_instagram: the platform master copy FAILED. {m_note}; " - f"the tenant copy is complete and the next run re-appends") - elif m_status == "ok": - out["enrichMaster"] = len(acc["snaps"]) + len(acc["psnaps"]) - except Exception as e: # noqa: BLE001 - out["enrichMasterFailed"] = 1 - log(f"[aios-auto] enrich_instagram: master write-through raised " - f"{type(e).__name__}: {e}") - return out - - -#: How many candidate columns an unbound-enrich sentence names before it stops. Three, because the -#: sentence is read in a run log line: naming forty columns is the same as naming none. -UNBOUND_HINT_MAX = 3 - - -def _unbound_hint(rt, table_key): - """⭐ WAVE 30 · T14 — the *"…which column to mark"* half of D-79's sentence. - - Returns `" — this database's text columns are Handle, Who"` or `""`. Never a binding. - - ⛔ IT SUGGESTS AND DOES NOT RESOLVE, and the distinction is the whole reason this is a string - rather than a fallback. `profile_field_key`'s own docstring refuses to guess — *"the failure - would be a run that reports success having enriched from the wrong column … a wrong number is - harder to notice than a missing one"*. That argument is about BINDING. It says nothing against - telling a person, in the sentence they are already reading, which columns are even eligible: - the enrich still refuses, nothing is written, and the human makes the choice. - ⚠ `text` ONLY, matching the writer F shipped for the flag (`ColumnMenu`'s toggle is gated on - `editType === "text"`), so the sentence cannot offer a column the editor would then refuse. - """ - fields = (ut_get(rt, table_key) or {}).get("fields") or [] - # ⚠ AND A PRESET/LOCKED COLUMN IS NOT OFFERED. `Platform` is a `text` column on every preset - # profile table and carries `automation.preset` + an `editRole`, so the field editor refuses to - # retype it — offering it would send a person to a control that says no, which is exactly the - # claim the docstring above makes and did not honour on its first draft. - names = [str(f.get("label") or f.get("key") or "").strip() for f in fields - if str(f.get("type") or "text") == "text" - and not (f.get("automation") or {}).get("preset") - and not str(f.get("editRole") or "").strip() - and str(f.get("label") or f.get("key") or "").strip()] - if not names: - return "" - shown = ", ".join(names[:UNBOUND_HINT_MAX]) - more = len(names) - UNBOUND_HINT_MAX - return (f". This database's text columns are {shown}" - + (f" (+{more} more)" if more > 0 else "")) - - -def _tt_write_tables(rt, tag, username, snaps, posts, psnaps, comments, log=print): - """The four `ut_tt_*` tables, created-if-needed and written in ONE coalesced store update. - Returns `(inserted_by_table, capped, missing)`. - - ⭐⭐ WAVE 30 · D-156 — ONE WRITER, TWO CALLERS, AND THE SECOND CALLER IS WHY IT EXISTS. - This was the tail of `_tt_enrich_flush`, i.e. reachable only from an INLINE enrich. The - deferred collector needs exactly the same write, and the repo has already paid once for the - version where it did not: `top_up_views` lived inside `pull_profile_bd`, so it ran only when - the Posts scrape answered in time, and every DEFERRED Instagram run wrote posts with a blank - Views column. The fix there was this same shape — one function, two callers, so the inline and - deferred paths cannot answer differently — and copying the block instead would reintroduce the - class rather than the bug. - - ⭐ ONE LOOP OVER `(table, rows, key)` RATHER THAN FOUR HAND-WRITTEN BLOCKS, and the field list - comes from `TT_TABLE_FIELDS` — so a fifth `ut_tt_*` table is one tuple, and no table can be - created with a field list that disagrees with its own declaration. - ⚠ NO ROWS ⇒ NO DATABASE. Spawning `ut_tt_comments` on a collect that carried none gives a - person a database to watch never fill, which is the visible half of green-over-nothing. - """ - plan = [(TT_SNAPSHOTS_TABLE, snaps or [], "snapshot_key"), - (TT_POSTS_TABLE, posts or [], "shortcode"), - (TT_POST_SNAPSHOTS_TABLE, psnaps or [], "post_snapshot_key"), - (TT_COMMENTS_TABLE, comments or [], "comment_key")] - written, capped, missing = {}, 0, [] - inserted = {} - for table_key, rows_in, key_field in plan: - if not rows_in: - continue - # ⭐ R9 (W31-T32) — THE RUN PATH IS THE SITE THAT ACTUALLY CREATED TODAY'S TABLES, and it - # is the one the ticket's `how:` does not name. `waves/wave30/proof/tiktok-e2e-ut_tt_posts - # .png` shows `ut_tt_posts` with 8 real rows offering "+ New record" — those rows arrived - # HERE, not through the save path (which, until W31-T34, returned before the child spawn for - # a discovery automation). Stamping only the save site would have left every table that - # already exists unlocked. ⭐ And an EXISTING table does come forward on the next call: - # `ut_ensure`'s skip test carries `not (record_mode and have.get("recordMode") != - # record_mode)`, so no separate migration is needed. - real_key = ut_ensure(rt, TT_TABLE_LABELS[table_key], TT_TABLE_FIELDS[table_key], username, - key=table_key, flow_tag=tag, lock_fields=True, - record_mode=tt_record_mode(table_key)) - missing.extend(ut_missing(rt, real_key)) - merged, counts_ = upsert_rows(dict((ut_get(rt, real_key) or {}).get("rows") or {}), - rows_in, key_field, cap=row_cap(real_key)) - written[real_key] = merged - inserted[table_key] = counts_["inserted"] - capped += counts_["capped"] - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - for k, rws in written.items(): - if cur.get(k) is not None: - cur[k]["rows"] = rws - _refresh_relations_inplace(cur, log=log) - return cur - - if written: - rt.update(UT_STORE_KEY, _up, flush="sync") # ONE coalesced update for every table - return inserted, capped, missing - - -def _tt_enrich_flush(rt, defn, username, acc, log): - """⭐⭐ WAVE 30 · T08 — the TikTok enrich action's append table, written ONCE for the whole run. - - The twin of `_enrich_flush` and deliberately NOT a call into it. That function is bound to - Instagram at four points — `ensure_ig_graph`, the four `IG_*_TABLE` keys, `PRESET_*` and the - write-through to the platform Instagram master — and the last of those is the one that would - look harmless: `ig_master.append_run` is what the metric FIELDS read (W29-T34), so a TikTok - creator's followers appended there would become an Instagram number in a pooled history that - no later run could tell apart. - - ⛔ EVERY KEY IT RETURNS IS PREFIXED `tt…`. `apply_actions` merges both flushes into one - `counts`, so `enriched`/`enrichBlocked` would be one name for two networks' numbers and a - mixed flow would report whichever flushed last. - - ⭐ WAVE 30 · T10 — FOUR TABLES NOW, AND EACH IS CREATED ONLY WHEN IT HAS A ROW. Spawning - `ut_tt_posts` on a run that captured no posts gives a person a database to watch never fill, - which is the visible half of green-over-nothing and the same complaint that produced - `discover_default_table` (*"a SECOND, empty database"*). So the create rides the rows. - """ - if "unbound" in (acc.get("notes") or []): - # D-79 on the second network: a COUNT, so `run_now`'s existing non-zero merge carries it - # and the run entry itself records that the step could not run. Its own key, so the - # summary can name TikTok rather than borrowing Instagram's sentence. - return {"ttEnrichUnbound": 1} - if not acc.get("profiles"): - # The selection's note outlives a run that read nothing — the same reason as the IG side: - # "every candidate was skipped" is exactly the run whose silence needs explaining. - notes = list(acc.get("notes") or []) - return {RUN_NOTES_KEY: notes} if notes else {} - out = {"ttEnriched": acc["ok"], "ttEnrichBlocked": acc["blocked"]} - if acc.get("notes"): - out[RUN_NOTES_KEY] = list(acc["notes"]) - # ⭐⭐ W30 · D-156 — QUEUED BEFORE THE DRY-RUN AND EMPTY-ROWS RETURNS BELOW, and the order is - # the point. A run whose media batch DEFERRED has no posts and no comments to write, so it - # takes the `not (snaps or posts or ...)` exit — the exact run that owns a paid snapshot id. - # Filing after that return would have collected nothing, forever, which is the shape the - # inline-vs-deferred split keeps producing. - # ⚠ Its own count key (`tt…`), like every other key here: `apply_actions` merges both - # networks' flushes into ONE dict, so sharing Instagram's name would let a mixed flow report - # one network's pending batches under the other's. - if not acc.get("dry"): - queued = queue_pending_metric_snapshots(rt, str(defn.get("id") or ""), - acc.get("pending") or []) - if queued: - out["ttEnrichMetricBatchesPending"] = queued - if acc.get("dry") or not (acc.get("snaps") or acc.get("posts") or acc.get("psnaps") - or acc.get("comments")): - # A dry run resolves nothing and writes nothing — not even `ut_ensure`, which CREATES. - if acc.get("dry"): - out["ttEnrichDryRun"] = acc["profiles"] - return out - tag = str(defn.get("id") or "") - inserted, capped, missing = _tt_write_tables( - rt, tag, username, acc.get("snaps"), acc.get("posts"), acc.get("psnaps"), - acc.get("comments"), log) - # LOUD, never silent (D-11): a full append table means the SERIES has stopped growing, which - # is the failure a chart cannot show you. - if capped: - out["ttEnrichCapped"] = capped - log(f"[aios-auto] enrich_tiktok: {capped} row(s) refused by a table's row cap") - if missing: - out["ttEnrichMissingTables"] = len(missing) - log(f"[aios-auto] enrich_tiktok: could not create {', '.join(missing)}") - out["ttEnrichPoints"] = len(acc.get("snaps") or []) - if inserted.get(TT_POSTS_TABLE): - out["ttEnrichPosts"] = inserted[TT_POSTS_TABLE] - if inserted.get(TT_COMMENTS_TABLE): - out["ttEnrichComments"] = inserted[TT_COMMENTS_TABLE] - # ⛔ AND NO PLATFORM-MASTER WRITE-THROUGH, which is a deliberate absence rather than a missing - # line. `ig_master` is Instagram's pooled history and there is no TikTok equivalent yet; the - # tenant's own `ut_tt_snapshots` is the complete record today, and inventing a second store - # for a series nobody reads would be the fork this function exists to avoid. - return out - - -_PENDING_METRIC_DATASETS = frozenset((BD_DS_POSTS, BD_DS_REELS, BD_DS_COMMENTS)) -_PENDING_METRIC_KINDS = frozenset(("posts", "comments")) - -#: ⭐⭐ WAVE 30 · D-156 — THE TIKTOK HALF OF THE METRIC QUEUE, BUILT LAZILY FROM `connectors_tt`'s -#: OWN CONSTANTS. Re-declaring the two ids here would be a second copy of a vendor identifier that -#: nothing compares — the drift `MAX_UT_ROWS` demonstrated at 12x — so this reads them from the one -#: module that owns them, through `_tt_module()` because `connectors_tt` imports THIS module. -#: ⚠ It is a MAP rather than a set because on TikTok one dataset is exactly one kind, which is why -#: this network needs no `_tag_metric_deferrals` twin: the id the transport already recorded says -#: whether a batch is posts or comments, so nothing downstream has to be told twice. -_TT_METRIC_KIND_BY_DATASET = None - - -def tt_metric_kind(dataset_id): - """`"posts"` / `"comments"` for a TikTok media dataset id; `""` for anything else. - - ⛔ THE `""` IS LOAD-BEARING AND IS THE PLATFORM TEST. `collect_pending_metric_snapshots` - branches on it, so a dataset this map does not know keeps Instagram's mappers — which is the - safe direction, because Instagram's are what every stored pre-wave-30 task needs. - """ - global _TT_METRIC_KIND_BY_DATASET - if _TT_METRIC_KIND_BY_DATASET is None: - _tt = _tt_module() - _TT_METRIC_KIND_BY_DATASET = {str(_tt.TT_DS_POSTS): "posts", - str(_tt.TT_DS_COMMENTS): "comments"} - return _TT_METRIC_KIND_BY_DATASET.get(str(dataset_id or ""), "") - - -def _pending_metric_tasks(defn): - """Read validated, deduplicated paid metric snapshots from automation continuation state. - - A snapshot ID is a vendor-issued capability for work already paid for. It is intentionally - stored as engine state beside discovery's pending corpus snapshot, never in a Profile cell or - the user-editable flow. Invalid/stale shapes are ignored rather than sent back to a vendor - endpoint, and the collector never starts a second scrape request. - """ - raw = ((defn or {}).get("state") or {}).get("pendingMetricSnapshots") or [] - out, seen = [], set() - for item in raw if isinstance(raw, list) else []: - if not isinstance(item, dict): - continue - sid = str(item.get("snapshotId") or "").strip() - dataset = str(item.get("datasetId") or "").strip() - kind = str(item.get("kind") or "").strip() - handle = str(item.get("influencer") or "").strip().lstrip("@").lower() - # ⭐ W30 · D-156 — BOTH NETWORKS' MEDIA CORPORA ARE COLLECTABLE NOW. The membership test - # stays a WHITELIST (a snapshot id is a vendor capability that has already been paid for; - # accepting an unknown dataset would send our key at a corpus no mapper here can read), - # and TikTok's half is asked of the map that owns it rather than listed again. - if (not sid.startswith("sd_") - or (dataset not in _PENDING_METRIC_DATASETS and not tt_metric_kind(dataset)) - or kind not in _PENDING_METRIC_KINDS or not handle): - continue - key = (sid, dataset, kind, handle) - if key in seen: - continue - seen.add(key) - out.append({"snapshotId": sid, "datasetId": dataset, "kind": kind, - "influencer": handle, "requestedAt": str(item.get("requestedAt") or ""), - "lastChecked": str(item.get("lastChecked") or ""), - "lastNote": _s(item.get("lastNote"), 160)}) - return out - - -def queue_pending_metric_snapshots(rt, auto_id, pending): - """Durably retain new engagement snapshots without creating another paid provider request.""" - aid = str(auto_id or "").strip() - if not aid: - return 0 - incoming = _pending_metric_tasks({"state": {"pendingMetricSnapshots": pending}}) - if not incoming: - return 0 - added = [0] - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - definition = cur.get(aid) - if not isinstance(definition, dict): - return cur - state = definition.setdefault("state", {}) - existing = _pending_metric_tasks(definition) - known = {(x["snapshotId"], x["datasetId"], x["kind"], x["influencer"]) - for x in existing} - for task in incoming: - key = (task["snapshotId"], task["datasetId"], task["kind"], task["influencer"]) - if key not in known: - existing.append(task) - known.add(key) - added[0] += 1 - state["pendingMetricSnapshots"] = existing - return cur - - _store_update(rt, _up, flush="sync") - return added[0] - - -def _match_profile_row(rows, handle): - """THIS handle's row out of a snapshot that may now hold several. `None` when it is not there. - - ⭐⭐ 2026-08-09 — `rows[0]` WAS SAFE ONLY WHILE EVERY SNAPSHOT HELD EXACTLY ONE PROFILE. With - `_prime_enrich_batch` buying five URLs per call, one snapshot id serves up to five records, and - taking the first row would write ONE profile's followers onto all of them — a wrong number that - looks exactly like a right one. Identity comes off the row (`account`/`username`), which is the - same field `_bd_profile` reads first. - - ⚠ THE SINGLE-ROW FALLBACK IS DELIBERATE AND NARROW. Every task queued before this change points - at a one-URL snapshot, and some of those rows have an unreadable identity; for exactly that - shape the task's own handle is still the best evidence. It applies only when the snapshot holds - ONE row, so it can never mis-assign inside a batch. - """ - want = str(handle or "").strip().lstrip("@").lower() - usable = [n for n in (rows or []) if isinstance(n, dict)] - for node in usable: - got = str(_first(node, "account", "username", default="") or "").strip() - if got.lstrip("@").lower() == want and want: - return node - if len(usable) == 1: - return usable[0] - return None - - -def _pending_profile_tasks(defn): - """Validated, deduplicated deferred PROFILE snapshots from automation state. - - ⭐⭐ ITS OWN LIST, NOT `pendingMetricSnapshots`, and the separation is load-bearing rather - than tidy. `_pending_metric_tasks` filters on `datasetId in {posts, reels, comments}` and - `kind in {posts, comments}` — so a profile entry appended to that list is silently dropped to - zero by its own validator, and even if it survived, the collector would hand it to - `_write_collected_metric_rows`, a posts/comments writer that has nothing to do with a profile - row. Reusing the name would have shipped a green no-op of exactly the class this change - exists to remove. - - ⚠ A PROFILE TASK CARRIES ITS DESTINATION (`table` + `rowId`). A collected profile is written - back as PRESET CELLS onto the record that asked for it, so unlike a metric batch it cannot be - resolved from the handle alone: two databases may both hold `@x`. - """ - raw = ((defn or {}).get("state") or {}).get("pendingProfileSnapshots") or [] - out, seen = [], set() - for item in raw if isinstance(raw, list) else []: - if not isinstance(item, dict): - continue - sid = str(item.get("snapshotId") or "").strip() - dataset = str(item.get("datasetId") or "").strip() - handle = str(item.get("influencer") or "").strip().lstrip("@").lower() - table = str(item.get("table") or "").strip() - row_id = str(item.get("rowId") or "").strip() - # ⛔ `sd_` ONLY. A `snap_…` corpus id sent to `/datasets/v3/…` is a flat 404 about a - # snapshot that is alive (§2c), and a malformed id must never be handed back to a vendor - # endpoint at all. - if (not sid.startswith("sd_") or dataset != BD_DS_PROFILES or not handle - or not table or not row_id): - continue - if sid in seen: - continue - seen.add(sid) - out.append({"snapshotId": sid, "datasetId": dataset, "kind": "profile", - "influencer": handle, "table": table, "rowId": row_id, - "requestedAt": str(item.get("requestedAt") or ""), - "lastChecked": str(item.get("lastChecked") or ""), - "lastNote": _s(item.get("lastNote"), 200)}) - return out - - -def queue_pending_profile_snapshots(rt, auto_id, pending): - """Durably retain deferred profile snapshots. Starts no new paid request. Returns how many - were newly added.""" - aid = str(auto_id or "").strip() - if not aid: - return 0 - incoming = _pending_profile_tasks({"state": {"pendingProfileSnapshots": pending}}) - if not incoming: - return 0 - added = [0] - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - definition = cur.get(aid) - if not isinstance(definition, dict): - return cur - state = definition.setdefault("state", {}) - existing = _pending_profile_tasks(definition) - known = {x["snapshotId"] for x in existing} - for task in incoming: - if task["snapshotId"] not in known: - existing.append(task) - known.add(task["snapshotId"]) - added[0] += 1 - state["pendingProfileSnapshots"] = existing - return cur - - _store_update(rt, _up, flush="sync") - return added[0] - - -def collect_pending_profile_snapshots(rt, defn, username="automation", log=print, step=_no_step): - """Finish deferred PROFILE reads the tenant has already paid for. Starts no new scrape. - - ⭐⭐ 2026-08-09 — THE HALF THAT DID NOT EXIST. `bd_scrape` has always accepted a `deferred` - list, and every post/reel/comment call passed one; the PROFILE call did not, so a profile the - vendor took longer than `BD_SCRAPE_WAIT` to collect was billed and its snapshot id thrown - away — on every run, forever. MEASURED on nurilab: `collection_duration` 320 s against a - 180 s budget, two abandoned `sd_…` snapshots in two runs. - - ⛔ IT CLOSES A TASK THAT FINISHED EMPTY. A pending entry that can never resolve is the same - forever-loop wearing a different mask, so `bd_snapshot_progress` deciding `done` with zero - records ends the task with the vendor's reason attached — and, when the vendor blames the - target rather than itself, records the not-found verdict so the handle stops being re-bought. - """ - tasks = _pending_profile_tasks(defn) - if not tasks: - return ("ok", "No pending profile reads.", {}, [], {"source": "idle"}) - step(f"Collecting {len(tasks)} deferred profile read{'' if len(tasks) == 1 else 's'}") - remaining, patches, affected, notes = [], {}, [], [] - acc = {"snaps": [], "posts": [], "psnaps": [], "comments": [], "pending": [], - "pendingProfiles": [], "gone": {}, "profiles": 0, "ok": 0, "blocked": 0, - "notes": [], "dry": False} - ready = waiting = closed = 0 - # ⭐ ONE SNAPSHOT, ONE STATUS CALL, ONE FETCH. A batched read gives several tasks the SAME - # `snapshotId`, and asking the vendor about it once per task would spend the round trips the - # batching just saved. Memoised for this collection pass only — a snapshot's state must be - # re-read on the NEXT run, which is a fresh call and a fresh dict. - _progress_memo, _rows_memo = {}, {} - - def _progress_of(sid): - if sid not in _progress_memo: - _progress_memo[sid] = bd_snapshot_progress(sid) - return _progress_memo[sid] - - def _store_profile(task, profile, via): - """Write ONE collected profile — snapshot row + preset cells on the record that asked. - - ONE implementation, TWO callers (the primary snapshot and the backup rung below), so the - two paths cannot drift into writing different things — the same rule `top_up_views` - follows for the inline/deferred split. - """ - res = {"state": "ok", "profile": profile, "posts": [], "comments": [], - "via": via, "note": ""} - pulled = _iso() - snap_row, idents, metric_rows, comment_rows = capture_rows(res, pulled) - acc["snaps"].append(snap_row) - acc["posts"].extend(idents) - acc["psnaps"].extend(metric_rows) - acc["comments"].extend(comment_rows) - _act_row_patch(patches, task["table"], task["rowId"], preset_cells(res, pulled)) - affected.append(task["rowId"]) - - def _backup_profile(handle): - """The second rung, asked ONLY once the first has finished and delivered nothing. - - ⭐ 2026-08-09 (owner: *"when it errors like this, just route to APIfy"*). THE GAP THIS - CLOSES: `pull_profile` has walked `ig_profile -> (primary, backup)` since 2026-08-08, but - a profile that DEFERRED never came back through `pull_profile` — it came back here, and - this collector had no second rung at all. So the one path where the primary is most - likely to have failed was the one path with no fallback. - - ⛔ NEVER A TOP-UP. It runs only in the finished-and-empty branch, so a profile the primary - answered is never re-bought from a second vendor — the mistake the capability split exists - to prevent. A refusal returns None and the caller reports blocked exactly as before. - """ - try: - import providers as _p - if not _p.PROVIDERS["apify"].can("ig_profile"): - return None - prof, note = apify_profile(str(handle)) - except Exception as exc: # noqa: BLE001 — a backup must not raise - log(f"[aios-auto] backup profile rung failed for @{handle}: " - f"{type(exc).__name__}: {exc}") - return None - if prof and prof.get("followers") is not None: - return prof - return None - - def _rows_of(sid): - if sid not in _rows_memo: - payload, err = bd_call(f"{BD_PATH_SNAPSHOT}/{sid}", {"format": "json"}) - got = _bd_rows(payload) if not err else [] - if got and (_bd_deferral(payload) or - (len(got) == 1 and str(got[0].get("status") or "") in - ("running", "building", "collecting"))): - got = [] - _rows_memo[sid] = got - return _rows_memo[sid] - - for task in tasks: - stale_h = _hours_since(task.get("requestedAt")) - state, records, empty_note = _progress_of(task["snapshotId"]) - if state in ("done", "failed") and not records: - acc["profiles"] += 1 - # ⭐ ASK THE BACKUP BEFORE GIVING UP. The primary has FINISHED and delivered nothing, - # so there is nothing left to wait for and no risk of buying the same record twice. - backup = _backup_profile(task["influencer"]) - if backup is not None: - ready += 1 - acc["ok"] += 1 - _store_profile(task, backup, "apify") - note = (f"@{task['influencer']}: the primary source returned nothing, so a backup " - f"source supplied the profile ({backup.get('followers')} followers)") - notes.append(note) - acc["notes"].append(note) - continue - closed += 1 - acc["blocked"] += 1 - note = f"@{task['influencer']}: {_s(empty_note, 240)}" - notes.append(note) - acc["notes"].append(note) - # ⭐ `failed` = the vendor finished, collected nothing, and blamed the TARGET. On a - # profile request that means the account could not be reached at all, so the verdict - # is remembered and the selection stops re-buying it (R9: permanently, until a - # human edits the handle cell — see `clear_gone`). - # ⚠ `done`-with-zero is NOT remembered: "we found no matches" is a statement about - # the query, and turning it into "this account does not exist" would silently retire - # live handles. - if state == "failed": - acc["gone"][_gone_key({}, task["influencer"])] = { - "at": _iso(), "handle": task["influencer"], "note": _s(empty_note, 300)} - continue - rows = _rows_of(task["snapshotId"]) if state != "running" else [] - if not rows: - # ⚠ BOUNDED. A snapshot the vendor never finishes must not be polled until the end of - # time; after `PENDING_PROFILE_MAX_HOURS` it is dropped WITH a sentence, never - # silently. An unbounded queue is the forever-loop this change removes, inverted. - if stale_h is not None and stale_h >= PENDING_PROFILE_MAX_HOURS: - closed += 1 - note = (f"@{task['influencer']}: the source never finished the profile read " - f"queued {int(stale_h)}h ago ({task['snapshotId']}). It was dropped; " - f"the next run will ask again") - notes.append(note) - acc["notes"].append(note) - continue - waiting += 1 - remaining.append({**task, "lastChecked": _iso(), - "lastNote": _s("still building", 200)}) - continue - # ⛔ THIS HANDLE'S ROW, NOT THE FIRST ONE — see `_match_profile_row`. A batched snapshot - # holds several profiles and `rows[0]` would write one creator's numbers onto every record - # in the chunk. - node = _match_profile_row(rows, task["influencer"]) - if node is None: - closed += 1 - acc["profiles"] += 1 - acc["blocked"] += 1 - note = (f"@{task['influencer']}: the source delivered {len(rows)} profile" - f"{'' if len(rows) == 1 else 's'} for that batch, none of them this handle. " - f"it was dropped from the batch and the next run will ask again") - notes.append(note) - acc["notes"].append(note) - continue - ready += 1 - acc["profiles"] += 1 - profile = _bd_profile(node, task["influencer"]) - if profile.get("followers") is None and profile.get("following") is None: - acc["blocked"] += 1 - note = (f"@{task['influencer']}: the source delivered the profile but no " - f"follower/following counts were readable in it") - notes.append(note) - acc["notes"].append(note) - continue - acc["ok"] += 1 - _store_profile(task, profile, "brightdata:deferred") - notes.append(f"@{task['influencer']}: collected the profile the source had already been " - f"paid for ({profile.get('followers')} followers)") - - set_state(rt, str(defn.get("id") or ""), - {"pendingProfileSnapshots": remaining or None}) - counts = _enrich_flush(rt, defn, username, acc, log) - counts.pop(RUN_NOTES_KEY, None) # this function owns the note list below - counts.update(_commit_action_writes(rt, str(_flow_table(defn) or ""), patches, {}, - username, log)) - counts.update({"profileBatchesCollected": ready, "profileBatchesPending": waiting, - "profileBatchesEmpty": closed}) - if notes: - counts[RUN_NOTES_KEY] = notes - head = (f"{ready} deferred profile read{'' if ready == 1 else 's'} collected" - if ready else "no deferred profile read was ready") - tail = "".join([f"; {closed} finished with nothing to collect" if closed else "", - f"; {waiting} still building" if waiting else ""]) - state = "ok" if ready and not closed else "partial" - return (state, head + tail, counts, affected, - {"source": "ok" if ready else "partial", "write": "ok" if ready else "idle"}) - - -def _write_collected_metric_rows(rt, defn, username, idents, snapshots, comments, log): - """Write a completed metric snapshot through the canonical Post/Comment graph once.""" - if not (idents or snapshots or comments): - return {"posts": 0, "snapshots": 0, "comments": 0} - profile_table = str(_flow_table(defn) or "") - if not profile_table: - raise Refused("the pending engagement snapshot has no Profile database to link to") - graph = ensure_ig_graph(rt, username, str(defn.get("id") or ""), - profile_table=profile_table) - post_key, ps_key, comment_key = (graph[IG_POSTS_TABLE], graph[IG_POST_SNAPSHOTS_TABLE], - graph[IG_COMMENTS_TABLE]) - old_posts, collapsed_posts = dedupe_canonical_rows( - dict((ut_get(rt, post_key) or {}).get("rows") or {}), "shortcode", newest_by="measured_at") - posts, c_post = upsert_rows(old_posts, idents, "shortcode", cap=row_cap(post_key)) - c_post["duplicates"] += collapsed_posts - psnaps, c_ps = upsert_rows(dict((ut_get(rt, ps_key) or {}).get("rows") or {}), snapshots, - "post_snapshot_key", cap=row_cap(ps_key)) - old_comments, collapsed_comments = dedupe_canonical_rows( - dict((ut_get(rt, comment_key) or {}).get("rows") or {}), "comment_key") - comments_rows, c_comments = upsert_rows(old_comments, comments, "comment_key", - cap=row_cap(comment_key)) - c_comments["duplicates"] += collapsed_comments - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - for key, rows in ((post_key, posts), (ps_key, psnaps), (comment_key, comments_rows)): - if cur.get(key) is not None: - cur[key]["rows"] = rows - _refresh_relations_inplace(cur, log=log) - return cur - - rt.update(UT_STORE_KEY, _up, flush="sync") - try: - import ig_master - status, note = ig_master.append_run(getattr(rt, "key", ""), [], idents, snapshots) - if status == "error": - log(f"[aios-auto] deferred post metrics master copy FAILED: {note}") - except Exception as exc: # noqa: BLE001 - log(f"[aios-auto] deferred post metrics master copy FAILED: {type(exc).__name__}: {exc}") - return {"posts": c_post["inserted"], "snapshots": c_ps["inserted"], - "comments": c_comments["inserted"]} - - -def collect_pending_metric_snapshots(rt, defn, username="automation", log=print, step=_no_step): - """Collect deferred paid Post/Reel/Comment snapshots; never launches a new scrape request.""" - tasks = _pending_metric_tasks(defn) - if not tasks: - return ("ok", "No pending post-engagement snapshots.", {}, [], {"capture_posts": "idle"}) - step(f"Collecting {len(tasks)} post-engagement batch{'' if len(tasks) == 1 else 'es'}") - remaining, idents, snapshots, comments = [], [], [], [] - # ⭐ W30 · D-156 — TikTok's rows accumulate SEPARATELY and are written by TikTok's own writer. - # One queue can hold both networks' snapshots (they are keyed by dataset), but the two write - # paths address different tables and neither may touch the other's. - tt_collected = {"posts": [], "psnaps": [], "comments": []} - ready, waiting, closed, run_notes = 0, 0, 0, [] - for task in tasks: - # ⭐⭐ 2026-08-09 — ASK THE STATUS DOCUMENT, NOT THE ROWS. `building` used to be - # `not rows or …`, so a snapshot the vendor had FINISHED with zero records was re-queued - # as "still building" on every tick — forever, because nothing about it would ever - # change. That is the same forever-loop the profile path was measured in, one dataset - # over, and it was latent here the whole time. - state, records, empty_note = bd_snapshot_progress(task["snapshotId"]) - if state in ("done", "failed") and not records: - # ⛔ CLOSED, NOT RE-QUEUED. The vendor is finished and there is nothing to collect; - # keeping the entry would be a pending task that can never resolve. - closed += 1 - run_notes.append(f"{task['influencer']}: {_s(empty_note, 160)}") - continue - payload, note = bd_call(f"{BD_PATH_SNAPSHOT}/{task['snapshotId']}", {"format": "json"}) - rows = _bd_rows(payload) if not note else [] - building = (state == "running" or not rows or _bd_deferral(payload) or - (len(rows) == 1 and str(rows[0].get("status") or "") in - ("running", "building", "collecting"))) - if building: - remaining.append({**task, "lastChecked": _iso(), - "lastNote": _s(note or "still building", 160)}) - waiting += 1 - continue - ready += 1 - pulled = _iso() - # ⭐⭐ WAVE 30 · D-156 — THE TIKTOK ARM. Before this, `_PENDING_METRIC_DATASETS` was - # Instagram's three, so a TikTok media batch the vendor deferred was paid for and - # collected by NOTHING — the note carried the snapshot id and a human was the only - # collector. It routes through `connectors_tt`'s own mappers and `tt_capture_rows`, - # never Instagram's, because every one of those stamps `PLATFORM_INSTAGRAM`. - # ⛔ TWO THINGS THE INSTAGRAM ARM DOES THAT THIS ONE MUST NOT, both deliberate: - # * NO `top_up_views`. TikTok's `play_count` arrives inline on the posts row, so there - # is no views capability to route to; calling it would buy an Instagram permalink. - # * NO `ig_master.append_run`. There is no TikTok master by design (`_tt_enrich_flush`), - # and a write-through would put a TikTok creator into Instagram's pooled history, - # which the metric FIELDS read — a wrong number in a permanent series. - if tt_metric_kind(task["datasetId"]): - _tt = _tt_module() - who = task["influencer"] - if task["kind"] == "posts": - mapped = [m for m in (_tt.normalize_post(r) for r in rows) if m] - for m in mapped: - # The snapshot was bought from THIS profile's own permalinks, so the backlink - # is a fact of the call even when a vendor row omits the author field. - m.setdefault("influencer_key", who) - res_tt = {"profile": {"handle": who}, "posts": mapped, "comments": []} - else: - mapped = [m for m in (_tt.normalize_comment(r) for r in rows) if m] - res_tt = {"profile": {"handle": who}, "posts": [], "comments": mapped} - tt_idents, tt_metrics, tt_comments = tt_capture_rows(res_tt, pulled) - tt_collected["posts"].extend(tt_idents) - tt_collected["psnaps"].extend(tt_metrics) - tt_collected["comments"].extend(tt_comments) - continue - if task["kind"] == "posts": - mapped_posts = [] - for raw in rows: - post = _bd_post_metrics(raw) - if post: - # The snapshot was created from this Profile's canonical post URLs. Keep - # that explicit backlink even when a vendor response omits `user_posted`. - post["influencer_key"] = task["influencer"] - mapped_posts.append(post) - if mapped_posts: - # ⭐⭐ 2026-08-09 — THE VIEWS TOP-UP RUNS HERE TOO, and its absence is why the - # owner's `theresalearns` run filled every column except Views. - # - # ⛔ Bright Data is DECLARED INCAPABLE of `ig_post_views` (`providers.py`), so a - # Posts row physically cannot carry a view count — MEASURED on the stored - # payloads: 12 rows, `content_type: "Reel"`, and no view/play key in any of them. - # Views only ever comes from the Apify capability. That top-up lived INSIDE - # `pull_profile_bd`, so it ran only when the Posts scrape answered within the wait - # budget; when the batch deferred — which is routine, and what happened here — the - # rows came back through THIS function and Apify was never asked. - # ⇒ `top_up_views` is now one function with two callers rather than a copy, so - # the inline and deferred paths cannot answer this differently again. - # ⚠ It mutates `mapped_posts` in place and must run BEFORE `capture_rows`, which - # is what freezes the values into the post + snapshot rows. - v_note = top_up_views(mapped_posts, log=log) - if v_note: - run_notes.append(f"@{task['influencer']}: {v_note}") - _unused, post_rows, metric_rows, embedded = capture_rows( - {"state": "ok", "profile": {"username": task["influencer"]}, - "posts": mapped_posts, "comments": [], "via": "brightdata:deferred"}, pulled) - idents.extend(post_rows) - snapshots.extend(metric_rows) - comments.extend(embedded) - else: - for raw in rows: - comment = _bd_comment(raw, influencer_key=task["influencer"]) - if comment: - comments.append(comment) - - set_state(rt, str(defn.get("id") or ""), - {"pendingMetricSnapshots": remaining or None}) - written = _write_collected_metric_rows(rt, defn, username, idents, snapshots, comments, log) - counts = {"metricBatchesCollected": ready, "metricBatchesPending": waiting, - "metricBatchesEmpty": closed, - "postEngagementSnapshots": written["snapshots"], - "commentsCollected": written["comments"]} - # ⭐ W30 · D-156 — the TikTok write, through the SAME function the inline enrich uses, so a - # collected batch and an inline one cannot land differently. Its counts carry the `tt` prefix - # for the reason every other TikTok count does. - if any(tt_collected.values()): - tt_inserted, tt_capped, tt_missing = _tt_write_tables( - rt, str(defn.get("id") or ""), username, [], tt_collected["posts"], - tt_collected["psnaps"], tt_collected["comments"], log) - for key, name in ((TT_POSTS_TABLE, "ttPostsCollected"), - (TT_POST_SNAPSHOTS_TABLE, "ttPostSnapshotsCollected"), - (TT_COMMENTS_TABLE, "ttCommentsCollected")): - if tt_inserted.get(key): - counts[name] = tt_inserted[key] - if tt_capped: - counts["ttCollectCapped"] = tt_capped - log(f"[aios-auto] collect: {tt_capped} TikTok row(s) refused by a table's row cap") - if tt_missing: - log(f"[aios-auto] collect: could not create {', '.join(tt_missing)}") - # ⚠ THE RUNNER CONTRACT STAYS A 5-TUPLE and the notes ride in `counts` under the reserved - # `RUN_NOTES_KEY`, which `run_now` pops. Widening the tuple for one runner would make four - # other call sites disagree about the shape of a run — and `_commit_run` already drops - # non-numeric count values, so a pop that is ever missed degrades to today's behaviour rather - # than to a crash. - if run_notes: - counts[RUN_NOTES_KEY] = run_notes - # ⚠ THE EMPTY ONES ARE NAMED, not silently dropped. A batch that finished with no records is - # a real outcome the tenant paid for and it must read as an answer, not as a disappearance. - tail = (f"; {closed} finished with nothing to collect" if closed else "") - if waiting: - return ("partial", - f"{ready} post-engagement batch{'' if ready == 1 else 'es'} collected; " - f"{waiting} still building and will be collected automatically{tail}", - counts, [], {"capture_posts": "partial", "write": "ok"}) - return ("ok", f"{ready} post-engagement batch{'' if ready == 1 else 'es'} collected{tail}", - counts, [], {"capture_posts": "ok", "write": "ok"}) - - -def ai_decide(rt, defn, act, row, row_id="", log=print): - """R4/C6: let the model pick this card's next stage. Returns the chosen stage LABEL, or "" - to leave the card for a person. - - ⛔ FAIL-CLOSED IN EVERY DIRECTION: no provider configured, a network failure, a malformed - answer, or a label the review does not offer all return "" — and "" means the card sits at - the review gate exactly as it would with no AI at all. The feature can be broken, absent or - wrong and the worst outcome is a human doing the work. - - ⚠⚠ PARKED SINCE WAVE 27 — THIS FUNCTION HAS NO CALLER, AND THAT IS RECORDED RATHER THAN - ACCIDENTAL. Its one caller was the `review` branch of the action walk, deleted with the board - under R3, which keeps review "as an AI decision without lanes". What is parked is genuinely - worth parking: the cheap-first provider ladder in `ai_review` (groq → cerebras → openrouter → - anthropic), the fail-closed posture above, and the audit shape `review_audit` writes. What is - MISSING is only the door — an action kind that asks a question and takes an answer, without a - stage column to write it into. **Do not delete this in a dead-code sweep without reading that - sentence first**; equally, do not treat it as shipped — nothing reaches it today. - """ - cfg = act.get("config") or {} - options = list(cfg.get("next") or []) - if not options: - return "" - try: - import ai_review - except Exception as e: # noqa: BLE001 - log(f"[aios-auto] ai review unavailable: {type(e).__name__}: {e}") - return "" - fields = [f.get("key") for f in - (ut_get(rt, ((defn.get("config") or {}).get("targetTable") - or (defn.get("trigger") or {}).get("table") or "")) or {} - ).get("fields") or []] - # ⭐ W35 · CONTRACT C7 (`NOTE E-16`) — ATTRIBUTE THE SPEND. `st=` is what lets the ledger write - # to the right tenant's store and `user=` is who it is billed to; without them the call is - # counted as UNATTRIBUTED, which is a meter that reports a total nobody can act on. - # ⚠ `createdBy` is the honest actor here: an AI review decision is made ON BEHALF of the - # automation, by a scheduler, with no person at the keyboard. Naming whoever last edited it - # would attribute a nightly run to an editor who was asleep. - choice, meta = ai_review.decide(prompt=cfg.get("prompt") or "", options=options, - row=row, fields=[f for f in fields if f], - label=cfg.get("label") or "Review", - st=rt, user=str(defn.get("createdBy") or "")) - if not choice: - if meta.get("problem"): - log(f"[aios-auto] ai review declined to answer: {meta['problem']}") - return "" - review_audit(rt, defn.get("id"), row_id, cfg.get("label") or "Review", - choice, meta.get("provider") or "ai", by="ai", - note=meta.get("reason") or "", model=meta.get("model") or "") - return choice - - -def find_records(rt, table_key, cond, limit=25): - """The `find_records` action's read: matching row ids, bounded and DISCLOSED (the caller puts - the count in the run log, where it drills — [[no-unverifiable-aggregates]]).""" - rows = (ut_get(rt, str(table_key or "")) or {}).get("rows") or {} - out = [] - for rid, row in rows.items(): - if lane_match(cond, row or {}): - out.append(str(rid)) - if len(out) >= max(1, min(int(limit or 25), FIND_LIMIT_MAX)): - break - return out - - -def _commit_action_writes(rt, table, patches, creates, username, log): - """The ONE write. Row patches merge into the target's rows; creates land in their own tables — - APPENDED, or UPSERTED on the action's `uniqueOn` key (C5 / owner ruling R1a). - - Returns the REALIZED create counts, and returning them is the point rather than a - convenience. `apply_actions` counts an ATTEMPT per create while it walks the records; only - this function knows how many of those became rows, how many matched one that was already - there, and how many the cap refused. A run that reported the attempt as "created" would be - the summary-disagrees-with-what-happened defect this module names in three other places. - - ⛔ THE UPSERT IS `upsert_rows`, NOT A MATCH LOOP WRITTEN HERE. D-6 closed on "ONE - implementation repo-wide" after a second one was deleted from `core/user_tables.py`; hand - rolling a third inside this function is that debt returning with a new name — and it would - quietly diverge on the two rules that took a wave each to get right (an orphan is COUNTED, - NEVER DELETED, and `capped` is its own count rather than folded into `skipped`). - """ - out = {"created": 0, "createUpdated": 0, "createUnchanged": 0, - "createCapped": 0, "createSkipped": 0} - if not patches and not creates: - return out - # ⭐ PATCHES ARE STAGED, NOT WRITTEN YET (2026-08-06). They used to commit here, one - # `ut_write_rows` per patched table, and that was free while the only patcher was a review - # gate on a table no create action touched. Owner item 1 stamps EVERY walked record with the - # step it reached, so the flow's own table is now patched on essentially every run — and a - # flow that also creates into that same table would have committed it TWICE per run, against - # the 20 s flush floor and the 256-commits/hr repo budget this module is shaped around. - # Staged into `staged` and handed to the creates pass, which writes each table exactly once. - staged = {} - for tkey, rowpatch in (patches or {}).items(): - cur = dict((ut_get(rt, tkey) or {}).get("rows") or {}) - for rid, vals in rowpatch.items(): - cur[str(rid)] = {**(cur.get(str(rid)) or {}), **vals} - staged[str(tkey)] = cur - # ⛔ ONE WRITE PER TABLE, even when several actions target it under different keys. The - # accumulator is keyed by (table, uniqueOn), so a naive loop would call `ut_write_rows` once - # per GROUP — two store commits for one table, against the 20 s flush floor and the 256/hr - # repo budget this whole module is shaped around. - by_table = {} - for (tkey, unique), new_rows in (creates or {}).items(): - by_table.setdefault(str(tkey), []).append((str(unique or ""), new_rows)) - for tkey, groups in by_table.items(): - t = ut_get(rt, tkey) - if t is None: - n = sum(len(r) for _u, r in groups) - log(f"[aios-auto] create_record: {tkey} no longer exists. {n} skipped") - out["createSkipped"] += n - continue - # The PATCHED rows when this table was also stamped this run, so the creates land on top - # of the stamp rather than on a copy of the store that predates it. - cur = staged.pop(tkey, None) - cur = dict(t.get("rows") or {}) if cur is None else cur - cap = row_cap(tkey) - for unique, new_rows in groups: - if unique: - cur, c = upsert_rows(cur, new_rows, unique, cap=cap) - out["created"] += c["inserted"] - out["createUpdated"] += c["updated"] - out["createUnchanged"] += c["unchanged"] - out["createCapped"] += c["capped"] - # ⚠ `skipped` here means "this row had no value for the unique key", which for a - # create action is a mapped value that interpolated to nothing — worth surfacing, - # because the symptom is otherwise a run that says it created less than it walked. - out["createSkipped"] += c["skipped"] - if c["capped"]: - log(f"[aios-auto] create_record: {tkey} at its {cap}-row cap. " - f"{c['capped']} row(s) not written") - continue - nxt = max([int(r) for r in cur if str(r).isdigit()] or [0]) + 1 - for i, vals in enumerate(new_rows): - if len(cur) >= cap: - # THE [:N] HONESTY RULE ([[no-unverifiable-aggregates]]): the number DROPPED is - # named, here and in the run's counts. This used to `break` with a log line - # that said the cap was hit and never said how much was lost. - out["createCapped"] += len(new_rows) - i - log(f"[aios-auto] create_record: {tkey} at its {cap}-row cap. " - f"{len(new_rows) - i} row(s) not written") - break - cur[str(nxt)] = dict(vals) - nxt += 1 - out["created"] += 1 - ut_write_rows(rt, tkey, cur) - # Whatever the creates pass did NOT claim: tables this run only STAMPED. Written last and - # once each, so "one store write per table" holds whether a table was patched, created into, - # or both. - for tkey, cur in staged.items(): - ut_write_rows(rt, tkey, cur) - return out - - -def _lane_sentence(cond, top=True): - """One condition tree → the sentence a step's `detail` carries on the canvas. - - C4: a GROUP renders as its children joined by "and"/"or" and parenthesised when nested, so a - label never claims a flat comparison the tree does not make. The SERVER composes it, for the - same reason it composes every other `detail` — a client paraphrase of a structure the engine - evaluates is a second implementation of the same sentence, free to drift from it. - - ⚠ The name is board-era ("lane") and the board is gone; the caller is `graph()`, which is - live. Renamed nothing on purpose: this string is compared in a gate and read in a log, and a - rename would be churn on a working function to fix a word. - """ - if cond is None: - return "Everything else" if top else "" - if isinstance(cond, dict): - for key, joiner in (("all", " and "), ("any", " or ")): - if key in cond: - parts = [_lane_sentence(c, False) for c in cond.get(key) or []] - parts = [p for p in parts if p] - if not parts: - return "" - inner = joiner.join(parts) - return inner if top or len(parts) == 1 else f"({inner})" - v = cond.get("value") - return f"{cond.get('field')} {cond.get('op')}" + ("" if v is None else f" {v}") - - -def _rid_num(rid): - return int(rid) if str(rid).isdigit() else 10 ** 9 - - -#: How many review decisions an automation remembers. Bounded like `runs` — an audit that can -#: grow a definition without limit is a serialisation cost wearing a compliance hat. -MAX_REVIEWS = 100 - - -def review_audit(rt, auto_id, row_id, from_label, to_label, username, - by="user", note="", model=""): - """C3-A2(5): a review decision is AUDITED — who moved which card where, when. Appended to - the definition (newest first, bounded). - - ⭐ WAVE 23 (R4/C6): an AI decision writes THE SAME ROW with `by: "ai"` plus the model that - made it and the one-line reason it gave. One audit log, not two — a reader asking "who - decided this card" must not have to know there are two places to look, and the moment an - AI decision is invisible beside a human one the log stops being an audit. - - ⚠ PARKED SINCE WAVE 27, with `ai_decide` and for the same reason. Both of its doors are gone: - `move_card` was deleted with the board (R3), and the grid door in `core.grid_events` wrote - this shape off a stage field's `flowId` — a field the migration now drops. Kept because the - SHAPE is the contract a future decision action would write, and re-deriving an audit format - is how two of them end up existing. - """ - entry = {"ts": _iso(), "user": _s(username, 80), "rowId": str(row_id), - "from": _s(from_label, 60), "to": _s(to_label, 60), - "by": "ai" if by == "ai" else "user"} - if note: - entry["note"] = _s(note, 300) - if model: - entry["model"] = _s(model, 60) - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - d = cur.get(str(auto_id)) - if d is not None: - d["reviews"] = ([entry] + list(d.get("reviews") or []))[:MAX_REVIEWS] - return cur - - _store_update(rt, _up, flush="sync") - - -# --------------------------------------------------------------------------------------------- -# METRIC FIELDS (wave 22, contract C7) + THE SUBJECT PURGE (D-24) -# --------------------------------------------------------------------------------------------- -# A metric field is `{measure, window, agg}` over the MASTER series (C6's pooling benefit — -# the value reflects every pull the platform has, not just this tenant's). Computed by the -# ENGINE and stored as machine cells (C7 amendment: run+tick time — the ut wire lives in files -# no session owns this wave), human-write-refused at the grid door, and every number drills to -# the exact snapshot rows behind it. -# -# ⛔ NO DATA IS BLANK, NEVER ZERO. A handle the master has never seen, a window with no -# snapshots in it, a post series with no measured engagement — all read as an EMPTY cell. Zero -# is a measurement ("they have none"); blank is an admission ("we have not looked / it was not -# readable") — the `_bd_posts_count` law, one layer up. - -METRIC_MEASURES = ("followers", "avg_engagement", "likes", "comments") -METRIC_WINDOWS = ("latest", "last_3_posts", "last_7d", "last_30d") -#: Which measures read the PROFILE series vs the POST series — and which windows/aggs each -#: side can answer. A profile count over `last_3_posts` is a question the data cannot answer; -#: refused at clean time, never bent (mirrored in `core.user_tables`, gate-pinned). -PROFILE_MEASURES = ("followers", "avg_engagement") -METRIC_AGGS = ("avg", "sum", "latest") - - -def metric_value(series, measure, window, agg="", today=None): - """One metric over one handle's master series → `(value_string_or_None, drill_rows)`. - - `today` is a PARAMETER ([[date-window-vocabulary]]) — the caller decides the reference - day; post-count windows count back from the newest post. `drill_rows` are the exact rows - the number came from, so the route can honour [[no-unverifiable-aggregates]] without - recomputing differently.""" - series = series or {} - today = today or _now() - if measure in PROFILE_MEASURES: - rows = [r for r in series.get("snapshots") or [] - if str(r.get(measure) if r.get(measure) is not None else "").strip() != ""] - if window in ("last_7d", "last_30d"): - days = 7 if window == "last_7d" else 30 - floor = today - _dt.timedelta(days=days) - rows = [r for r in rows - if (_parse_iso(r.get("pulled_at")) or _dt.datetime.min) >= floor] - if not rows: - return None, [] - if window == "latest" or agg == "latest": - picked = [rows[-1]] - else: - picked = rows - vals = [_lane_num(r.get(measure)) for r in picked] - vals = [v for v in vals if v is not None] - if not vals: - return None, [] - out = vals[-1] if (window == "latest" or agg == "latest") else \ - (sum(vals) if agg == "sum" else sum(vals) / len(vals)) - if measure == "avg_engagement": - # The vendor's rate is 0-1; the pct cell renders POINTS (the semantic-pct vs - # transform-pct scar) — scaled exactly once, here. - return f"{out * 100:.2f}", picked - return (f"{out:.0f}" if float(out).is_integer() else f"{out:.2f}"), picked - # --- post measures: select POSTS by window, then each post's LATEST measured snapshot. - posts = [p for p in series.get("posts") or [] if str(p.get("posted_at") or "").strip()] - posts.sort(key=lambda p: str(p.get("posted_at"))) - if window == "latest": - picked_posts = posts[-1:] - elif window == "last_3_posts": - picked_posts = posts[-3:] - else: - days = 7 if window == "last_7d" else 30 - floor = today - _dt.timedelta(days=days) - picked_posts = [p for p in posts - if (_parse_iso(str(p.get("posted_at")).replace(" ", "T")) - or _dt.datetime.min) >= floor] - vals, drill = [], [] - for p in picked_posts: - snaps = (series.get("postSnapshots") or {}).get(str(p.get("shortcode") or "")) or [] - for snap in reversed(snaps): - raw = snap.get(measure) - v = _lane_num(raw) - if v is not None and str(raw).strip() != "": - vals.append(v) - drill.append(snap) - break - if not vals: - return None, [] - out = sum(vals) if (agg or "sum") == "sum" else \ - (vals[-1] if agg == "latest" else sum(vals) / len(vals)) - return (f"{out:.0f}" if float(out).is_integer() else f"{out:.2f}"), drill - - -def metric_fields_of_table(t): - return [f for f in ((t or {}).get("fields") or []) if isinstance(f.get("metric"), dict)] - - -def _table_handle(row, url_field): - return str(row.get("handle") or ig_handle(str(row.get(url_field or "") or "")) - or "").strip().lower() - - -def compute_metric_cells(rt, table_key, today=None, tables=None, persist=False): - """Recompute every metric cell on ONE table from the master series. One coalesced write, - only when something actually changed (the flush-ceiling law); zero reads when the table - has no metric fields or the master is off. Returns the number of rows touched.""" - owned = tables is not None - blob = tables if owned else None - t = ((blob or {}).get(str(table_key)) if owned else ut_get(rt, table_key)) - mfields = metric_fields_of_table(t) - if not mfields: - return 0 - import ig_master - if not ig_master.configured(): - return 0 - url_field = next((f.get("key") for f in (t.get("fields") or []) - if f.get("type") == "url"), "") - rows = t.get("rows") or {} - handles = {rid: _table_handle(row or {}, url_field) for rid, row in rows.items()} - series = ig_master.series_for({h for h in handles.values() if h}) - changes = {} - for rid, row in rows.items(): - s = series.get(handles.get(rid) or "") - for f in mfields: - bag = f["metric"] - val, _drill = (metric_value(s, bag.get("measure"), bag.get("window"), - bag.get("agg") or "", today=today) - if s else (None, [])) - want = "" if val is None else str(val) - if str((row or {}).get(f["key"], "")) != want: - changes.setdefault(str(rid), {})[f["key"]] = want - if not changes: - return 0 - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - tt = cur.get(table_key) - if tt is not None: - for rid, vals in changes.items(): - tt.setdefault("rows", {}).setdefault(rid, {}).update(vals) - return cur - - if owned: - _up(blob) - if persist: - rt.update(UT_STORE_KEY, _up, flush="sync") + # An ORDINARY link: the cell IS the relation. Ids the linked table no longer holds + # are dropped rather than carried — a link to a deleted row is not a link. + lrows = (linked.get("rows") or {}) + resolved[lk_key] = { + str(rid): [(i, lrows[i]) for i in + [s.strip() for s in str((row or {}).get(lk_key) or "").split(",")] + if i and i in lrows] + for rid, row in rows.items()} + + changes = {} + for rid, row in rows.items(): + rid = str(rid) + row = row or {} + for f in links: + fk = str(f["key"]) + # ⛔ A REFUSED LINK IS SKIPPED, NOT BLANKED — the difference between "this could not + # be resolved, here is why" and "there are no linked records". Its cell keeps the last + # value that WAS resolvable; the refusal rides out in `limits`. + if fk in refused: + continue + hits = (resolved.get(fk) or {}).get(rid) or [] + if f["link"].get("single"): + hits = hits[:1] + # ⛔ THE CAP IS A DISPLAY CAP AND THE ROLLUPS DO NOT READ THROUGH IT. A derived cell + # is a projection of `resolved`, which is uncapped and is what every rollup below + # consumes — so a profile with 900 posts shows the first 500 ids and still averages + # over all 900. Same argument the `posts` window already makes: shedding is safe + # precisely because the authoritative store still holds everything. + want = ",".join(i for i, _r in hits[:_ut().LINK_MAX_IDS]) + if str(row.get(fk, "")) != want: + changes.setdefault(rid, {})[fk] = want + for f in rollups: + fk, bag = str(f["key"]), f["rollup"] + lk_key = str(bag.get("link") or "") + # Same law one loop up: a rollup whose LINK was refused folds nothing and is left + # alone, rather than printing a 0 that reads as a measurement. + if lk_key in refused: + continue + hits = list((resolved.get(lk_key) or {}).get(rid) or []) + # ⚠ A rollup whose link field does not exist (renamed, deleted) resolves to NOTHING + # and therefore to a blank cell — never to a stale number. A column that keeps + # printing yesterday's answer after its input is gone is the worst of the options. + # ⭐⭐ WAVE 28 / CONTRACT C1 — SCOPE FIRST, FILTER SECOND, AND THE TWO USED TO BE THE + # OTHER WAY ROUND. The conditions block stood HERE, above the ranking, so + # "last 10 posts where views > X" meant *the 10 most recent of the posts over X* + # rather than *the ones over X among the last 10* — two different windows wearing one + # sentence. Harmless while a threshold was a literal; incoherent the moment a + # threshold is a statistic OF the window, because the set being described and the set + # doing the describing would be different sets. + # ⛔ THIS ORDER IS THE CONTRACT, not an implementation choice: `core.user_tables`'s + # `ROLLUP_REF_OPS` note states it ("the scope picks the window, THEN the threshold is + # computed over that window, THEN the conditions filter it") and the validator half + # was written against it. + # ⚠ IT IS A BEHAVIOUR CHANGE FOR EXACTLY ONE SHAPE: a stored rollup carrying BOTH + # `conditions` AND `limit`. No shipped preset does (measured across `odoo_relational` + # and the IG presets — the one preset with conditions, `_OPEN_ONLY`, is a `countall` + # with no limit), so the blast radius is user-built rollups only. + # ⭐⭐ 2026-08-09 (owner) — THE PRE-FILTER, ABOVE THE RANKING. Owner: *"instead of last + # 12 posts, we also want to make it so its last N record, where the record's Status is + # video."* `conditions` cannot answer that: C1 moved them BELOW the window on purpose, + # so they select among the rows the window already kept. `where` selects WHICH rows + # the window is spent on. + # ⛔ ABOVE `distinctBy` TOO, not merely above the sort. Dedup keeps the first row per + # identity; run it first and a carousel could claim the slot its reel sibling needed, + # so the window would come up short for a reason nothing on screen explains. + # ⚠ NO `ref` REACHES HERE — `_clean_rollup` refuses a set-statistic threshold in this + # list, because at this point there is no fixed set for a statistic to be about. + where = list(bag.get("where") or []) + if where: + where_matches = lambda pair: [ + _rollup_condition_matches(pair[1], condition, + linked_types.get(lk_key) or {}, ref_value=None) + for condition in where] + if str(bag.get("whereConj") or "and") == "or": + hits = [pair for pair in hits if any(where_matches(pair))] + else: + hits = [pair for pair in hits if all(where_matches(pair))] + sort_by = str(bag.get("sortBy") or "") + if sort_by: + ftype = (linked_types.get(lk_key) or {}).get(sort_by, "text") + # ⛔ PARTITION, THEN SORT. A row whose sort cell is blank or unparseable is not + # rankable, and it must land at the END whichever direction is asked for — which + # a sentinel inside the sort key cannot do, because `reverse` flips the sentinel + # too (see `_sort_key`). Unrankable rows are appended, so a `limit` spends its + # window on rows that HAVE the value before it falls back to ones that do not. + keyed = [(pair, _sort_key(pair[1].get(sort_by), ftype)) for pair in hits] + rankable = [(p, k) for p, k in keyed if k is not None] + rankable.sort(key=lambda pk: pk[1], + reverse=str(bag.get("sortDir") or "desc") == "desc") + hits = [p for p, _k in rankable] + [p for p, k in keyed if k is None] + distinct_by = str(bag.get("distinctBy") or "") + if distinct_by: + seen, unique = set(), [] + for pair in hits: + identity = str(pair[1].get(distinct_by) or "").strip().lower() + # A blank is not an identity. Keep it rather than collapsing every unknown + # record into one synthetic duplicate. + if identity and identity in seen: + continue + if identity: + seen.add(identity) + unique.append(pair) + hits = unique + limit = int(bag.get("limit") or 0) + if limit: + hits = hits[:limit] + # --- the window is now FIXED, so a set-statistic threshold has a set to be about. + conditions = list(bag.get("conditions") or []) + if conditions: + # ⚠ RESOLVED ONCE PER LEAF, NOT ONCE PER ROW. The threshold is a property of the + # window; computing it inside `matches` would recompute the same mean for every + # candidate and — worse — would invite computing it over a set that the filter is + # already shrinking underneath it. + # ⚠ `.get("sigmas", 0.0)`, never `... or 0.0` — a legitimate `sigmas: 0` ("beyond + # the mean") is falsy, and the `or` spelling would silently rewrite it to the same + # number by accident. It reads identically and is right for the wrong reason, + # which is how it survives a review. + refs = [_rollup_ref_threshold( + [r for _i, r in hits], str(c.get("field") or ""), + (c.get("ref") or {}).get("sigmas", 0.0)) + if isinstance(c, dict) and c.get("ref") is not None else None + for c in conditions] + matches = lambda pair: [ + _rollup_condition_matches(pair[1], condition, + linked_types.get(lk_key) or {}, ref_value=ref) + for condition, ref in zip(conditions, refs)] + if str(bag.get("conditionConj") or "and") == "or": + hits = [pair for pair in hits if any(matches(pair))] + else: + hits = [pair for pair in hits if all(matches(pair))] + src = str(bag.get("field") or "") + # D-92 — the SOURCE column's declared type, read from the same `linked_types` map the + # sort path above uses. `countall` folds a synthetic `[1]*n` with no source column at + # all, so the default stands for it. + want = _rollup_fold(str(bag.get("fn") or ""), + [r.get(src) for _i, r in hits] if src else [1] * len(hits), + (linked_types.get(lk_key) or {}).get(src, "text")) + if str(row.get(fk, "")) != want: + changes.setdefault(rid, {})[fk] = want + return changes, limits + + +def compute_relation_cells(rt, table_key, tables=None): + """Recompute every DERIVED LINK cell and every ROLLUP cell on ONE table. Returns rows touched. + + Zero store reads when the table declares neither kind — the same cheap-by-construction shape + `compute_metric_cells` has, so walking every table on a tick costs a dict scan per table. + + ⭐ W41-T18 — THE PERSISTING WRAPPER OVER `relation_cells`, WHICH IS WHERE THE WORK MOVED. A + caller that wants the cells themselves (the grid render for a registry module, the Relational + pivot) calls that one and reads BOTH halves of its answer; this one exists for the tick, which + only ever wanted the count. + + ⛔ A REGISTRY MODULE PERSISTS NOTHING HERE, AND THE COUNT IS STILL TRUE. `customer_data` and + `product_data` store no rows in the `user_tables` document — their grids are assembled per + render from the tenant's Odoo pool — so there is nothing on disk for a derived cell to update + and `_up` below would find no table to write into. The count reports cells RESOLVED, and the + cells themselves are what `relation_cells` hands back; a caller that needs them must call it. + + ⛔⛔ THE SIGNATURE IS FROZEN, AND IT COST A GATE RUN TO LEARN WHY. A first cut added a + `log=None` kwarg here so the tick could print a refusal. `verify_automation`'s NC55 WRAPS this + function (`newest_wins(rt, table_key, tables=None)`) — as any wrapper reasonably would — and + the new keyword made every call a `TypeError`. `_refresh_relations_inplace` CATCHES every + exception and logs it, so the whole relational pass went dead with nothing red on the main + run: 2224/2224 still passed and only the negative-control sweep noticed, two checks deep in + another section. ⇒ Refusals ride out on `relation_cells`' SECOND RETURN VALUE, where the + caller that renders them reads them, and nothing about this door changes shape. + """ + changes, _limits = relation_cells(rt, table_key, tables=tables) + if not changes: + return 0 + if str(table_key) in LINKABLE_MODULES: + return len(changes) + + # A run that just wrote linked rows passes its in-flight user_tables bucket here so the + # relation refresh joins the SAME coalesced commit. The standalone/tick path below keeps + # the public helper's old persist-on-change behaviour. + if tables is not None: + tt = tables.get(table_key) + if tt is not None: + for r, vals in changes.items(): + tt.setdefault("rows", {}).setdefault(r, {}).update(vals) return len(changes) + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + tt = cur.get(table_key) + if tt is not None: + for r, vals in changes.items(): + tt.setdefault("rows", {}).setdefault(r, {}).update(vals) + return cur + rt.update(UT_STORE_KEY, _up, flush="sync") return len(changes) - - -# ── ⭐⭐ THE RELATIONAL PASS (2026-08-07) — derived LINK cells and ROLLUP cells ──────────────── -# -# Owner: *"adding relational database function with links and rollups… exactly like how Airtable -# does it… and this rollup needs to have formula that we can use to calculate things like average -# Views over last N posts."* -# -# ⛔ WHY THIS IS SERVER-SIDE AND MATERIALISED, when `formula` is client-side and is not. A formula -# reads ONE ROW; a rollup reads ANOTHER TABLE'S ROWS, which the client has not loaded and must not -# have to. So this rides `compute_metric_cells`' pattern exactly — recompute, diff, ONE coalesced -# write only when something changed — and inherits its flush-ceiling discipline for free. -# -# ⭐ AND R1's "ONE STORE FOR ONE SERIES" SURVIVES, which is the thing to check before touching -# this. The authoritative post record is `ut_ig_posts`; the authoritative engagement series is -# `ut_ig_post_snapshots`. A rollup cell is a PROJECTION refreshed from them — the same standing as -# a `metric` cell, and the same standing as the `posts` json window (which is even allowed to SHED -# posts to fit, precisely because the store still holds them). ⛔ A rollup may only ever READ. The -# moment one writes a number nothing else can re-derive, it has become a third copy. - -def _ut(): - """`core.user_tables`, imported LAZILY — and the laziness is measured, not stylistic. - - Pulling this module in at import time initialises the store layer earlier than - `automation_engine` used to, and the last time that happened it moved a store-commit COUNT - from three to four on an unrelated gate. The relational pass - needs three constants and two predicates from the field layer; it does not need to change - when this module is imported. - """ - import core.user_tables as _m - return _m - - -#: The fns by family — how a value is folded, and what a blank means in each. -_ROLLUP_NUM_FNS = frozenset({"sum", "average", "stdev", "min", "max"}) -_ROLLUP_BOOL_FNS = frozenset({"and", "or", "xor"}) -#: The count/order family — folded by their own arms above the lanes. -_ROLLUP_SEQ_FNS = frozenset({"countall", "counta", "count", "latest"}) -#: The text family. ⛔ IT IS A NAMED SET NOW AND IT USED TO BE THE FALL-THROUGH, which is how -#: wave 28 nearly shipped a wrong number that looked like data: `stdev` validated and STORED -#: (`core.user_tables.ROLLUP_FNS`) a commit before this function learned it, and an fn no arm -#: claims fell past both lanes into the join below — so a "Std deviation" column rendered -#: `"137684, 19561, 8123"`. Filled, plausible, and not a statistic. An unrecognised fn now -#: returns `""` ([[gate-answers-the-wrong-question]]: blank is the honest answer to a question -#: nothing can answer; a comma-joined list is a different question's answer wearing this label). -_ROLLUP_TEXT_FNS = frozenset({"concatenate", "arrayjoin", "arraycompact", "arrayunique"}) -#: ⭐⭐ WHAT THIS FOLD ACTUALLY IMPLEMENTS, DERIVED FROM THE ARMS RATHER THAN RESTATED. -#: `verify_automation` asserts this is IDENTICAL to `core.user_tables.ROLLUP_FNS` name for name -#: (contract C1's parity leg), so the validator can never again accept a function the fold cannot -#: compute — in EITHER direction. A hand-listed copy in the gate would have gone green on the -#: defect above, because the defect was that the two lists already disagreed. -ROLLUP_FOLD_FNS = frozenset(_ROLLUP_NUM_FNS | _ROLLUP_BOOL_FNS | _ROLLUP_SEQ_FNS - | _ROLLUP_TEXT_FNS) -_ROLLUP_TEXT_FNS = frozenset({"concatenate", "arrayjoin", "arraycompact", "arrayunique"}) -#: What `arrayjoin` puts between values. Airtable uses ", "; `concatenate` uses nothing. -_ROLLUP_JOIN = ", " -#: A rollup's own cell ceiling. The text fns can concatenate a whole column into one cell, and a -#: cell nobody can read is not an aggregate. -ROLLUP_MAX_CHARS = 4000 - - -def _sort_key(value, ftype): - """One cell → a sortable key, typed by the LINKED field's declared type. `None` when the cell - is blank or does not parse as its declared type. - - ⛔ TYPE-AWARE ON PURPOSE. `posted_at` is a `date` and `views` is an `int`; sorting either as a - string puts `2026-9-1` before `2026-10-1` and `9` after `100`. Since `sortBy` is what decides - WHICH rows a `limit` keeps, getting this wrong does not mis-order a display — it silently - averages the wrong twelve posts. - - ⛔⛔ **BLANKS ARE PARTITIONED OUT BY THE CALLER, NEVER RANKED BY A FLAG — AND THE FLAG VERSION - SHIPPED BROKEN FOR ONE DEPLOY.** This returned `(1, 0.0, "")` for a blank and `(0, key, "")` - otherwise, documented as "a blank sorts LAST under desc". It does the OPPOSITE: `reverse=True` - flips the whole tuple, so `(1, …)` sorted FIRST and an undated row displaced the most recent - real one out of the window. A sentinel inside the key cannot mean "last" in both directions, - because the direction is applied to the sentinel too. - ⚠ AND IT WAS NOT A CORNER CASE ON THE RUNG THAT MATTERS: `wave20-split` MEASURED `datetime` as - `None` on 24/24 posts from the Profiles dataset, and `_bd_post_identity` writes `posted_at` - only when the vendor sent one — so on a paid profile pull EVERY post row is blank here, every - key tied, and `limit 12` took whatever twelve came first in dict order. The e2e test passed - because its fixture posts all carried dates. - ⇒ `None` means "not comparable", the caller keeps those rows at the END whichever way it - sorts, and "an unknown date is not a recent one" is finally what the code does. - """ - raw = "" if value is None else str(value).strip() - if raw == "": - return None - if ftype in ("int", "currency", "pct", "rating"): - n = _lane_num(raw) - return (n,) if n is not None else None - if ftype == "date": - d = _parse_iso(raw.replace(" ", "T")) - return (d.timestamp(),) if d is not None else None - return (raw.lower(),) - - -def _sample_stdev(nums): - """SAMPLE standard deviation (n-1) of `nums`, or None under two values. - - ⭐ ONE IMPLEMENTATION, TWO READERS, and that is the point of lifting four lines into a - function: `_rollup_fold` RENDERS this number into a "Std deviation" column and - `_rollup_ref_threshold` COMPARES rows against it inside a `sigmas` condition. A second copy - would let a column and the filter beside it disagree about the same word on the same set — - the exact drift `top_up_views` was extracted to prevent one module over - ([[one-evaluator-per-question]]). - ⛔ None means UNANSWERABLE and no caller may read it as 0. - """ - if len(nums) < 2: - return None - mean = sum(nums) / len(nums) - return (sum((n - mean) ** 2 for n in nums) / (len(nums) - 1)) ** 0.5 - - -def _rollup_ref_threshold(rows, field, sigmas): - """`mean + sigmas*stdev` of `field` over `rows` — a statistic OF THE SCOPED SET (contract C1). - - ⛔ `rows` MUST already be the scoped window: ranked by `sortBy`, deduped, and cut by `limit`, - and NOT yet filtered by the conditions this threshold feeds. That order is the contract - (`core.user_tables.ROLLUP_REF_OPS`'s note says so in as many words) and both other orders - produce a plausible number: computing it before `limit` answers "2 sigma of everything this - account ever posted" under a column that says "of the last 10", and computing it after the - filter makes the threshold depend on the rows it is choosing — a definition that chases - itself. - - Returns None when the window cannot answer — fewer than two numeric values in `field`. - ⛔ THE CALLER MUST DROP THE ROW, NOT KEEP IT. "Beyond 2 sigma of one post" is not a question - with a permissive answer; letting an unanswerable leaf pass everything would silently turn - "the outliers" into "all of them", which is this module's worst failure mode wearing a filter. - """ - nums = [n for n in (_lane_num((r or {}).get(field)) for r in rows) if n is not None] - sd = _sample_stdev(nums) - if sd is None: - return None - return (sum(nums) / len(nums)) + float(sigmas) * sd - - -def _rollup_fold(fn, values, ftype="text"): - """`values` (raw cells, in the order the window kept them) → the aggregate, as a STRING. - - Returns `""` for "nothing to aggregate", NEVER `0`. ⛔ That distinction is this module's - oldest law and it bites hardest here: `sum` over no linked records is not zero, it is a - question with no rows to answer it, and a 0 in an "Avg views" column reads as a measurement - that the creator gets no views. - ⚠ The ONE exception is the count family, where zero IS the answer — "how many linked records" - over an empty set is genuinely 0, not unknown. - - `ftype` is the SOURCE column's DECLARED type on the linked table (2026-08-10, D-92). Only - `min`/`max` read it today; it defaults to `text` so every existing caller and every test that - folds a bare list keeps its exact previous answer. - """ - if fn == "countall": - return str(len(values)) - if fn == "latest": - # Ordering belongs to the rollup bag (`sortBy` is mandatory for this function). Preserve - # a blank on the newest row rather than reaching backwards and presenting an older value - # as current. - return "" if not values or values[0] is None else str(values[0])[:ROLLUP_MAX_CHARS] - if fn == "counta": - return str(len([v for v in values if str(v or "").strip() != ""])) - if fn == "count": - # Airtable's COUNT counts NUMERIC values; COUNTA counts non-empty ones. Keeping them - # distinct is the whole reason both exist. - return str(len([v for v in values if _lane_num(v) is not None])) - # ⭐⭐ 2026-08-10 — D-92 CLOSED: `min`/`max` OVER A DATE COLUMN. - # - # Both were numeric-only, so a rollup over `posted_at`, `due_date` or `order_date` rendered - # BLANK forever while looking completely configured — the failure this module refuses - # everywhere else, arriving through the one fold that had no type awareness. "Earliest order" - # and "latest invoice due" are the two most ordinary date rollups there are, and - # `odoo_relational` already ships `latest` over `due_date`, so the vocabulary claimed dates - # and the fold did not. - # - # ⛔ NOT FIXED BY SNIFFING THE VALUES. An ISO date sorts correctly as a string, so a - # string-compare fallback would have worked on well-formed data and silently mis-ordered a - # `Aug 5, 2026` or a `2026-9-1` — [[measure-the-real-call]]'s shape. The DECLARED type is - # already at this call site (`linked_types`), and `_sort_key` is already the one function that - # turns a typed cell into a comparable key, blanks partitioned out. This reuses both rather - # than growing a second idea of what a date is. - # ⚠ RETURNS THE CELL, NOT THE KEY. `_sort_key` yields a comparison tuple; the answer a person - # wants in the column is the stored date string exactly as the source row spells it. - # ⚠ `min`/`max` ONLY. `sum`/`average`/`stdev` over dates are not blank by oversight — the mean - # of two timestamps is a number this product has no column type for, and inventing one here - # would be a value with no author. - if fn in ("min", "max") and ftype == "date": - keyed = [(k, str(v)) for k, v in - ((_sort_key(v, "date"), v) for v in values) if k is not None] - if not keyed: - return "" - return (min(keyed) if fn == "min" else max(keyed))[1][:ROLLUP_MAX_CHARS] - if fn in _ROLLUP_NUM_FNS: - nums = [n for n in (_lane_num(v) for v in values) if n is not None] - if not nums: - return "" - if fn == "stdev": - # ⭐⭐ SAMPLE standard deviation (n-1), and the divisor is a ruling, not a preference - # (C1 / `core.user_tables.ROLLUP_FNS`'s note): a rollup folds the rows that happen to - # be LINKED, which is a sample of an account's posting history and not its entirety. - # ⚠ Fixture to check a refactor against: [2,4,4,4,5,5,7,9] -> 2.14. The POPULATION - # form gives 2.00 on the same input, so a test that ever reads 2.00 has silently - # switched divisors. - # ⛔ FEWER THAN TWO VALUES IS "" AND NEVER "0". n-1 = 0 would divide by zero, but the - # honest reason is upstream of the arithmetic: one measurement has no spread to - # report, and a 0 in a "Std deviation" column reads as PERFECT CONSISTENCY — the - # single most confident thing this column can say, asserted from a single row. Same - # law as the blank `sum`, and it bites harder here. - out = _sample_stdev(nums) - if out is None: - return "" - else: - out = (sum(nums) if fn == "sum" else min(nums) if fn == "min" - else max(nums) if fn == "max" else sum(nums) / len(nums)) - return f"{out:.0f}" if float(out).is_integer() else f"{out:.2f}" - if fn in _ROLLUP_BOOL_FNS: - # A checkbox cell is '1'/'' in this product, so truth is "non-blank and not a zero". - flags = [str(v or "").strip() not in ("", "0", "false", "False") for v in values] - if not flags: - return "" - hit = (all(flags) if fn == "and" else any(flags) if fn == "or" - else sum(1 for f in flags if f) % 2 == 1) - return "1" if hit else "" - # --- the text family. ⛔ CLAIMED BY NAME, NEVER BY FALL-THROUGH — see `_ROLLUP_TEXT_FNS`. - # An fn no arm above recognises returns "" rather than a comma-joined dump of every value, - # which is the shape a not-yet-implemented aggregate wore for one commit of wave 28. - if fn not in _ROLLUP_TEXT_FNS: - return "" - vals = [str(v).strip() for v in values if str(v or "").strip() != ""] - if fn == "arrayunique": - seen, uniq = set(), [] - for v in vals: - if v.lower() not in seen: - seen.add(v.lower()) - uniq.append(v) - vals = uniq - if not vals: - return "" - text = ("".join(vals) if fn == "concatenate" else _ROLLUP_JOIN.join(vals)) - return text[:ROLLUP_MAX_CHARS] - - -def _link_from_key(fields, bag): - """Which column on THIS table supplies the join value. - - Declared `from` wins; otherwise the PROFILE-flagged column, then the PINNED one. ⭐ That - fallback chain is what makes an Instagram database link up with no configuration at all — - the "automatically" in the owner's instruction — and it is the same chain the grid already - uses to decide a table's identity column, rather than a second opinion about it. - """ - declared = str((bag or {}).get("from") or "").strip() - if declared: - return declared - prof = next((f for f in fields if isinstance(f.get("profile"), dict)), None) - if prof: - return str(prof.get("key") or "") - pin = next((f for f in fields if f.get("pinned") is True), None) - return str(pin.get("key") or "") if pin else "" - - -def _linked_rows_by_join(linked, on_key): - """`{join value (lower-cased) -> [(row_id, row)]}` over one linked table, built ONCE. - - ⚠ Lower-cased because the join values this exists for are Instagram handles, which the - profile flag already normalises to lower case on one side and which a hand-typed cell on the - other side may not. A join that misses on case is a relation that silently reports zero. - """ - idx = {} - for rid, row in ((linked or {}).get("rows") or {}).items(): - k = str((row or {}).get(on_key) or "").strip().lower() - if k: - idx.setdefault(k, []).append((str(rid), row or {})) - return idx - - -def _rollup_condition_matches(row, condition, field_types, ref_value=None): - """Evaluate one Airtable-style linked-record condition against a candidate row. - - `ref_value` is the threshold a `ref: {sigmas}` leaf compares against, already computed by the - caller over the SCOPED set (`_rollup_ref_threshold`). ⛔ It is passed IN rather than computed - here because this function sees one row and the statistic is a property of the whole window — - a version that reached for the set from inside would be recomputing the same mean once per - row, and would have to be handed the window anyway. - ⚠ `None` means the window could not answer, and the leaf then matches NOTHING. See the - threshold helper for why the permissive reading is the dangerous one. - """ - field = str((condition or {}).get("field") or "") - op = str((condition or {}).get("op") or "") - raw = (row or {}).get(field) - text = str(raw or "").strip() - if op == "is_empty": - return text == "" - if op == "is_not_empty": - return text != "" - if (condition or {}).get("ref") is not None: - # ⛔ NUMERIC LANE ONLY, BOTH SIDES. The validator already restricts `ref` to the ordering - # ops, and a row whose cell is blank or unparseable has no position relative to a computed - # threshold — it is not "below" it. Dropping it is the same partition law `_sort_key` - # follows: unrankable is not a rank ([[sentinel-in-a-sort-key]]). - left_num = _lane_num(text) - if ref_value is None or left_num is None: - return False - return ((op == "gt" and left_num > ref_value) - or (op == "gte" and left_num >= ref_value) - or (op == "lt" and left_num < ref_value) - or (op == "lte" and left_num <= ref_value)) - wanted = str((condition or {}).get("value") or "").strip() - if op == "contains": - return wanted.casefold() in text.casefold() - if op == "not_contains": - return wanted.casefold() not in text.casefold() - if op in ("eq", "neq"): - left_num, right_num = _lane_num(text), _lane_num(wanted) - equal = (left_num == right_num if left_num is not None and right_num is not None - else text.casefold() == wanted.casefold()) - return equal if op == "eq" else not equal - ftype = (field_types or {}).get(field, "text") - left = _sort_key(text, ftype) - right = _sort_key(wanted, ftype) - if left is None or right is None: - return False - return ((op == "gt" and left > right) or (op == "gte" and left >= right) - or (op == "lt" and left < right) or (op == "lte" and left <= right)) - - -def compute_relation_cells(rt, table_key, tables=None): - """Recompute every DERIVED LINK cell and every ROLLUP cell on ONE table. Returns rows touched. - - Zero store reads when the table declares neither kind — the same cheap-by-construction shape - `compute_metric_cells` has, so walking every table on a tick costs a dict scan per table. - """ - store = tables if tables is not None else ut_all(rt) - t = (store or {}).get(table_key) - fields = list((t or {}).get("fields") or []) - links = [f for f in fields if _ut().is_derived_link(f)] - rollups = [f for f in fields if isinstance(f.get("rollup"), dict)] - if not links and not rollups: - return 0 - rows = (t or {}).get("rows") or {} - by_key = {str(f.get("key")): f for f in fields} - - # --- resolve every link field ONCE per table, not once per row. - # `resolved[link_key][row_id] = [(linked_row_id, linked_row), ...]` - resolved, linked_types = {}, {} - for f in links + [by_key.get(str((r.get("rollup") or {}).get("link"))) for r in rollups]: - lk_key = str((f or {}).get("key") or "") - if not lk_key or lk_key in resolved or not isinstance((f or {}).get("link"), dict): - continue - bag = f["link"] - linked = (store or {}).get(str(bag.get("table") or "")) or {} - linked_types[lk_key] = {str(lf.get("key")): str(lf.get("type") or "text") - for lf in (linked.get("fields") or [])} - if bag.get("inverse"): - # Airtable's reciprocal side: this row is linked to every SOURCE row whose ordinary - # link cell contains this row id. The source cell remains the one relationship truth. - source_rows = linked.get("rows") or {} - inverse_key = str(bag.get("inverse") or "") - inverse_index = {} - for source_id, source_row in source_rows.items(): - for target_id in [part.strip() for part in - str((source_row or {}).get(inverse_key) or "").split(",")]: - if target_id: - inverse_index.setdefault(target_id, []).append( - (str(source_id), source_row or {})) - resolved[lk_key] = {str(rid): inverse_index.get(str(rid), []) for rid in rows} - elif bag.get("on"): - idx = _linked_rows_by_join(linked, str(bag["on"])) - from_key = _link_from_key(fields, bag) - resolved[lk_key] = { - str(rid): idx.get(str((row or {}).get(from_key) or "").strip().lower(), []) - for rid, row in rows.items()} if from_key else {} - else: - # An ORDINARY link: the cell IS the relation. Ids the linked table no longer holds - # are dropped rather than carried — a link to a deleted row is not a link. - lrows = (linked.get("rows") or {}) - resolved[lk_key] = { - str(rid): [(i, lrows[i]) for i in - [s.strip() for s in str((row or {}).get(lk_key) or "").split(",")] - if i and i in lrows] - for rid, row in rows.items()} - - changes = {} - for rid, row in rows.items(): - rid = str(rid) - row = row or {} - for f in links: - fk = str(f["key"]) - hits = (resolved.get(fk) or {}).get(rid) or [] - if f["link"].get("single"): - hits = hits[:1] - # ⛔ THE CAP IS A DISPLAY CAP AND THE ROLLUPS DO NOT READ THROUGH IT. A derived cell - # is a projection of `resolved`, which is uncapped and is what every rollup below - # consumes — so a profile with 900 posts shows the first 500 ids and still averages - # over all 900. Same argument the `posts` window already makes: shedding is safe - # precisely because the authoritative store still holds everything. - want = ",".join(i for i, _r in hits[:_ut().LINK_MAX_IDS]) - if str(row.get(fk, "")) != want: - changes.setdefault(rid, {})[fk] = want - for f in rollups: - fk, bag = str(f["key"]), f["rollup"] - lk_key = str(bag.get("link") or "") - hits = list((resolved.get(lk_key) or {}).get(rid) or []) - # ⚠ A rollup whose link field does not exist (renamed, deleted) resolves to NOTHING - # and therefore to a blank cell — never to a stale number. A column that keeps - # printing yesterday's answer after its input is gone is the worst of the options. - # ⭐⭐ WAVE 28 / CONTRACT C1 — SCOPE FIRST, FILTER SECOND, AND THE TWO USED TO BE THE - # OTHER WAY ROUND. The conditions block stood HERE, above the ranking, so - # "last 10 posts where views > X" meant *the 10 most recent of the posts over X* - # rather than *the ones over X among the last 10* — two different windows wearing one - # sentence. Harmless while a threshold was a literal; incoherent the moment a - # threshold is a statistic OF the window, because the set being described and the set - # doing the describing would be different sets. - # ⛔ THIS ORDER IS THE CONTRACT, not an implementation choice: `core.user_tables`'s - # `ROLLUP_REF_OPS` note states it ("the scope picks the window, THEN the threshold is - # computed over that window, THEN the conditions filter it") and the validator half - # was written against it. - # ⚠ IT IS A BEHAVIOUR CHANGE FOR EXACTLY ONE SHAPE: a stored rollup carrying BOTH - # `conditions` AND `limit`. No shipped preset does (measured across `odoo_relational` - # and the IG presets — the one preset with conditions, `_OPEN_ONLY`, is a `countall` - # with no limit), so the blast radius is user-built rollups only. - # ⭐⭐ 2026-08-09 (owner) — THE PRE-FILTER, ABOVE THE RANKING. Owner: *"instead of last - # 12 posts, we also want to make it so its last N record, where the record's Status is - # video."* `conditions` cannot answer that: C1 moved them BELOW the window on purpose, - # so they select among the rows the window already kept. `where` selects WHICH rows - # the window is spent on. - # ⛔ ABOVE `distinctBy` TOO, not merely above the sort. Dedup keeps the first row per - # identity; run it first and a carousel could claim the slot its reel sibling needed, - # so the window would come up short for a reason nothing on screen explains. - # ⚠ NO `ref` REACHES HERE — `_clean_rollup` refuses a set-statistic threshold in this - # list, because at this point there is no fixed set for a statistic to be about. - where = list(bag.get("where") or []) - if where: - where_matches = lambda pair: [ - _rollup_condition_matches(pair[1], condition, - linked_types.get(lk_key) or {}, ref_value=None) - for condition in where] - if str(bag.get("whereConj") or "and") == "or": - hits = [pair for pair in hits if any(where_matches(pair))] - else: - hits = [pair for pair in hits if all(where_matches(pair))] - sort_by = str(bag.get("sortBy") or "") - if sort_by: - ftype = (linked_types.get(lk_key) or {}).get(sort_by, "text") - # ⛔ PARTITION, THEN SORT. A row whose sort cell is blank or unparseable is not - # rankable, and it must land at the END whichever direction is asked for — which - # a sentinel inside the sort key cannot do, because `reverse` flips the sentinel - # too (see `_sort_key`). Unrankable rows are appended, so a `limit` spends its - # window on rows that HAVE the value before it falls back to ones that do not. - keyed = [(pair, _sort_key(pair[1].get(sort_by), ftype)) for pair in hits] - rankable = [(p, k) for p, k in keyed if k is not None] - rankable.sort(key=lambda pk: pk[1], - reverse=str(bag.get("sortDir") or "desc") == "desc") - hits = [p for p, _k in rankable] + [p for p, k in keyed if k is None] - distinct_by = str(bag.get("distinctBy") or "") - if distinct_by: - seen, unique = set(), [] - for pair in hits: - identity = str(pair[1].get(distinct_by) or "").strip().lower() - # A blank is not an identity. Keep it rather than collapsing every unknown - # record into one synthetic duplicate. - if identity and identity in seen: - continue - if identity: - seen.add(identity) - unique.append(pair) - hits = unique - limit = int(bag.get("limit") or 0) - if limit: - hits = hits[:limit] - # --- the window is now FIXED, so a set-statistic threshold has a set to be about. - conditions = list(bag.get("conditions") or []) - if conditions: - # ⚠ RESOLVED ONCE PER LEAF, NOT ONCE PER ROW. The threshold is a property of the - # window; computing it inside `matches` would recompute the same mean for every - # candidate and — worse — would invite computing it over a set that the filter is - # already shrinking underneath it. - # ⚠ `.get("sigmas", 0.0)`, never `... or 0.0` — a legitimate `sigmas: 0` ("beyond - # the mean") is falsy, and the `or` spelling would silently rewrite it to the same - # number by accident. It reads identically and is right for the wrong reason, - # which is how it survives a review. - refs = [_rollup_ref_threshold( - [r for _i, r in hits], str(c.get("field") or ""), - (c.get("ref") or {}).get("sigmas", 0.0)) - if isinstance(c, dict) and c.get("ref") is not None else None - for c in conditions] - matches = lambda pair: [ - _rollup_condition_matches(pair[1], condition, - linked_types.get(lk_key) or {}, ref_value=ref) - for condition, ref in zip(conditions, refs)] - if str(bag.get("conditionConj") or "and") == "or": - hits = [pair for pair in hits if any(matches(pair))] - else: - hits = [pair for pair in hits if all(matches(pair))] - src = str(bag.get("field") or "") - # D-92 — the SOURCE column's declared type, read from the same `linked_types` map the - # sort path above uses. `countall` folds a synthetic `[1]*n` with no source column at - # all, so the default stands for it. - want = _rollup_fold(str(bag.get("fn") or ""), - [r.get(src) for _i, r in hits] if src else [1] * len(hits), - (linked_types.get(lk_key) or {}).get(src, "text")) - if str(row.get(fk, "")) != want: - changes.setdefault(rid, {})[fk] = want - if not changes: - return 0 - - # A run that just wrote linked rows passes its in-flight user_tables bucket here so the - # relation refresh joins the SAME coalesced commit. The standalone/tick path below keeps - # the public helper's old persist-on-change behaviour. - if tables is not None: - tt = tables.get(table_key) - if tt is not None: - for r, vals in changes.items(): - tt.setdefault("rows", {}).setdefault(r, {}).update(vals) - return len(changes) - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - tt = cur.get(table_key) - if tt is not None: - for r, vals in changes.items(): - tt.setdefault("rows", {}).setdefault(r, {}).update(vals) - return cur - - rt.update(UT_STORE_KEY, _up, flush="sync") - return len(changes) - - -def _refresh_relations_inplace(tables, log=print): - """Refresh every relation against one mutable user_tables bucket; perform no store write.""" - touched = 0 - for tk, table in list((tables or {}).items()): - fields = (table or {}).get("fields") or [] - if not any(f.get("type") == "rollup" or _ut().is_derived_link(f) for f in fields): - continue - try: - touched += compute_relation_cells(None, tk, tables=tables) - except Exception as e: # noqa: BLE001 - log(f"[aios-auto] relation refresh {tk} failed: {type(e).__name__}: {e}") - return touched - - + + +def _refresh_relations_inplace(tables, log=print, rt=None): + """Refresh every relation against one mutable user_tables bucket; perform no store write. + + ⭐ W41-T18 — `rt` RIDES THROUGH, and it used to be dropped on the floor. This path passed + `None`, which is fine for a `ut_*` link (the bucket is already in hand) and is the difference + between resolving and refusing for a link whose TARGET is `customer_data` or `product_data`: + a registry topic's pool is per tenant, so without the handle it cannot be built at all. + ⚠ Nothing else changes — `compute_relation_cells` reads `rt` only when `tables is None` + (`ut_all`) or when it persists, and both of those branches are unreachable from here. + ⛔ `rt` RIDES POSITIONALLY, in the slot that already existed. The `except Exception` below + swallows a `TypeError` as readily as a store outage, so ANY change to the shape of the call + on the next line disables the entire relational pass with nothing red — measured, on a + keyword this function briefly added (see `compute_relation_cells`). + """ + touched = 0 + for tk, table in list((tables or {}).items()): + fields = (table or {}).get("fields") or [] + if not any(f.get("type") == "rollup" or _ut().is_derived_link(f) for f in fields): + continue + try: + touched += compute_relation_cells(rt, tk, tables=tables) + except Exception as e: # noqa: BLE001 + log(f"[aios-auto] relation refresh {tk} failed: {type(e).__name__}: {e}") + return touched + + def refresh_relations(rt, log=print, tables=None, persist=True): - """The tick half of the relational pass — the twin of `refresh_metrics`. - - ⚠ Runs for EVERY table, because a link can point anywhere: a rollup on table A goes stale - when table B gains a row, and A has no way to know that happened. Cheap by construction — a - table declaring neither kind costs one dict scan. - """ + """The tick half of the relational pass — the twin of `refresh_metrics`. + + ⚠ Runs for EVERY table, because a link can point anywhere: a rollup on table A goes stale + when table B gains a row, and A has no way to know that happened. Cheap by construction — a + table declaring neither kind costs one dict scan. + """ if tables is not None: - local = _refresh_relations_inplace(tables, log=log) + local = _refresh_relations_inplace(tables, log=log, rt=rt) if not local or not persist: return local actual = [0] def _up(cur): cur = cur if isinstance(cur, dict) else {} - actual[0] = _refresh_relations_inplace(cur, log=log) + actual[0] = _refresh_relations_inplace(cur, log=log, rt=rt) return cur rt.update(UT_STORE_KEY, _up, flush="async") return actual[0] snapshot = { - str(key): {**(table or {}), - "rows": {str(rid): dict(row or {}) - for rid, row in ((table or {}).get("rows") or {}).items()}} - for key, table in (ut_all(rt) or {}).items() - } - if not _refresh_relations_inplace(snapshot, log=log): - return 0 - actual = [0] - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - actual[0] = _refresh_relations_inplace(cur, log=log) - return cur - - # ⭐ ASYNC, and for TWO independent reasons (2026-08-09, the lost-record bug). - # - # 1. COST, which was wave 27 item 2's whole point: this pass runs after every row write, and - # `flush="sync"` made each one a blocking HF commit against the 256-commits/hr budget. - # A derived-cell recompute has no business forcing a commit on someone typing. - # 2. It was the deterministic TRIGGER of the bug: `add_row` writes `flush='async'`, so a new - # row lives only in the store cache for 2-20s, and this call — on the SAME key - # (`UT_STORE_KEY == user_tables`) — used to `_read_strict` that cache away and upload the - # result. `POST /rows` answered 201 and the row was gone. - # - # ⚠ THE ROOT FIX IS IN `core/store.py` (a sync RMW no longer discards a dirty cache) and it - # is what makes the other ~39 sync writers of this key safe. This line is not that fix and - # must not be mistaken for it — it removes the trigger and the cost, nothing more. Both - # landed together on purpose: one is correctness, one is the hot path. - rt.update(UT_STORE_KEY, _up, flush="async") - return actual[0] - - + str(key): {**(table or {}), + "rows": {str(rid): dict(row or {}) + for rid, row in ((table or {}).get("rows") or {}).items()}} + for key, table in (ut_all(rt) or {}).items() + } + if not _refresh_relations_inplace(snapshot, log=log, rt=rt): + return 0 + actual = [0] + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + actual[0] = _refresh_relations_inplace(cur, log=log, rt=rt) + return cur + + # ⭐ ASYNC, and for TWO independent reasons (2026-08-09, the lost-record bug). + # + # 1. COST, which was wave 27 item 2's whole point: this pass runs after every row write, and + # `flush="sync"` made each one a blocking HF commit against the 256-commits/hr budget. + # A derived-cell recompute has no business forcing a commit on someone typing. + # 2. It was the deterministic TRIGGER of the bug: `add_row` writes `flush='async'`, so a new + # row lives only in the store cache for 2-20s, and this call — on the SAME key + # (`UT_STORE_KEY == user_tables`) — used to `_read_strict` that cache away and upload the + # result. `POST /rows` answered 201 and the row was gone. + # + # ⚠ THE ROOT FIX IS IN `core/store.py` (a sync RMW no longer discards a dirty cache) and it + # is what makes the other ~39 sync writers of this key safe. This line is not that fix and + # must not be mistaken for it — it removes the trigger and the cost, nothing more. Both + # landed together on purpose: one is correctness, one is the hot path. + rt.update(UT_STORE_KEY, _up, flush="async") + return actual[0] + + def refresh_metrics(rt, today=None, log=print, tables=None, persist=True): - """The tick half of the C7 amendment: `today` advances at tick cadence, so a date-window - metric can never go staler than one tick while a scheduler exists. Cheap by construction — - a table without metric fields costs a dict scan and nothing else.""" + """The tick half of the C7 amendment: `today` advances at tick cadence, so a date-window + metric can never go staler than one tick while a scheduler exists. Cheap by construction — + a table without metric fields costs a dict scan and nothing else.""" snapshot = tables if tables is not None else ut_all(rt) touched = 0 for tk, t in snapshot.items(): @@ -13319,1180 +13691,1180 @@ def refresh_metrics(rt, today=None, log=print, tables=None, persist=True): try: touched += compute_metric_cells(rt, tk, today=today, tables=snapshot, persist=persist) - except Exception as e: # noqa: BLE001 - log(f"[aios-auto] metric refresh {tk} failed: {type(e).__name__}: {e}") - return touched - - -def purge_subject(rt, handle): - """D-24: right-to-erasure for ONE Instagram subject — every row about them leaves the - tenant's four `ut_ig_*` tables AND the platform master (R2 made the master half - non-optional: a purge that missed the pooled copy would not be erasure). Returns - `{table: removed}` counts, master rows prefixed `master:` — every count drills to what is - now ABSENT, which is the one aggregate whose drill is emptiness.""" - subject = str(handle or "").strip().lstrip("@").lower() - if not subject: - return {} - counts = {} - tables = ut_all(rt) - post_rows = (tables.get("ut_ig_posts") or {}).get("rows") or {} - codes = {str(r.get("shortcode") or "") for r in post_rows.values() - if str((r or {}).get("influencer_key") or "").strip().lower() == subject} - - keeps = { - "ut_ig_snapshots": lambda r: str((r or {}).get("influencer_key") - or "").strip().lower() != subject, - "ut_ig_posts": lambda r: str((r or {}).get("influencer_key") - or "").strip().lower() != subject, - "ut_ig_post_snapshots": lambda r: str((r or {}).get("shortcode") or "") not in codes, - DISCOVER_TABLE: lambda r: str((r or {}).get("handle") - or "").strip().lower() != subject, - } - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - for tk, keep in keeps.items(): - t = cur.get(tk) - if t is None: - continue - rows = t.get("rows") or {} - nxt = {rid: r for rid, r in rows.items() if keep(r)} - counts[tk] = len(rows) - len(nxt) - t["rows"] = nxt - return cur - - rt.update(UT_STORE_KEY, _up, flush="sync") - import ig_master - for bucket, n in (ig_master.purge_handle(subject) or {}).items(): - counts[f"master:{bucket}"] = n - return counts - - -# --------------------------------------------------------------------------------------------- -# TRIGGERS (wave 22, contract C3 + amendment A2 — owner ruling R4; closes D-33) -# --------------------------------------------------------------------------------------------- -# Six ways an automation starts, exactly: manual | schedule | event_field | record_created | -# webhook | email. The first two are what always existed (Run now; cron via the tick). The four -# new ones are EVENTS, and A2 makes their discipline LAW rather than taste: -# -# * **EDGE, NEVER LEVEL (A2(1)).** A condition trigger fires on entering the matching state, -# not for being in it. Implemented as per-record ARMED state over successive evaluations -# (`state.eventDisarmed`): a record fires when it matches while armed, DISARMS, and re-arms -# only by evaluating False — Airtable's documented leave-and-re-enter rule, without needing -# a before-image of a row whose truth is spread over strata. Enabling a trigger SEEDS the -# disarmed set with everything currently matching, so already-matching records do not fire -# (`_seed_event_state`). A settle window coalesces write bursts (the per-keystroke scar). -# * **LOOP PREVENTION IS STRUCTURAL (A2(2)).** The hooks live on the HUMAN doors only -# (`grid_events.overlay_patch`, `user_tables.add_row`); the engine's own writers -# (`ut_write_rows`, the runners' coalesced updates, `patch_cells` from `move_card`) never -# emit — so an automation's write cannot fire event triggers, its own or a sibling's, by -# construction. The circuit breaker on top (>60 fires/5 min auto-pauses with the reason as -# a statusNote) catches whatever construction did not foresee. -# * **FLOOD HOLD (A2(3)).** One evaluation yielding more than 100 candidate records holds -# instead of running — a partial run entry names the count and the deliberate way through -# (Run now). C4's discovery guard is this rule's special case. -# * **REFIRE DEFAULTS (A2(4)), hard-coded this wave:** record_created fires once per record -# EVER (a high-water mark over row ids, so an undo-restored row cannot re-fire); -# event_field fires every transition. - -# ── WAVE 23 · C3 — the trigger vocabulary v2 (owner ruling R2). ────────────────────────���────── -# Airtable's phrasing, because the owner asked for Airtable's builder and a trigger list that -# renames the same events is a second vocabulary to learn for no gain. -# -# ⚠ `event_field` KEPT ITS KEY and changed its LABEL to "When a record matches conditions". -# Renaming the key would have orphaned every stored trigger in production for a caption; the key -# is the contract with the store, the label is the contract with the reader, and they are allowed -# to disagree. What genuinely widened is its SHAPE: the watched field is now OPTIONAL, so the -# trigger covers Airtable's condition-only form (any write to the table, evaluated against a C4 -# tree) as well as wave 22's watch-one-field form. Both are the same edge rule underneath. -# -# ⛔ PLANNED ≠ STORABLE. `button_clicked` / `comment_added` ride the wire so the picker can show -# them faded with a reason (R2: "never a dead control") — and `clean_trigger` REFUSES them with a -# sentence. A vocabulary that renders an option the validator rejects is the wave-9 silent-drop -# class wearing a friendlier face; here the two lists are separate on purpose and the refusal -# names the state rather than pretending the key is unknown. -# ── WAVE 24 · C-TRIG (owner ruling R6). ─────────────────────────────────────────────────────── -# ⭐ INSTAGRAM DISCOVERY BECOMES A TRIGGER. It was a KIND you picked in a create wizard; the -# wizard is deleted, and "when an Instagram profile fits a criteria" is the honest shape anyway — -# it is the event this flow starts from. Picking it sets the definition's kind to -# `discover_instagram` (law 1), which is the ONLY way that kind is reachable now. -# -# ⛔ IT IS NOT A TABLE TRIGGER AND NOT A ROW TRIGGER. It watches nothing: it MAKES rows, on the -# schedule (or on Run now), so it stays out of both lists below and its node switch flips the -# CRON — see `TRIGGER_SCHEDULE_KEYS`. -# -# ⭐⭐ WAVE 29 (item 7 · D-9 · R1) — `tiktok_profile_match` MOVED HERE FROM `TRIGGER_PLANNED`, and -# that move is the whole of "TikTok is a real trigger now". The label was written a wave early in -# the owner's own words and has not changed; what changed is which tuple it sits in, because -# `clean_trigger` refuses the planned list with a sentence and accepts this one. -TRIGGER_KEYS = ("manual", "schedule", "event_field", "record_updated", "record_created", - "enters_view", "webhook", "email", "form_submitted", "ig_profile_match", - "tiktok_profile_match") -#: ⭐ WAVE 25 · C2 / owner ruling R9 — `web_page_changed` JOINS THE PLANNED LIST, and joining THIS -#: tuple rather than `TRIGGER_KEYS` is the whole of its implementation. `clean_trigger` refuses -#: everything here with a sentence, so the faded row is a wall; the picker shows it so the Scraper -#: section is not a section of one. -#: ⚠ `web_page_changed` IS NOT THE WEB ACTION (D-51). A trigger that notices a page changed and an -#: action that drives a browser are different builds; this row must not be read as progress on D-51. -TRIGGER_PLANNED = ("button_clicked", "comment_added", "web_page_changed") -TRIGGER_LABELS = { - "manual": "Manual", - "schedule": "At a scheduled time", - "event_field": "When a record matches conditions", - "record_updated": "When a record is updated", - "record_created": "When a record is created", - "enters_view": "When a record enters a view", - "webhook": "When a webhook is received", - "email": "When an email arrives", - "form_submitted": "When a form is submitted", - "ig_profile_match": "When an Instagram profile fits a criteria", - "button_clicked": "When a button is clicked", - "comment_added": "When a comment is added", - "web_page_changed": "When a website page changes", - "tiktok_profile_match": "When a TikTok profile fits a criteria", -} - -# ── WAVE 25 · C2 — THE PICKER TAXONOMY, and it lives HERE beside the vocabulary it describes. ── -# `group` already rode the wire as "Standard"/"Sources" (`routes_automation`), which is a -# distinction about where a trigger came FROM rather than about what a person is choosing. The -# question the picker actually asks is: does this fire on TIME, on your own DATA, or because -# something OUTSIDE said so. Three answers, and every trigger has exactly one. -# -# ⛔ TWO CONTROLS, NOT ONE, AND THE FIRST DRAFT HAD ONLY THE WRONG HALF. Indexing this map -# directly (`TRIGGER_GROUP_OF[k]`) makes an unclassified trigger a KeyError — which is exactly the -# incident `_triggers_vocab`'s `per.get(k, ...)` comment records: a key added to `TRIGGER_KEYS` -# without remembering a dict beside it 500'd `GET /automations`, the payload the whole automation -# surface polls every 2.5 s, with every gate green. A mis-grouped row is a cosmetic bug; a 500 is -# the surface. So: -# * RUNTIME fails SOFT — an unclassified trigger falls into `other`, which sorts LAST (the rule -# `ACTION_GROUP_ORDER` already uses: an unordered group sorts last, never first, because -# appearing at the top looks deliberate) and is honestly captioned rather than smuggled into -# Database. -# * THE GATE fails HARD — `verify_automation` asserts that NO shipped trigger lands in `other`, -# so the fallback is provably dead code in production and the classification is still -# mandatory. The fallback catches the accident; the gate stops it shipping. -# ⭐ WAVE 34 · R19 — CONNECTOR SITS ABOVE DATABASE, DIRECTLY UNDER TIME. Owner, verbatim: -# *"In the Trigger picker, the Connector section sits directly above the Database section, just -# under the Time trigger types."* So the two orders below are SWAPPED against wave 24's, and -# nothing else moved: same keys, same labels, same fallback. -# ⛔ THIS IS THE WHOLE OF R19 AND IT IS DELIBERATELY NOT A CLIENT CHANGE. `steps.ts::groupTriggers` -# sorts by each option's `groupOrder` and by nothing else, so re-ordering an array on the client -# would look right in a fixture and be wrong in production the moment the server re-sorted. -# ⚠ `verify_steps.py` CANNOT WITNESS THIS EDIT: its C2 leg supplies its OWN `groupOrder` in a -# fixture and asserts the client honours it, which stays true whatever these numbers say. The -# check that binds R19 to this table lives in `verify_automation` beside the vocab section. -TRIGGER_GROUPS = {"time": {"label": "Time", "order": 1}, - "connector": {"label": "Connector", "order": 2}, - "database": {"label": "Database", "order": 3}, - "other": {"label": "Other", "order": 99}} -#: Where an unclassified trigger goes. ⚠ Reaching this in production is a BUG the gate exists to -#: prevent — it is the soft landing, not a category anybody should be adding triggers to. -TRIGGER_GROUP_FALLBACK = "other" -TRIGGER_GROUP_OF = { - "manual": "time", "schedule": "time", - "event_field": "database", "record_updated": "database", "record_created": "database", - "enters_view": "database", "form_submitted": "database", - "button_clicked": "database", "comment_added": "database", - "email": "connector", "webhook": "connector", "ig_profile_match": "connector", - "web_page_changed": "connector", "tiktok_profile_match": "connector", -} -#: The SUB-group inside "Connector" — which connected thing this trigger comes through. -#: ⚠ THESE KEYS ARE GROUPING HANDLES FOR THE PICKER, NOT connector-directory slugs, and the two -#: genuinely differ: the directory's OAuth row for Gmail is `google` (the provider), while a -#: person choosing a trigger is picking *Gmail* (the product). A client that joined this key -#: against `/connectors/directory` would match `scraper` and `webhooks` and miss `gmail` — so it -#: must group by it and render `label`, never look it up. Said here because the miss would be -#: silent and partial, which is the worst shape. -#: (⚠ that example USED to read "`scraper` and `tiktok`" — R3 retired `tiktok` as a handle, and -#: the sentence is corrected here rather than left to rot into a lie about a key that is gone.) -TRIGGER_CONNECTOR = { - "email": {"key": "gmail", "label": "Gmail"}, - "webhook": {"key": "webhooks", "label": "Webhooks"}, - # ⭐ WAVE 30 · R3 — ONE "SCRAPER" BUCKET, AND IT HOLDS BOTH PLATFORMS. - # The owner, verbatim and for the third wave running: *"I say this multiple times already the - # damn Tiktok and Instagram belongs in the same bucket when creating the automation its under - # Scraper … Only when I click 'Scraper' under each automation trigger and actions would I see - # the option to choose either Instagram OR TikTok. That's it."* - # - # ⛔ THIS SUPERSEDES WAVE 29's RULE, and the old rule was not a typo — it was an argument: - # *"TikTok is its own connector, not the Scraper's … because the sub-group answers WHICH - # PRODUCT and never HOW BUILT."* Coherent, and not what was asked for. Instagram and TikTok are - # two PRODUCTS of one CAPABILITY (a social scraper bought from one vendor); a person opening - # this picker is choosing the capability first and the platform second. `verify_automation` - # asserted the old rule as an assertion AND as prose — a shipped gate forbidding the owner's - # ruling is most of why this complaint survived two waves — so it is INVERTED in this same - # change, comment included. - # - # ⚠ The Scraper sub-group now holds THREE rows: two built (Instagram, TikTok) and one faded - # (the page-change trigger). No display ORDER is emitted here — the client groups on `key` and - # owns its own ordering (contract C1). `tiktok` ceases to exist as a grouping handle. - "ig_profile_match": {"key": "scraper", "label": "Scraper"}, - "web_page_changed": {"key": "scraper", "label": "Scraper"}, - "tiktok_profile_match": {"key": "scraper", "label": "Scraper"}, -} -#: Triggers that watch a database and therefore need one named before they can fire. -TRIGGER_TABLE_KEYS = ("event_field", "record_updated", "record_created", "enters_view", - "form_submitted") -#: ⭐ WAVE 24 — triggers whose NODE SWITCH means the CRON rather than the trigger itself. -#: `manual`/`schedule` are not stored at all; `ig_profile_match` is stored and IS schedule-driven, -#: so flipping its node must flip the schedule. -#: -#: ⚠ THIS REPLACES A HAND-LISTED TUPLE IN `toggle_node` THAT WAS ALREADY WRONG. It read -#: `("event_field", "record_created", "webhook", "email")` — omitting `record_updated`, -#: `enters_view` and `form_submitted`, all three of which have been storable since wave 23. For -#: those, clicking the trigger node's switch flipped the CRON under a node labelled "When a -#: record is updated": a switch that lies, which is exactly what the tuple at `graph()` warns -#: about eight lines into its own comment. Derived from one named set now, so a trigger added to -#: `TRIGGER_KEYS` cannot silently join the wrong side of it. -#: ⚠ WAVE 29 — `tiktok_profile_match` BELONGS HERE FOR THE SAME REASON `ig_profile_match` DOES, -#: and forgetting it is precisely the failure this constant's own note describes: it watches no -#: table and MAKES rows on the schedule, so its node switch has nothing to flip but the cron. Left -#: out, a person clicking the TikTok trigger node's switch would toggle the trigger itself while -#: the schedule kept firing — a switch that lies. -TRIGGER_SCHEDULE_KEYS = ("manual", "schedule", "ig_profile_match", "tiktok_profile_match") -#: ⭐ WAVE 25 — DEBT D-55: "the cron drives this one", on the wire at last. -#: -#: ⛔ `TRIGGER_SCHEDULE_KEYS` MUST NOT SHIP VERBATIM, and the one-element difference is the entire -#: reason this constant exists rather than the tuple above being sent. That set answers "which -#: way does this trigger's NODE SWITCH flip" — and `manual` is in it only because a manual -#: automation's switch has nothing else to flip. Shipping it as "the cron drives this" would draw -#: a schedule face on the one trigger whose whole sentence is "It runs only when you press Run -#: now": a control contradicting its own description. -#: -#: D-55's history is why it is DERIVED rather than listed: the client carried -#: `CRON_DRIVEN_TRIGGERS = ["schedule", "ig_profile_match"]` — a hand-kept copy of a server fact -#: that fails VISIBLY but silently (a new cron-driven trigger simply shows no schedule face). -#: Subtracting from the engine's own set means a trigger added there cannot be forgotten here. -TRIGGER_CRON_KEYS = frozenset(TRIGGER_SCHEDULE_KEYS) - {"manual"} -#: Triggers the ROW HOOKS drive (as opposed to the tick, or an inbound HTTP call). Named once so -#: `grid_hook` and the gates read the same list instead of two matching `in (...)` tuples. -TRIGGER_ROW_KEYS = ("event_field", "record_updated", "record_created", "enters_view") -MAX_WATCH_FIELDS = 12 -#: The settle window for field-change bursts (A2(1)). 0 evaluates INLINE — the gates run there, -#: and so would a deployment that prefers immediacy over coalescing. -EVENT_SETTLE_SECONDS = float(os.environ.get("AIOS_EVENT_SETTLE_SECONDS") or 15) -FIRE_LIMIT = 60 # A2(2): fires per window before the breaker pauses -FIRE_WINDOW_SECONDS = 300 -FLOOD_LIMIT = 100 # A2(3): candidate records one evaluation may act on -EMAIL_SEEN_CAP = 500 # message-id dedupe memory per automation -EMAIL_MAX_PER_POLL = 25 # bounded by construction — a poll is a tick guest -CONSECUTIVE_FAILURE_PAUSE = 5 # airtable-brief rec 6: a dead credential must not burn quota - -EMAIL_FIELDS = [ - field_def("email_id", "Email id"), field_def("email_from", "From"), - field_def("email_subject", "Subject"), field_def("email_date", "Date"), - field_def("email_snippet", "Snippet"), field_def("email_seen_at", "Seen at"), -] - - -def clean_trigger(raw, previous=None): - """Validate a definition's `trigger`. Returns `(trigger|None, error)` — None is legal and - means what it always meant: manual + whatever `schedule` says. - - ⚠ A3 (2026-08-05): the stored/wire name is `key` (`kind` accepted on input for symmetry - with the definition's own vocabulary). And an INCOMPLETE event trigger is STORED INERT - rather than refused — the picker writes `{key}` first and the table/field after, the - wave-18 unconfigured-automation-column precedent exactly; `configured: false` rides the - wire so the surface says "finish setting this up" instead of snapping back to Manual. It - cannot fire while incomplete (the hooks match on the table it does not name), which is the - fail-closed direction. MALFORMED parts (an unknown comparison, a valueless compare, a - condition on a field the trigger does not watch) are still refused with the sentence — - incomplete is a state, wrong is not. - """ - if raw in (None, "", {}): - return (dict(previous) if isinstance(previous, dict) and previous else None), None - if not isinstance(raw, dict): - return None, "the trigger must be an object" - prev = previous if isinstance(previous, dict) else {} - key = _s(raw.get("key") or raw.get("kind") or prev.get("key") or prev.get("kind"), - 30).strip() - if key in TRIGGER_PLANNED: - # Declared on the wire, refused at the door — see the TRIGGER_PLANNED note. The sentence - # says WHY rather than "unknown trigger", because the picker legitimately showed it. - return None, (f"{TRIGGER_LABELS[key]!r} is on the list but not built yet. " - f"it renders so you can see it is coming, and it cannot be saved") - if key not in TRIGGER_KEYS: - return None, (f"{key or 'that trigger'!r} is not one of: " + ", ".join(TRIGGER_KEYS)) - if key == "schedule": - # ⛔ STILL NOT STORED, and the original reasoning holds for THIS key alone: `schedule` - # already owns the cron (`defn['schedule']` = `{cron, enabled}`), so a stored - # `{key:'schedule'}` would be a second copy of that fact, free to disagree with it. - return None, None - if key == "manual": - # ⭐⭐ 2026-08-07 (owner ruling) — **MANUAL IS A REAL, STORED CHOICE NOW.** - # Owner: *"Make it so that when you choose Manual, it IS a manual automation that the user - # can just press Run to make the full flow work."* - # - # ⛔ THIS SPLITS A PAIR THAT SHOULD NEVER HAVE BEEN ONE. The old line refused both keys - # together with one argument — *"storing a no-op trigger would be a second copy of that - # fact"* — and that argument is TRUE OF `schedule` AND FALSE OF `manual`. A schedule has - # another home; **manual has none.** Nothing anywhere recorded "this automation is - # manual", so storing it is not a duplicate: it is the only record there has ever been. - # - # ⚠ WHAT THE CONFLATION COST, measured live: picking Manual wrote nothing, so - # `chosen` (`!!trigger || schedule.enabled`) stayed false, the Builder kept showing the - # "nobody has decided yet" empty state, and Configuration — including the Database picker - # a plain automation cannot do without — never rendered. The owner reported it twice. The - # previous note reasoned that a manual option *"would bounce straight back to this state - # on the next reload"* and concluded the option should be HIDDEN; the honest conclusion - # was that it should be STORED. - # - # ⚠ DELIBERATELY BARE. No `enabled`, no `paused`: a manual trigger cannot be switched off - # (Run now always works, which is the whole of what it means) and a switch that governs - # nothing is worse than no switch. `graph()` keeps this node on the SCHEDULE panel so the - # cron stays reachable — picking Manual says how it fires today, never that it may not be - # scheduled tomorrow. - return {"key": "manual"}, None - out = {"key": key, - "enabled": bool(raw["enabled"]) if "enabled" in raw else - bool(prev.get("enabled", True)), - "paused": bool(raw["paused"]) if "paused" in raw else bool(prev.get("paused"))} - if key in TRIGGER_TABLE_KEYS: - table = _s(raw.get("table") if "table" in raw else prev.get("table"), 60).strip() - if table and not table.startswith(UT_PREFIX): - return None, ("event triggers watch blank databases (ut_*) this wave. " - f"{table!r} is not one") - out["table"] = table - if key == "event_field": - # ⭐ WAVE 24 · C-TRIG LAW 4 (owner item 7) — THE WATCHED FIELD IS GONE. "When a record - # matches conditions" is a CONDITION trigger and nothing else: the field picker made it a - # second, quieter way to express the same narrowing, and the owner asked for one. - # ⚠ MIGRATION, NEVER A REFUSAL (law 6). A stored `field` is simply not read, so it is - # dropped on this definition's next clean — silently, and exactly once, because nothing - # writes the key back. A refusal here would have 400'd the live automations that carry it. - cond, cerr = clean_cond(raw.get("when") if "when" in raw else prev.get("when"), - where="the trigger") - if cerr: - return None, cerr - out["when"] = cond - if key == "record_updated": - # Airtable's shape: watch named fields, or leave the list empty for "any field". Empty - # is the WIDER reading and it is the default there too, so it stays the default here. - watch_raw = raw.get("fields") if "fields" in raw else prev.get("fields") - if watch_raw in (None, ""): - watch = [] - elif not isinstance(watch_raw, list): - return None, "the watched-field list must be a list of field keys" - else: - watch = [_s(f, 80).strip() for f in watch_raw if _s(f, 80).strip()] - if len(watch) > MAX_WATCH_FIELDS: - return None, (f"a record-updated trigger watches at most {MAX_WATCH_FIELDS} " - f"fields. Leave the list empty to watch every field") - out["fields"] = watch - # ⭐ WAVE 24 · C-TRIG LAW 5 (owner item 7) — THE CONDITION IS GONE, and this REMOVES A - # SHIPPED CAPABILITY. Watched `fields` is now the whole of this trigger's configuration: - # "a record was updated" is an event, and asking it to also be a filter was the overlap - # with `event_field` the owner asked to end. Stated loudly in the contract AND here so - # nobody restores it as a bug fix. - # ⚠ Same migration shape as law 4: a stored `when` stops being read, so `_row_gate` - # naturally returns "no gate" for it — the write itself becomes the event — rather than - # this needing a second removal anywhere. - if key == "enters_view": - out["viewId"] = _s(raw.get("viewId") if "viewId" in raw else prev.get("viewId"), - 80).strip() - if key == "form_submitted": - # Blank = any form on that database. Naming one narrows to it, which is what a table - # carrying an intake form AND a correction form needs. - out["formToken"] = _s(raw.get("formToken") if "formToken" in raw - else prev.get("formToken"), 64).strip() - if key == "webhook": - # The token is MINTED here, once, and survives every later patch — rotating it on - # every Save would silently break the external caller the URL was given to. - out["token"] = _s(prev.get("token"), 64) or _secrets_token() - # ⭐ WAVE 24 — DEBT D-41: the request BODY, mapped onto record fields by config. - # ⚠ BOTH HALVES ARE OPTIONAL, and that is what keeps this additive: a webhook with no - # map behaves exactly as it did — it fires the flow and reads nothing — so the live - # webhook automations are untouched. `webhook` deliberately stays OUT of - # `TRIGGER_TABLE_KEYS`: joining it would make a table REQUIRED for `configured`, and - # every existing webhook trigger would go unconfigured and stop firing. - table = _s(raw.get("table") if "table" in raw else prev.get("table"), 60).strip() - if table and not table.startswith(UT_PREFIX): - return None, ("a webhook writes into a blank database (ut_*). " - f"{table!r} is not one") - out["table"] = table - fmap, ferr = clean_body_map(raw.get("fieldMap") if "fieldMap" in raw - else prev.get("fieldMap")) - if ferr: - return None, ferr - out["fieldMap"] = fmap - if key == "email": - out["query"] = _s(raw.get("query") if "query" in raw else prev.get("query"), - 200).strip() or "in:inbox is:unread" - out["configured"] = _trigger_configured(out) - return out, None - - -#: D-41 ceilings. 40 mapped cells is `MAX_ACTION_VALUES` doubled — a webhook payload is somebody -#: else's schema and is legitimately wider than an action's hand-written value list. -MAX_BODY_FIELDS = 40 -MAX_BODY_DEPTH = 5 - - -def clean_body_map(raw): - """D-41: `{"": ""}` for a webhook. Returns `(map, error)`. - - Paths are DOTTED into nested objects (`customer.email`). ⛔ NO ARRAY INDEXING in v1, stated - rather than half-supported: `items.0.sku` would read as working for the first element and - silently write nothing the day a payload arrives with the list empty, which is the shape of - bug this module keeps paying for. A path that resolves to nothing writes nothing. - """ - if raw in (None, ""): - return {}, None - if not isinstance(raw, dict): - return None, "the webhook field map must be an object of {body path: field key}" - if len(raw) > MAX_BODY_FIELDS: - return None, f"a webhook maps at most {MAX_BODY_FIELDS} values onto a record" - out = {} - for path, field in raw.items(): - p = _s(path, 200).strip() - if not p: - return None, "a webhook mapping has an empty body path" - if len(p.split(".")) > MAX_BODY_DEPTH: - return None, (f"{p!r} reaches more than {MAX_BODY_DEPTH} levels into the payload. " - f"map a shallower value") - fk = re.sub(r"[^a-z0-9_]+", "_", _s(field, 60).strip().lower()).strip("_") - if not fk: - return None, f"the value at {p!r} is not mapped to a field" - out[p] = fk[:60] - return out, None - - -def body_value(body, path): - """One dotted path into a decoded JSON body, or None. Scalars only — a mapped value that is - an object or a list answers None rather than a stringified `{...}` in a cell, because the - Row contract is scalar and a serialised dict in a grid cell is unreadable and unfilterable.""" - cur = body - for part in str(path or "").split("."): - if not isinstance(cur, dict): - return None - cur = cur.get(part) - return cur if isinstance(cur, (str, int, float, bool)) else None - - -def webhook_row(rt, defn, body): - """D-41: write ONE record from a webhook payload. Returns `(row_id, mapped_count, note)`. - - ⚠ `note` EXISTS BECAUSE THE CAP WAS SILENT. A table at its row ceiling returned the same - `("", 0)` as "no map configured" and the door answered a cheerful 200 — indistinguishable, - from the only side the caller is on, from a payload whose paths did not resolve. That is the - D-11 class (a table that quietly stops growing), and the caller here is a machine that will - keep posting. The note rides the 200: the flow still fires, and the answer says why no row - was written. - - ⛔ THROUGH THE ENGINE'S OWN WRITER, so it emits no row events — the structural loop - prevention law (A2(2)). A sibling automation watching this table does NOT fire on a - webhook-written row, exactly as it does not fire on a scrape's rows. The webhook's OWN flow - fires, because `hook_fire` fires it explicitly, which is the difference between "this - trigger fired" and "a write happened". - """ - trg = (defn or {}).get("trigger") or {} - table, fmap = str(trg.get("table") or ""), dict(trg.get("fieldMap") or {}) - if not table or not fmap or not isinstance(body, dict): - return "", 0, "" # no map configured — nothing to report - t = ut_get(rt, table) - if t is None: - return "", 0, f"{table} no longer exists, so nothing was written" - values = {} - for path, fkey in fmap.items(): - v = body_value(body, path) - if v is not None: - values[fkey] = _s(v, 500) if isinstance(v, str) else v - if not values: - return "", 0, ("none of the mapped paths resolved to a value in this payload. " - "check the paths against what you are sending") - rows = dict((t.get("rows") or {})) - if len(rows) >= row_cap(table): - return "", 0, (f"{table} is at its {row_cap(table)}-row limit, so no record was " - f"written (the flow still ran)") - rid = str(max([int(r) for r in rows if str(r).isdigit()] or [0]) + 1) - rows[rid] = values - ut_write_rows(rt, table, rows) - return rid, len(values), "" - - -def _trigger_configured(trg, config=None): - """Is this trigger complete enough to fire? One reader, because "configured" is asserted in - three places (the wire, the graph node, the hooks) and three copies of a boolean is how a - surface says "ready" about a trigger the engine skips. - - ⭐ WAVE 24 (A2) — `config` is OPTIONAL and only `ig_profile_match` reads it, because that is - the one trigger whose configuration lives in the DEFINITION's config (the discovery filters) - rather than on the trigger. `clean_trigger` calls this without it and so answers - conservatively (False); `clean_definition` calls it again with the validated config and - refines. Conservative-then-refined is the fail-closed order — the reverse would flash - "ready" on a trigger with nothing to search for. - """ - key = str((trg or {}).get("key") or "") - # ⭐ WAVE 29 — BOTH discovery triggers, and they answer identically: a corpus search with no - # filter is not a search, it is a request for the whole index. Named as a pair rather than - # `or`-ed onto the Instagram line so a third network joins by adding a key, not by editing a - # boolean expression. - if key in ("ig_profile_match", "tiktok_profile_match"): - return bool((config or {}).get("predicates")) - if key in TRIGGER_TABLE_KEYS and not trg.get("table"): - return False - if key == "event_field": - # C-TRIG law 4: the CONDITION is now the whole of it. No condition = "fire on anything, - # ever" — which is not a trigger, it is a description of the table. - # ⚠ STATED CONSEQUENCE OF THE MIGRATION: a live automation that narrowed by FIELD alone - # and carried no condition becomes `configured: false` on its next clean. It stops - # firing, and it SAYS SO — the graph node reads "Finish setting this trigger up before it - # can fire" and `configured` rides the wire. Visibly unfinished, never silently inert. - return bool(trg.get("when")) - if key == "enters_view": - return bool(trg.get("viewId")) - return True - - -def _secrets_token(): - import secrets as _sec - return _sec.token_urlsafe(24) - - -# --- the circuit breaker (A2(2)) — process memory, like _RUNNING: a counter that outlives the -# process would keep punishing an automation for a storm that ended with the restart. -_FIRES = {} -_FIRES_LOCK = threading.Lock() - - -def _breaker_trips(tenant, auto_id, now=None): - now = now if now is not None else time.time() - key = (tenant, str(auto_id)) - with _FIRES_LOCK: - log = [t for t in _FIRES.get(key, []) if now - t < FIRE_WINDOW_SECONDS] - log.append(now) - _FIRES[key] = log - return len(log) > FIRE_LIMIT - - -def _pause_trigger(rt, auto_id, note): - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - d = cur.get(str(auto_id)) - if d is not None: - trg = d.get("trigger") - if isinstance(trg, dict): - trg["paused"] = True - d["statusNote"] = _s(note, 200) - return cur - _store_update(rt, _up, flush="sync") - - -def trigger_fire(rt, tenant, auto_id, log=print, rows=None): - """One trigger firing — breaker first, then the ordinary async run. False when it did not - start (breaker, or already running — both are answers, not errors).""" - if _breaker_trips(tenant, auto_id): - note = (f"auto-paused: more than {FIRE_LIMIT} trigger fires in " - f"{FIRE_WINDOW_SECONDS // 60} minutes. Something is writing this trigger's " - f"subject in a loop") - _pause_trigger(rt, auto_id, note) - log(f"[aios-auto] breaker: {auto_id} {note}") - return False - return run_async(rt, tenant, auto_id, username="automation", log=log, rows=rows) - - -# --- the settle buffer (A2(1)) — per (tenant, automation), coalescing a burst into ONE -# evaluation. With EVENT_SETTLE_SECONDS == 0 the evaluation is INLINE (deterministic for gates). -_SETTLE = {} -_SETTLE_LOCK = threading.Lock() - - -def _settle_buffer(rt, tenant, auto_id, row_id, field="", after=None, log=print): - """Buffer ONE touched row for a coalesced evaluation, carrying the written cell. - - ⚠ WAVE 23 — THE WRITTEN VALUE IS STILL LOAD-BEARING, and an earlier draft of this wave - dropped it on the theory that `_settle_eval` could just read the row. It cannot, and the - reason is worth stating because it is invisible from this file: an ordinary ut cell typed at - the grid door lands in the editor's PER-USER OVERLAY stratum - (`grid_events.overlay_patch` → `table_store.patch_overlay`), not in the `user_tables` - definition rows. Only stage-field writes go through `patch_cells` to the shared rows. So for - the common case the value that just changed exists ONLY in this event, and a definition-row - read sees the pre-write value — the trigger would evaluate stale and never fire. - `_settle_eval` therefore MERGES: the definition row underneath (which is what lets a C4 tree - read the record's other columns) with the written cells on top. - """ - key = (tenant, str(auto_id)) - cell = {str(field): after} if field else {} - if EVENT_SETTLE_SECONDS <= 0: - with _SETTLE_LOCK: - buf = _SETTLE.setdefault(key, {"rows": {}}) - buf["rows"].setdefault(str(row_id), {}).update(cell) - _settle_eval(rt, tenant, auto_id, log=log) - return - with _SETTLE_LOCK: - buf = _SETTLE.setdefault(key, {"rows": {}}) - buf["rows"].setdefault(str(row_id), {}).update(cell) - timer = buf.get("timer") - if timer is not None: - timer.cancel() # the burst continues — push the window out - timer = threading.Timer(EVENT_SETTLE_SECONDS, _settle_eval, - args=(rt, tenant, auto_id), kwargs={"log": log}) - timer.daemon = True - buf["timer"] = timer - timer.start() - - -def view_filter(rt, table_key, view_id): - """`(tree, fields, problem)` for one saved view on a user table — the substrate the - `enters_view` trigger tests membership against (C3-v2 / owner R2). - - Personal strata first (`find_view`), then the shared bucket, because a view somebody shared - is exactly the kind an automation gets pointed at. A view that has been deleted answers a - PROBLEM rather than an empty tree: an empty tree matches everything, so degrading to one - would turn "when a record enters Overdue" into "on every write", which is the widening this - module refuses everywhere else. - """ - key = str(table_key or "") - vid = str(view_id or "").strip() - if not key or not vid: - return None, [], "the trigger names no view" - try: - import core.table_store as table_store - tops = table_store.make(f"{key}_table_workspace", st=rt) - found = tops.find_view(vid) - view = (found[1] if found else None) or tops.shared_view(vid) - except Exception as e: # noqa: BLE001 - return None, [], f"the view could not be read ({type(e).__name__})" - if not isinstance(view, dict): - return None, [], f"view {vid!r} no longer exists on {key}" - cfg = view.get("config") or {} - tree = {"nodes": cfg.get("filters") or [], "conj": cfg.get("filterConj") or "and"} - return tree, list((ut_get(rt, key) or {}).get("fields") or []), "" - - -def _row_gate(rt, defn, trg): - """The MATCH gate for a row-event trigger: `(row -> bool) | None`, plus a problem string. - - None means the trigger has NO match gate — the write itself is the event (wave 22's "the - field changed", and `record_updated` over any field). A problem means the gate cannot be - built, and the caller must then fire NOTHING: a gate we cannot evaluate is not a gate that - passes. - """ - key = trg.get("key") - if key == "enters_view": - tree, fields, problem = view_filter(rt, trg.get("table"), trg.get("viewId")) - if problem: - return None, problem - import harness.filter_eval as filter_eval - return (lambda row: filter_eval.matches(tree, row, fields)), "" - when = trg.get("when") - if when: - return (lambda row: lane_match(when, row)), "" - return None, "" - - -def _settle_eval(rt, tenant, auto_id, log=print): - """Evaluate one settled burst: edge over per-record armed state, flood hold, then fire.""" - with _SETTLE_LOCK: - buf = _SETTLE.pop((tenant, str(auto_id)), None) - written = dict((buf or {}).get("rows") or {}) - touched = list(written) - if not touched: - return - d = all_definitions(rt).get(str(auto_id)) - trg = (d or {}).get("trigger") or {} - if trg.get("key") not in ("event_field", "record_updated", "enters_view") \ - or trg.get("paused") or not trg.get("enabled", True) \ - or not trg.get("configured", True): - return - gate, problem = _row_gate(rt, d, trg) - if problem: - # LOUD, once, and it does not fire. A trigger pointed at a deleted view is a broken - # automation, not a quiet no-op — the note is what the surface shows instead of "On". - if (d.get("statusNote") or "") != problem: - _pause_trigger(rt, auto_id, problem) - log(f"[aios-auto] trigger gate: {auto_id} {problem}") - return - fired = [] - if gate is not None: - rows = (ut_get(rt, trg.get("table") or "") or {}).get("rows") or {} - disarmed = set((d.get("state") or {}).get("eventDisarmed") or []) - nxt = set(disarmed) - for rid in touched: - # The merge (see `_settle_buffer`): the shared definition row underneath so a C4 - # tree can read the record's other columns, the just-written cells on top because - # for an ordinary ut column this event is the ONLY place that value exists yet. - if gate({**(rows.get(rid) or rows.get(str(rid)) or {}), - **(written.get(rid) or {})}): - if rid not in disarmed: - fired.append(rid) # false→true THIS evaluation: the edge - nxt.add(rid) - else: - nxt.discard(rid) # left the state — re-armed (Airtable's rule) - if nxt != disarmed: - set_state(rt, auto_id, {"eventDisarmed": sorted(nxt)[:5000]}) - if not fired: - return - else: - fired = list(touched) # "the field changed" — the burst is the edge - if len(fired) > FLOOD_LIMIT: - _commit_run(rt, auto_id, "partial", - f"the trigger matched {len(fired)} records in one evaluation. More than " - f"the {FLOOD_LIMIT}-record flood hold, so nothing ran. Press Run now to " - f"process them deliberately", {"held": len(fired)}, True) - return - trigger_fire(rt, tenant, auto_id, log=log, rows=fired) - - -def _seed_event_state(rt, defn): - """A2(1)'s enable rule: records ALREADY matching when the trigger is set start DISARMED, so - turning the trigger on fires nothing — the first fire needs a real false→true transition. - Evaluated over the definition rows (the machine-written truth this trigger class watches). - - Wave 23: one seeder for all three gated triggers, reading the SAME `_row_gate` the evaluation - reads. Two implementations of "does this row match" is how a seed disagrees with the edge it - is supposed to arm, and the symptom would be a flood of fires the moment somebody enables it. - """ - trg = (defn or {}).get("trigger") or {} - if trg.get("key") not in ("event_field", "record_updated", "enters_view") \ - or not trg.get("configured"): - return - gate, problem = _row_gate(rt, defn, trg) - if gate is None or problem: - return # no match gate ⇒ nothing to arm; a broken gate seeds nothing - rows = (ut_get(rt, trg.get("table") or "") or {}).get("rows") or {} - matching = sorted(str(rid) for rid, row in rows.items() if gate(row or {})) - set_state(rt, defn.get("id"), {"eventDisarmed": matching[:5000]}) - - -def grid_hook(evt): - """THE listener the human write doors emit into (registered onto - `core.user_tables.ROW_HOOKS` by `routes_automation` at import — the one place that may - import both sides). Never raises into a write path; a broken trigger listener must not - break typing into a cell.""" - try: - st = evt.get("st") - if st is None: - return - tenant = str(getattr(st, "key", "") or "royal-imports") - table = str(evt.get("table") or "") - kind = str(evt.get("type") or "") - # ⭐⭐ WAVE 31 · T35 (D-134) — `cached=True`, and this is the ONLY caller that passes it. - # This function runs once per ROW EVENT, so a 20,000-row import used to perform 20,000 - # whole-document deep copies of the automations bucket, under the store lock, to re-read a - # trigger set that had not changed. See `all_definitions` for why the memo is safe here - # and nowhere else. - defs = all_definitions(st, cached=True) - field = str(evt.get("field") or "") - rid = str(evt.get("rowId") or "") - for aid, d in defs.items(): - trg = (d or {}).get("trigger") or {} - key = str(trg.get("key") or "") - if trg.get("paused") or not trg.get("enabled", True) \ - or not trg.get("configured", True) \ - or key not in TRIGGER_ROW_KEYS \ - or str(trg.get("table") or "") != table: - continue - if kind == "record_created" and key == "record_created": - hw = _ig_int((d.get("state") or {}).get("rcHighwater")) or 0 - ridn = int(rid) if rid.isdigit() else None - if ridn is None or ridn <= hw: - continue # once per record EVER (A2(4)) — undo-proof - set_state(st, aid, {"rcHighwater": ridn}) - # ⛔⛔ W31-T35 — MIRROR THE WRITE INTO THE MEMO'S OWN COPY, IN THE SAME STATEMENT. - # This is the hazard a definitions memo creates and the reason D-134 is not a - # one-line change: this branch READS `rcHighwater` and WRITES it, so within one - # import burst the second row would compare against the highwater the FIRST row - # set — and read the pre-write value out of the memo, fire again, and break - # A2(4)'s *"once per record EVER — undo-proof"*. `set_state` goes through - # `_store_update`, which drops the memo, but `defs` is the object already in hand - # for the rest of THIS event; keeping the two in step is what makes the memo safe - # rather than merely fast [[read-path-cannot-witness-write-path]]. - (d.setdefault("state", {}))["rcHighwater"] = ridn - trigger_fire(st, tenant, aid, rows=[rid]) - elif kind == "event_field" and key == "event_field": - # ⭐ WAVE 24 · law 4 — the per-FIELD narrowing is gone with the stored key. Every - # write on the watched table settles, and the CONDITION decides whether it fires - # (`_row_gate`). The old `not trg.get("field") or str(...) == field` test would - # now always take its first arm anyway; leaving a read of a key the validator no - # longer writes is the drift seat this migration exists to close. - _settle_buffer(st, tenant, aid, rid, field, evt.get("after")) - elif kind == "event_field" and key == "record_updated" and ( - not (trg.get("fields") or []) or field in (trg.get("fields") or [])): - _settle_buffer(st, tenant, aid, rid, field, evt.get("after")) - elif key == "enters_view": - # BOTH kinds feed it: a row can enter a view by being edited into its filter or - # by being CREATED already inside it. Listening only to edits would silently miss - # every new record — the half of the definition a reader assumes is covered. - _settle_buffer(st, tenant, aid, rid, field, evt.get("after")) - except Exception as e: # noqa: BLE001 - print(f"[aios-auto] trigger hook failed: {type(e).__name__}: {e}") - - -def form_fired(rt, table_key, row_id, values=None, form_token=""): - """⭐ THE FROZEN SIGNATURE session D calls from the public form door (contract C9/W23-W7). - - One submitted form row → every `form_submitted` automation watching that database fires. - Returns the list of automation ids that started, so the door can log what it set off (and so - the gate can assert it, rather than asserting a side effect nobody can see). - - ⛔ THIS IS A HUMAN DOOR, deliberately: an anonymous submission is a person filling in a form, - so it fires triggers exactly like typing into a cell does. The loop-prevention law is not - weakened by that — the engine's own writers still never reach here (only `routes_forms` calls - it), so an automation cannot create a form row and re-fire itself. - - `values` is accepted and unused today: the row is already written when this is called, and - the flow reads it from the table. It stays in the signature because the caller HAS it and a - later refire policy ("only when field X was submitted") needs it — a parameter added later - would mean changing D's call site in a wave that does not own it. - """ - started, table = [], str(table_key or "") - if not table: - return started - tenant = str(getattr(rt, "key", "") or "royal-imports") - for aid, d in all_definitions(rt).items(): - trg = (d or {}).get("trigger") or {} - if trg.get("key") != "form_submitted" or trg.get("paused") \ - or not trg.get("enabled", True) or not trg.get("configured", True): - continue - if str(trg.get("table") or "") != table: - continue - want = str(trg.get("formToken") or "") - if want and not hmac.compare_digest(want, str(form_token or "")): - continue # this automation watches a DIFFERENT form on that table - if trigger_fire(rt, tenant, aid, rows=[str(row_id)] if row_id else None): - started.append(aid) - return started - - -def hook_fire(rt, tenant, auto_id, token, body=None): - """The webhook trigger's decision, separated from FastAPI so the gate can drive it. - Returns `(status, payload)` — 404 unknown, 403 wrong/missing token or wrong trigger kind, - 409 already running, 200 started. - - ⭐ WAVE 24 (D-41): `body` is the decoded JSON payload, or None when the caller sent none or - sent something that is not JSON. It is written onto a record BEFORE the flow fires — the - flow's actions walk the record the webhook just created, which is the whole point of mapping - it. `body=None` is the pre-wave behaviour exactly, so an existing caller is unaffected. - """ - import hmac as _hmac - defn = all_definitions(rt).get(str(auto_id)) - if defn is None: - return 404, {"error": "unknown_automation"} - trg = defn.get("trigger") or {} - want = str(trg.get("token") or "") - if trg.get("key") != "webhook" or not want: - return 403, {"error": "no_webhook", "message": - "this automation has no webhook trigger"} - if trg.get("paused") or not trg.get("enabled", True): - return 403, {"error": "webhook_off", "message": "the webhook trigger is turned off"} - if not _hmac.compare_digest(want, str(token or "")): - return 403, {"error": "bad_token", "message": "that token is not valid"} - # D-41: map the payload onto a record FIRST, so the flow that fires next walks it. - row_id, mapped, note = webhook_row(rt, defn, body) - started = trigger_fire(rt, tenant, auto_id, rows=[row_id] if row_id else None) - out = {"started": bool(started), "at": _iso(), - # Answered even when zero, so a caller wiring a map up can see whether their paths - # resolved. Silence here would make "my JSON is not landing" undebuggable from the - # outside, which is the only side the caller is on. - "rowId": row_id or None, "mapped": mapped} - if note: - out["note"] = note # a 200 that wrote no row SAYS which reason it was - return 200, out - - -def email_poll(rt, tenant, auto_id, defn, log=print, _list=None, _read=None): - """The email trigger's tick half (C3): poll the CREATOR's Gmail through C5's seam, write a - row per NEW matching message, fire the flow. `_list`/`_read` are injection points so the - gate drives this without a network; production leaves them None. - - Fail-closed and QUIET when unconnected: the statusNote says so ONCE (not a run entry per - tick — 96 identical failures a day is a klaxon, not a status). Bounded everywhere: at most - `EMAIL_MAX_PER_POLL` new messages per tick, the flood hold above that, one coalesced write. - """ - trg = defn.get("trigger") or {} - if trg.get("key") != "email" or trg.get("paused") or not trg.get("enabled", True): - return None - import oauth_connect - creator = str(defn.get("createdBy") or "").strip() or "admin" - token, err = oauth_connect.google_creds(rt, creator) - if err: - if (defn.get("statusNote") or "") != err: - def _note(cur): - cur = cur if isinstance(cur, dict) else {} - dd = cur.get(str(auto_id)) - if dd is not None: - dd["statusNote"] = _s(err, 200) - return cur - _store_update(rt, _note, flush="sync") - return None - lister = _list or (lambda q, n: oauth_connect.gmail_list(token, q, n)) - reader = _read or (lambda mid: oauth_connect.gmail_message(token, mid)) - ids, lerr = lister(trg.get("query") or "", EMAIL_MAX_PER_POLL + FLOOD_LIMIT) - if lerr: - return _commit_run(rt, auto_id, "partial", f"the Gmail poll did not answer. {lerr}", - {}, True) - seen = set((defn.get("state") or {}).get("emailSeen") or []) - fresh = [m for m in ids if m not in seen] - if not fresh: - return None # nothing new is not a run — no history spam - if len(fresh) > FLOOD_LIMIT: - return _commit_run(rt, auto_id, "partial", - f"{len(fresh)} new emails matched in one poll. More than the " - f"{FLOOD_LIMIT}-record flood hold, so nothing was written. Narrow " - f"the query, or press Run now after adjusting it", - {"held": len(fresh)}, True) - fresh = fresh[:EMAIL_MAX_PER_POLL] - rows_in, notes = [], [] - for mid in fresh: - row, rerr = reader(mid) - if row: - rows_in.append(row) - elif rerr: - notes.append(rerr) - table_key = (defn.get("config") or {}).get("targetTable") or "" - if not table_key: - return _commit_run(rt, auto_id, "error", - "the email trigger has nowhere to write. The automation names no " - "target database", {}, False) - ut_ensure(rt, (defn.get("config") or {}).get("targetLabel") or defn.get("name") or "Inbox", - EMAIL_FIELDS, username=str(defn.get("createdBy") or "automation"), key=table_key) - existing = dict((ut_get(rt, table_key) or {}).get("rows") or {}) - merged, counts = upsert_rows(existing, rows_in, "email_id", cap=row_cap(table_key)) - ut_write_rows(rt, table_key, merged) - new_seen = (list(seen) + fresh)[-EMAIL_SEEN_CAP:] - set_state(rt, auto_id, {"emailSeen": new_seen}) - summary = (f"{len(fresh)} new email{'' if len(fresh) == 1 else 's'} matched. " - f"{counts['inserted']} row{'' if counts['inserted'] == 1 else 's'} written") - if notes: - summary += f". {notes[0][:100]}" - entry = _commit_run(rt, auto_id, "ok" if not notes else "partial", summary, counts, - True, affected=list(merged)[:200]) - trigger_fire(rt, tenant, auto_id, log=log) - return entry - - -def compose_sentence(defn): - """The one-sentence server-composed summary (airtable-brief rec 7): rendered from the - definition so it cannot lie about what runs.""" - cfg = defn.get("config") or {} - trg = defn.get("trigger") or {} - sched = defn.get("schedule") or {} - kind = defn.get("kind") - if trg.get("key") == "event_field": - # Law 4: no watched field any more, so the sentence stops naming one. It said - # "When None changes on ut_x" the moment the key stopped being stored. - head = f"When a record in {trg.get('table')} matches conditions" - elif trg.get("key") == "record_updated": - head = f"When a record in {trg.get('table')} is updated" - elif trg.get("key") == "ig_profile_match": - head = "When an Instagram profile fits the criteria" - elif trg.get("key") == "tiktok_profile_match": - head = "When a TikTok profile fits the criteria" - elif trg.get("key") == "record_created": - head = f"When a record is created in {trg.get('table')}" - elif trg.get("key") == "webhook": - head = "When the webhook is called" - elif trg.get("key") == "email": - head = f"When an email matches {trg.get('query')}" - elif sched.get("enabled"): - head = f"{_cron_label(sched.get('cron'))}" - else: - head = "When you press Run now" - if kind == "scrape_db": - host = urlparse(str(cfg.get("url") or "")).hostname or "the page" - body = f"read {host} and upsert rows into {cfg.get('targetTable') or 'a new database'}" - elif kind == "field_instagram": - # ⚠ NO RUNG CLAUSE (R5). There is one way to capture a profile now, so ", exact counts - # first" / ", anonymous only" described a choice that no longer exists. - body = f"capture Instagram profiles for {cfg.get('targetTable') or 'the database'}" - elif kind in DISCOVERY_KINDS: - # ⭐⭐ WAVE 30 · T05 — the arm covers both kinds, and the network is NAMED from the kind - # rather than hard-coded into the sentence. Instagram-only, a TikTok search fell into the - # `plain` branch below and introduced itself as *"do nothing yet — this automation has no - # actions"*: a flatly false sentence, on the one surface whose stated promise is that it - # cannot lie about what runs. (The `plain` branch's own comment records the mirror-image - # incident — it USED to be this arm, and described every plain automation as an Instagram - # search for 0 profiles. The same two branches have now mis-described each other's - # automations in both directions, which is why neither may be a fallthrough.) - _dplatform, _dtable, _, _ = discovery_facts(kind) - body = (f"search {_dplatform} for up to {cfg.get('recordsLimit') or 0} profiles into " - f"{cfg.get('targetTable') or _dtable}") - else: - # ⭐ WAVE 24 — `plain` describes itself from its FLOW, because the flow is all it has. - # ⛔ THIS BRANCH USED TO BE `discover_instagram`'s, so before the arm above existed every - # plain automation would have introduced itself as "search Instagram for up to 0 profiles - # into ut_ig_candidates" — a sentence composed from a definition that says none of it, - # on the one surface whose whole promise is that it "cannot lie about what runs". - n = sum(1 for _ in walk_actions((defn.get("flow") or {}).get("actions"))) - tbl = cfg.get("targetTable") or trg.get("table") or "" - body = ((f"run {n} action{'' if n == 1 else 's'}" + (f" on {tbl}" if tbl else "")) - if n else "do nothing yet. This automation has no actions") - lanes = cfg.get("lanes") or [] - tail = f", then route each record across {len(lanes)} lanes" if lanes else "" - return f"{head}, {body}{tail}." - - -def run_now(rt, tenant, auto_id, username="automation", log=print, rows=None): - """Execute one automation SYNCHRONOUSLY. The route wraps this in a thread; the tick calls it - directly. Returns the run entry, or None when it was already running (the 409).""" - defn = all_definitions(rt).get(str(auto_id)) - if defn is None: - return None - # ⛔⛔ WAVE 32 · T45 (owner item 10) — AN UNCONFIGURED ACTION BLOCKS THE RUN, HERE, WHERE EVERY - # DOOR PASSES. The route checks too so a person gets a 400 rather than a silent no-op, but the - # tick and the webhook do not go through the route; a client-only block is not a block (D-112). - # ⚠ BEFORE `_claim`, deliberately: claiming and then refusing would leave the automation - # marked running until the release, i.e. a refusal that also produces a phantom 409 for the - # next honest attempt. - _refusal = run_refusal(defn) - if _refusal: - log(f"[aios-auto] refused: {_refusal}") - return None - if not _claim(tenant, auto_id): - return None - # ⛔ THE TABLES AN AUTOMATION MAKES BELONG TO THE AUTOMATION'S CREATOR, not to whoever - # happened to press Run — and above all not to the scheduler, which is not a person and - # cannot own anything (see `ut_ensure`). Without this the owner of a database was decided by - # whether a human or a cron got to the first run first. - owner = str(defn.get("createdBy") or "").strip() - if owner and username in MACHINE_OWNERS: - username = owner - try: - _step(tenant, auto_id, "running") - # A deferred metric snapshot is paid work already in Bright Data's queue. Collect it - # first and do not start another profile scrape while it is outstanding: re-running the - # action would buy duplicate engagement reads and reintroduce the timeout this handoff - # exists to remove. The normal scheduler calls this path too via `pending_collect_ids`. - # ⭐ 2026-08-09 — THE PROFILE HANDOFF IS COLLECTED FIRST, for the same reason and one - # rung earlier: a profile snapshot the vendor is still building is paid work, and - # starting a fresh scrape for the same handle would buy the identical row a second time. - # Ahead of the metric collector because the profile IS the thing the run was asked for; - # the engagement batches hang off it. - if _pending_profile_tasks(defn): - state, summary, counts, affected, steps = collect_pending_profile_snapshots( - rt, defn, username=username, log=log, - step=lambda text: _step(tenant, auto_id, text)) - return _commit_run(rt, auto_id, state, summary, - {k: v for k, v in counts.items() if k != RUN_NOTES_KEY}, - state != "error", affected, steps, - notes=counts.get(RUN_NOTES_KEY)) - if _pending_metric_tasks(defn): - state, summary, counts, affected, steps = collect_pending_metric_snapshots( - rt, defn, username=username, log=log, - step=lambda text: _step(tenant, auto_id, text)) - return _commit_run(rt, auto_id, state, summary, - {k: v for k, v in counts.items() if k != RUN_NOTES_KEY}, - state != "error", affected, steps, - notes=counts.get(RUN_NOTES_KEY)) - runner = RUNNERS.get(defn.get("kind")) - if runner is None: - return _commit_run(rt, auto_id, "error", - f"unknown automation kind {defn.get('kind')!r}", {}, False) - try: - # ⭐ WAVE 24 (item 6, on D's measurement) — THE LIVE STEP, closed over this run. - # `status.step` was already on the wire and D's half renders it; measured against the - # code, it was set exactly ONCE ("running") and never again, so the word would have - # been identical whether a run was mid-vendor-wait or genuinely hung. Rendering a - # constant as a progress indicator is worse than rendering nothing: it looks like an - # answer. The runners move it now, and the 120 s Bright Data wait counts out loud. - state, summary, counts, affected, steps = runner( - rt, defn, username=username, log=log, - step=lambda text: _step(tenant, auto_id, text), rows=rows) - except Refused as e: - return _commit_run(rt, auto_id, "error", f"refused: {e}", {}, False) - except Exception as e: # noqa: BLE001 - log(f"[aios-auto] run {auto_id} failed: {type(e).__name__}: {e}") - return _commit_run(rt, auto_id, "error", - f"{type(e).__name__}: {str(e)[:200]}", {}, False) - # ⭐ WAVE 23 (C4/C5) — THE FLOW RUNS HERE, after the machine steps and before the run is - # committed, over the records this run actually touched. ONE call site rather than three - # inside the runners: every kind gets actions and endings for free, and a fourth runner - # cannot forget to opt in. - # - # ⚠ Its absence was the wave's most expensive near-miss: actions were stored, validated, - # wired to the wire and covered by twelve gate checks that all called `apply_actions` - # DIRECTLY — so the whole feature was green and unreachable. A person would have built a - # flow, pressed Run now, and watched nothing happen. The gate now drives `run_now`. - # - # A failing action must not fail the RUN: the machine steps already wrote their rows and - # reporting that as an error would misdescribe what happened. It degrades to `partial` - # with the reason in the summary — the cap_note discipline. - # ⚠ BOUND BEFORE THE `try`. The `except` below falls through to the same `_commit_run`, - # which now reads this name — an assignment only on the success path would turn any - # action failure into a NameError inside the handler that exists to prevent exactly that. - # ⚠ THE RUNNER PRODUCES NOTES TOO, and its are the ones that survive a run which walked - # NOTHING — the branch where every candidate was a known-dead handle, i.e. exactly the - # run a person stares at wondering why the automation stopped doing anything. - run_notes = list((counts or {}).pop(RUN_NOTES_KEY, None) or []) - try: - a_counts = apply_actions(rt, defn, _flow_table(defn), affected or [], - username=username, log=log, - step=lambda text: _step(tenant, auto_id, text)) - # ⭐⭐ D-103 — POPPED BEFORE THE MERGE. The per-record reasons ride inside `counts` so - # the runner contract keeps its shape, and they must leave before the merge or they - # would be a "count" everywhere downstream. - run_notes += list(a_counts.pop(RUN_NOTES_KEY, None) or []) - counts = {**(counts or {}), **{k: v for k, v in a_counts.items() if v}} - if a_counts.get("enrichMetricBatchesPending"): - batches = int(a_counts["enrichMetricBatchesPending"]) - state = "partial" if state != "error" else state - summary += (f". {batches} post-engagement batch" - f"{'' if batches == 1 else 'es'} still building; Views and other " - "metrics will be collected automatically without another paid read") - if a_counts.get("enrichUnbound"): - # ⛔ D-79(2): AND THE FIX GOES IN THE SUMMARY, not only in the log. The run is - # `partial` because it genuinely did part of its job — it walked the records — and - # the sentence names the ONE thing that has to change, in the two places a person - # can change it. The old behaviour was `ok` with an empty table. - state = "partial" if state != "error" else state - summary += (". The Instagram step did not run: this database has no profile " - "column. Name one on the step, or mark a text column as the " - "Instagram profile" - + _unbound_hint(rt, _flow_table(defn))) - if a_counts.get("ttEnrichUnbound"): - # ⛔ WAVE 30 · T08 — ITS OWN SENTENCE, not the one above with a word swapped by a - # variable. A flow may carry BOTH steps, and the fix a person has to apply is - # per-column: naming an Instagram profile column does nothing for a TikTok step, - # so a single sentence covering "the enrich step" would send them to the wrong - # place half the time. Both may appear on one run, which is correct. - state = "partial" if state != "error" else state - summary += (". The TikTok step did not run: this database has no TikTok profile " - "column. Name one on the step, or mark a text column as a TikTok " - "profile" - + _unbound_hint(rt, _flow_table(defn))) - if a_counts.get("ttEnrichBlocked"): - state = "partial" if state != "error" else state - tt_note = next((n for n in run_notes if "(TikTok): " in n), "") - summary += (f". {int(a_counts['ttEnrichBlocked'])} TikTok profile read(s) were " - "blocked" + (f". {_s(tt_note, 220)}" if tt_note else "")) - if a_counts.get("enrichProfileBatchesPending"): - # ⭐ The paid profile the vendor is still building. Said out loud so a run that - # looks like a failure is read as the handoff it is — the tick finishes it. - n = int(a_counts["enrichProfileBatchesPending"]) - state = "partial" if state != "error" else state - summary += (f". {n} profile read{'' if n == 1 else 's'} took longer than the " - "wait allows and will be collected automatically, at no extra cost") - if a_counts.get("enrichBlocked"): - state = "partial" if state != "error" else state - # ⭐⭐ D-103 — THE REASON IS IN THE SENTENCE, not only behind a click. "1 profile - # read(s) were blocked" is the exact string the owner read three mornings running - # before asking "wtf is going on"; it names a quantity and withholds the one - # thing that would let anybody act. The first note is the vendor's own words. - # ⚠ ...and the Instagram selector SKIPS the tagged TikTok lines for the same - # reason. Two sentences quoting each other's vendor reason is worse than one. - blocked_note = next((n for n in run_notes - if ": " in n and "(TikTok): " not in n), "") - summary += (f". {int(a_counts['enrichBlocked'])} profile read(s) were blocked" - + (f". {_s(blocked_note, 220)}" if blocked_note else "")) - # ⭐ WAVE 25 · C5 — A FULL TARGET IS A `partial` RUN THAT SAYS SO. D-11 made this the - # law for the runners' OWN writes (`cap_note`), and `create_record` never joined: it - # logged the cap and rolled up `ok`, so a flow that had silently stopped writing - # looked exactly like one that had nothing to write. Same rule, same sentence shape. - if a_counts.get("createCapped"): - state = "partial" if state != "error" else state - summary = (f"{summary}. {a_counts['createCapped']} row(s) NOT created: a target " - f"database is at its row cap") - except Exception as e: # noqa: BLE001 - log(f"[aios-auto] actions on {auto_id} failed: {type(e).__name__}: {e}") - state = "partial" if state != "error" else state - summary = f"{summary}. The actions did not finish ({type(e).__name__})" - return _commit_run(rt, auto_id, state, summary, counts, state != "error", affected, - steps, notes=run_notes) - finally: - _release(tenant, auto_id) - - + except Exception as e: # noqa: BLE001 + log(f"[aios-auto] metric refresh {tk} failed: {type(e).__name__}: {e}") + return touched + + +def purge_subject(rt, handle): + """D-24: right-to-erasure for ONE Instagram subject — every row about them leaves the + tenant's four `ut_ig_*` tables AND the platform master (R2 made the master half + non-optional: a purge that missed the pooled copy would not be erasure). Returns + `{table: removed}` counts, master rows prefixed `master:` — every count drills to what is + now ABSENT, which is the one aggregate whose drill is emptiness.""" + subject = str(handle or "").strip().lstrip("@").lower() + if not subject: + return {} + counts = {} + tables = ut_all(rt) + post_rows = (tables.get("ut_ig_posts") or {}).get("rows") or {} + codes = {str(r.get("shortcode") or "") for r in post_rows.values() + if str((r or {}).get("influencer_key") or "").strip().lower() == subject} + + keeps = { + "ut_ig_snapshots": lambda r: str((r or {}).get("influencer_key") + or "").strip().lower() != subject, + "ut_ig_posts": lambda r: str((r or {}).get("influencer_key") + or "").strip().lower() != subject, + "ut_ig_post_snapshots": lambda r: str((r or {}).get("shortcode") or "") not in codes, + DISCOVER_TABLE: lambda r: str((r or {}).get("handle") + or "").strip().lower() != subject, + } + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + for tk, keep in keeps.items(): + t = cur.get(tk) + if t is None: + continue + rows = t.get("rows") or {} + nxt = {rid: r for rid, r in rows.items() if keep(r)} + counts[tk] = len(rows) - len(nxt) + t["rows"] = nxt + return cur + + rt.update(UT_STORE_KEY, _up, flush="sync") + import ig_master + for bucket, n in (ig_master.purge_handle(subject) or {}).items(): + counts[f"master:{bucket}"] = n + return counts + + +# --------------------------------------------------------------------------------------------- +# TRIGGERS (wave 22, contract C3 + amendment A2 — owner ruling R4; closes D-33) +# --------------------------------------------------------------------------------------------- +# Six ways an automation starts, exactly: manual | schedule | event_field | record_created | +# webhook | email. The first two are what always existed (Run now; cron via the tick). The four +# new ones are EVENTS, and A2 makes their discipline LAW rather than taste: +# +# * **EDGE, NEVER LEVEL (A2(1)).** A condition trigger fires on entering the matching state, +# not for being in it. Implemented as per-record ARMED state over successive evaluations +# (`state.eventDisarmed`): a record fires when it matches while armed, DISARMS, and re-arms +# only by evaluating False — Airtable's documented leave-and-re-enter rule, without needing +# a before-image of a row whose truth is spread over strata. Enabling a trigger SEEDS the +# disarmed set with everything currently matching, so already-matching records do not fire +# (`_seed_event_state`). A settle window coalesces write bursts (the per-keystroke scar). +# * **LOOP PREVENTION IS STRUCTURAL (A2(2)).** The hooks live on the HUMAN doors only +# (`grid_events.overlay_patch`, `user_tables.add_row`); the engine's own writers +# (`ut_write_rows`, the runners' coalesced updates, `patch_cells` from `move_card`) never +# emit — so an automation's write cannot fire event triggers, its own or a sibling's, by +# construction. The circuit breaker on top (>60 fires/5 min auto-pauses with the reason as +# a statusNote) catches whatever construction did not foresee. +# * **FLOOD HOLD (A2(3)).** One evaluation yielding more than 100 candidate records holds +# instead of running — a partial run entry names the count and the deliberate way through +# (Run now). C4's discovery guard is this rule's special case. +# * **REFIRE DEFAULTS (A2(4)), hard-coded this wave:** record_created fires once per record +# EVER (a high-water mark over row ids, so an undo-restored row cannot re-fire); +# event_field fires every transition. + +# ── WAVE 23 · C3 — the trigger vocabulary v2 (owner ruling R2). ─────────────────────────────── +# Airtable's phrasing, because the owner asked for Airtable's builder and a trigger list that +# renames the same events is a second vocabulary to learn for no gain. +# +# ⚠ `event_field` KEPT ITS KEY and changed its LABEL to "When a record matches conditions". +# Renaming the key would have orphaned every stored trigger in production for a caption; the key +# is the contract with the store, the label is the contract with the reader, and they are allowed +# to disagree. What genuinely widened is its SHAPE: the watched field is now OPTIONAL, so the +# trigger covers Airtable's condition-only form (any write to the table, evaluated against a C4 +# tree) as well as wave 22's watch-one-field form. Both are the same edge rule underneath. +# +# ⛔ PLANNED ≠ STORABLE. `button_clicked` / `comment_added` ride the wire so the picker can show +# them faded with a reason (R2: "never a dead control") — and `clean_trigger` REFUSES them with a +# sentence. A vocabulary that renders an option the validator rejects is the wave-9 silent-drop +# class wearing a friendlier face; here the two lists are separate on purpose and the refusal +# names the state rather than pretending the key is unknown. +# ── WAVE 24 · C-TRIG (owner ruling R6). ─────────────────────────────────────────────────────── +# ⭐ INSTAGRAM DISCOVERY BECOMES A TRIGGER. It was a KIND you picked in a create wizard; the +# wizard is deleted, and "when an Instagram profile fits a criteria" is the honest shape anyway — +# it is the event this flow starts from. Picking it sets the definition's kind to +# `discover_instagram` (law 1), which is the ONLY way that kind is reachable now. +# +# ⛔ IT IS NOT A TABLE TRIGGER AND NOT A ROW TRIGGER. It watches nothing: it MAKES rows, on the +# schedule (or on Run now), so it stays out of both lists below and its node switch flips the +# CRON — see `TRIGGER_SCHEDULE_KEYS`. +# +# ⭐⭐ WAVE 29 (item 7 · D-9 · R1) — `tiktok_profile_match` MOVED HERE FROM `TRIGGER_PLANNED`, and +# that move is the whole of "TikTok is a real trigger now". The label was written a wave early in +# the owner's own words and has not changed; what changed is which tuple it sits in, because +# `clean_trigger` refuses the planned list with a sentence and accepts this one. +TRIGGER_KEYS = ("manual", "schedule", "event_field", "record_updated", "record_created", + "enters_view", "webhook", "email", "form_submitted", "ig_profile_match", + "tiktok_profile_match") +#: ⭐ WAVE 25 · C2 / owner ruling R9 — `web_page_changed` JOINS THE PLANNED LIST, and joining THIS +#: tuple rather than `TRIGGER_KEYS` is the whole of its implementation. `clean_trigger` refuses +#: everything here with a sentence, so the faded row is a wall; the picker shows it so the Scraper +#: section is not a section of one. +#: ⚠ `web_page_changed` IS NOT THE WEB ACTION (D-51). A trigger that notices a page changed and an +#: action that drives a browser are different builds; this row must not be read as progress on D-51. +TRIGGER_PLANNED = ("button_clicked", "comment_added", "web_page_changed") +TRIGGER_LABELS = { + "manual": "Manual", + "schedule": "At a scheduled time", + "event_field": "When a record matches conditions", + "record_updated": "When a record is updated", + "record_created": "When a record is created", + "enters_view": "When a record enters a view", + "webhook": "When a webhook is received", + "email": "When an email arrives", + "form_submitted": "When a form is submitted", + "ig_profile_match": "When an Instagram profile fits a criteria", + "button_clicked": "When a button is clicked", + "comment_added": "When a comment is added", + "web_page_changed": "When a website page changes", + "tiktok_profile_match": "When a TikTok profile fits a criteria", +} + +# ── WAVE 25 · C2 — THE PICKER TAXONOMY, and it lives HERE beside the vocabulary it describes. ── +# `group` already rode the wire as "Standard"/"Sources" (`routes_automation`), which is a +# distinction about where a trigger came FROM rather than about what a person is choosing. The +# question the picker actually asks is: does this fire on TIME, on your own DATA, or because +# something OUTSIDE said so. Three answers, and every trigger has exactly one. +# +# ⛔ TWO CONTROLS, NOT ONE, AND THE FIRST DRAFT HAD ONLY THE WRONG HALF. Indexing this map +# directly (`TRIGGER_GROUP_OF[k]`) makes an unclassified trigger a KeyError — which is exactly the +# incident `_triggers_vocab`'s `per.get(k, ...)` comment records: a key added to `TRIGGER_KEYS` +# without remembering a dict beside it 500'd `GET /automations`, the payload the whole automation +# surface polls every 2.5 s, with every gate green. A mis-grouped row is a cosmetic bug; a 500 is +# the surface. So: +# * RUNTIME fails SOFT — an unclassified trigger falls into `other`, which sorts LAST (the rule +# `ACTION_GROUP_ORDER` already uses: an unordered group sorts last, never first, because +# appearing at the top looks deliberate) and is honestly captioned rather than smuggled into +# Database. +# * THE GATE fails HARD — `verify_automation` asserts that NO shipped trigger lands in `other`, +# so the fallback is provably dead code in production and the classification is still +# mandatory. The fallback catches the accident; the gate stops it shipping. +# ⭐ WAVE 34 · R19 — CONNECTOR SITS ABOVE DATABASE, DIRECTLY UNDER TIME. Owner, verbatim: +# *"In the Trigger picker, the Connector section sits directly above the Database section, just +# under the Time trigger types."* So the two orders below are SWAPPED against wave 24's, and +# nothing else moved: same keys, same labels, same fallback. +# ⛔ THIS IS THE WHOLE OF R19 AND IT IS DELIBERATELY NOT A CLIENT CHANGE. `steps.ts::groupTriggers` +# sorts by each option's `groupOrder` and by nothing else, so re-ordering an array on the client +# would look right in a fixture and be wrong in production the moment the server re-sorted. +# ⚠ `verify_steps.py` CANNOT WITNESS THIS EDIT: its C2 leg supplies its OWN `groupOrder` in a +# fixture and asserts the client honours it, which stays true whatever these numbers say. The +# check that binds R19 to this table lives in `verify_automation` beside the vocab section. +TRIGGER_GROUPS = {"time": {"label": "Time", "order": 1}, + "connector": {"label": "Connector", "order": 2}, + "database": {"label": "Database", "order": 3}, + "other": {"label": "Other", "order": 99}} +#: Where an unclassified trigger goes. ⚠ Reaching this in production is a BUG the gate exists to +#: prevent — it is the soft landing, not a category anybody should be adding triggers to. +TRIGGER_GROUP_FALLBACK = "other" +TRIGGER_GROUP_OF = { + "manual": "time", "schedule": "time", + "event_field": "database", "record_updated": "database", "record_created": "database", + "enters_view": "database", "form_submitted": "database", + "button_clicked": "database", "comment_added": "database", + "email": "connector", "webhook": "connector", "ig_profile_match": "connector", + "web_page_changed": "connector", "tiktok_profile_match": "connector", +} +#: The SUB-group inside "Connector" — which connected thing this trigger comes through. +#: ⚠ THESE KEYS ARE GROUPING HANDLES FOR THE PICKER, NOT connector-directory slugs, and the two +#: genuinely differ: the directory's OAuth row for Gmail is `google` (the provider), while a +#: person choosing a trigger is picking *Gmail* (the product). A client that joined this key +#: against `/connectors/directory` would match `scraper` and `webhooks` and miss `gmail` — so it +#: must group by it and render `label`, never look it up. Said here because the miss would be +#: silent and partial, which is the worst shape. +#: (⚠ that example USED to read "`scraper` and `tiktok`" — R3 retired `tiktok` as a handle, and +#: the sentence is corrected here rather than left to rot into a lie about a key that is gone.) +TRIGGER_CONNECTOR = { + "email": {"key": "gmail", "label": "Gmail"}, + "webhook": {"key": "webhooks", "label": "Webhooks"}, + # ⭐ WAVE 30 · R3 — ONE "SCRAPER" BUCKET, AND IT HOLDS BOTH PLATFORMS. + # The owner, verbatim and for the third wave running: *"I say this multiple times already the + # damn Tiktok and Instagram belongs in the same bucket when creating the automation its under + # Scraper … Only when I click 'Scraper' under each automation trigger and actions would I see + # the option to choose either Instagram OR TikTok. That's it."* + # + # ⛔ THIS SUPERSEDES WAVE 29's RULE, and the old rule was not a typo — it was an argument: + # *"TikTok is its own connector, not the Scraper's … because the sub-group answers WHICH + # PRODUCT and never HOW BUILT."* Coherent, and not what was asked for. Instagram and TikTok are + # two PRODUCTS of one CAPABILITY (a social scraper bought from one vendor); a person opening + # this picker is choosing the capability first and the platform second. `verify_automation` + # asserted the old rule as an assertion AND as prose — a shipped gate forbidding the owner's + # ruling is most of why this complaint survived two waves — so it is INVERTED in this same + # change, comment included. + # + # ⚠ The Scraper sub-group now holds THREE rows: two built (Instagram, TikTok) and one faded + # (the page-change trigger). No display ORDER is emitted here — the client groups on `key` and + # owns its own ordering (contract C1). `tiktok` ceases to exist as a grouping handle. + "ig_profile_match": {"key": "scraper", "label": "Scraper"}, + "web_page_changed": {"key": "scraper", "label": "Scraper"}, + "tiktok_profile_match": {"key": "scraper", "label": "Scraper"}, +} +#: Triggers that watch a database and therefore need one named before they can fire. +TRIGGER_TABLE_KEYS = ("event_field", "record_updated", "record_created", "enters_view", + "form_submitted") +#: ⭐ WAVE 24 — triggers whose NODE SWITCH means the CRON rather than the trigger itself. +#: `manual`/`schedule` are not stored at all; `ig_profile_match` is stored and IS schedule-driven, +#: so flipping its node must flip the schedule. +#: +#: ⚠ THIS REPLACES A HAND-LISTED TUPLE IN `toggle_node` THAT WAS ALREADY WRONG. It read +#: `("event_field", "record_created", "webhook", "email")` — omitting `record_updated`, +#: `enters_view` and `form_submitted`, all three of which have been storable since wave 23. For +#: those, clicking the trigger node's switch flipped the CRON under a node labelled "When a +#: record is updated": a switch that lies, which is exactly what the tuple at `graph()` warns +#: about eight lines into its own comment. Derived from one named set now, so a trigger added to +#: `TRIGGER_KEYS` cannot silently join the wrong side of it. +#: ⚠ WAVE 29 — `tiktok_profile_match` BELONGS HERE FOR THE SAME REASON `ig_profile_match` DOES, +#: and forgetting it is precisely the failure this constant's own note describes: it watches no +#: table and MAKES rows on the schedule, so its node switch has nothing to flip but the cron. Left +#: out, a person clicking the TikTok trigger node's switch would toggle the trigger itself while +#: the schedule kept firing — a switch that lies. +TRIGGER_SCHEDULE_KEYS = ("manual", "schedule", "ig_profile_match", "tiktok_profile_match") +#: ⭐ WAVE 25 — DEBT D-55: "the cron drives this one", on the wire at last. +#: +#: ⛔ `TRIGGER_SCHEDULE_KEYS` MUST NOT SHIP VERBATIM, and the one-element difference is the entire +#: reason this constant exists rather than the tuple above being sent. That set answers "which +#: way does this trigger's NODE SWITCH flip" — and `manual` is in it only because a manual +#: automation's switch has nothing else to flip. Shipping it as "the cron drives this" would draw +#: a schedule face on the one trigger whose whole sentence is "It runs only when you press Run +#: now": a control contradicting its own description. +#: +#: D-55's history is why it is DERIVED rather than listed: the client carried +#: `CRON_DRIVEN_TRIGGERS = ["schedule", "ig_profile_match"]` — a hand-kept copy of a server fact +#: that fails VISIBLY but silently (a new cron-driven trigger simply shows no schedule face). +#: Subtracting from the engine's own set means a trigger added there cannot be forgotten here. +TRIGGER_CRON_KEYS = frozenset(TRIGGER_SCHEDULE_KEYS) - {"manual"} +#: Triggers the ROW HOOKS drive (as opposed to the tick, or an inbound HTTP call). Named once so +#: `grid_hook` and the gates read the same list instead of two matching `in (...)` tuples. +TRIGGER_ROW_KEYS = ("event_field", "record_updated", "record_created", "enters_view") +MAX_WATCH_FIELDS = 12 +#: The settle window for field-change bursts (A2(1)). 0 evaluates INLINE — the gates run there, +#: and so would a deployment that prefers immediacy over coalescing. +EVENT_SETTLE_SECONDS = float(os.environ.get("AIOS_EVENT_SETTLE_SECONDS") or 15) +FIRE_LIMIT = 60 # A2(2): fires per window before the breaker pauses +FIRE_WINDOW_SECONDS = 300 +FLOOD_LIMIT = 100 # A2(3): candidate records one evaluation may act on +EMAIL_SEEN_CAP = 500 # message-id dedupe memory per automation +EMAIL_MAX_PER_POLL = 25 # bounded by construction — a poll is a tick guest +CONSECUTIVE_FAILURE_PAUSE = 5 # airtable-brief rec 6: a dead credential must not burn quota + +EMAIL_FIELDS = [ + field_def("email_id", "Email id"), field_def("email_from", "From"), + field_def("email_subject", "Subject"), field_def("email_date", "Date"), + field_def("email_snippet", "Snippet"), field_def("email_seen_at", "Seen at"), +] + + +def clean_trigger(raw, previous=None): + """Validate a definition's `trigger`. Returns `(trigger|None, error)` — None is legal and + means what it always meant: manual + whatever `schedule` says. + + ⚠ A3 (2026-08-05): the stored/wire name is `key` (`kind` accepted on input for symmetry + with the definition's own vocabulary). And an INCOMPLETE event trigger is STORED INERT + rather than refused — the picker writes `{key}` first and the table/field after, the + wave-18 unconfigured-automation-column precedent exactly; `configured: false` rides the + wire so the surface says "finish setting this up" instead of snapping back to Manual. It + cannot fire while incomplete (the hooks match on the table it does not name), which is the + fail-closed direction. MALFORMED parts (an unknown comparison, a valueless compare, a + condition on a field the trigger does not watch) are still refused with the sentence — + incomplete is a state, wrong is not. + """ + if raw in (None, "", {}): + return (dict(previous) if isinstance(previous, dict) and previous else None), None + if not isinstance(raw, dict): + return None, "the trigger must be an object" + prev = previous if isinstance(previous, dict) else {} + key = _s(raw.get("key") or raw.get("kind") or prev.get("key") or prev.get("kind"), + 30).strip() + if key in TRIGGER_PLANNED: + # Declared on the wire, refused at the door — see the TRIGGER_PLANNED note. The sentence + # says WHY rather than "unknown trigger", because the picker legitimately showed it. + return None, (f"{TRIGGER_LABELS[key]!r} is on the list but not built yet. " + f"it renders so you can see it is coming, and it cannot be saved") + if key not in TRIGGER_KEYS: + return None, (f"{key or 'that trigger'!r} is not one of: " + ", ".join(TRIGGER_KEYS)) + if key == "schedule": + # ⛔ STILL NOT STORED, and the original reasoning holds for THIS key alone: `schedule` + # already owns the cron (`defn['schedule']` = `{cron, enabled}`), so a stored + # `{key:'schedule'}` would be a second copy of that fact, free to disagree with it. + return None, None + if key == "manual": + # ⭐⭐ 2026-08-07 (owner ruling) — **MANUAL IS A REAL, STORED CHOICE NOW.** + # Owner: *"Make it so that when you choose Manual, it IS a manual automation that the user + # can just press Run to make the full flow work."* + # + # ⛔ THIS SPLITS A PAIR THAT SHOULD NEVER HAVE BEEN ONE. The old line refused both keys + # together with one argument — *"storing a no-op trigger would be a second copy of that + # fact"* — and that argument is TRUE OF `schedule` AND FALSE OF `manual`. A schedule has + # another home; **manual has none.** Nothing anywhere recorded "this automation is + # manual", so storing it is not a duplicate: it is the only record there has ever been. + # + # ⚠ WHAT THE CONFLATION COST, measured live: picking Manual wrote nothing, so + # `chosen` (`!!trigger || schedule.enabled`) stayed false, the Builder kept showing the + # "nobody has decided yet" empty state, and Configuration — including the Database picker + # a plain automation cannot do without — never rendered. The owner reported it twice. The + # previous note reasoned that a manual option *"would bounce straight back to this state + # on the next reload"* and concluded the option should be HIDDEN; the honest conclusion + # was that it should be STORED. + # + # ⚠ DELIBERATELY BARE. No `enabled`, no `paused`: a manual trigger cannot be switched off + # (Run now always works, which is the whole of what it means) and a switch that governs + # nothing is worse than no switch. `graph()` keeps this node on the SCHEDULE panel so the + # cron stays reachable — picking Manual says how it fires today, never that it may not be + # scheduled tomorrow. + return {"key": "manual"}, None + out = {"key": key, + "enabled": bool(raw["enabled"]) if "enabled" in raw else + bool(prev.get("enabled", True)), + "paused": bool(raw["paused"]) if "paused" in raw else bool(prev.get("paused"))} + if key in TRIGGER_TABLE_KEYS: + table = _s(raw.get("table") if "table" in raw else prev.get("table"), 60).strip() + if table and not table.startswith(UT_PREFIX): + return None, ("event triggers watch blank databases (ut_*) this wave. " + f"{table!r} is not one") + out["table"] = table + if key == "event_field": + # ⭐ WAVE 24 · C-TRIG LAW 4 (owner item 7) — THE WATCHED FIELD IS GONE. "When a record + # matches conditions" is a CONDITION trigger and nothing else: the field picker made it a + # second, quieter way to express the same narrowing, and the owner asked for one. + # ⚠ MIGRATION, NEVER A REFUSAL (law 6). A stored `field` is simply not read, so it is + # dropped on this definition's next clean — silently, and exactly once, because nothing + # writes the key back. A refusal here would have 400'd the live automations that carry it. + cond, cerr = clean_cond(raw.get("when") if "when" in raw else prev.get("when"), + where="the trigger") + if cerr: + return None, cerr + out["when"] = cond + if key == "record_updated": + # Airtable's shape: watch named fields, or leave the list empty for "any field". Empty + # is the WIDER reading and it is the default there too, so it stays the default here. + watch_raw = raw.get("fields") if "fields" in raw else prev.get("fields") + if watch_raw in (None, ""): + watch = [] + elif not isinstance(watch_raw, list): + return None, "the watched-field list must be a list of field keys" + else: + watch = [_s(f, 80).strip() for f in watch_raw if _s(f, 80).strip()] + if len(watch) > MAX_WATCH_FIELDS: + return None, (f"a record-updated trigger watches at most {MAX_WATCH_FIELDS} " + f"fields. Leave the list empty to watch every field") + out["fields"] = watch + # ⭐ WAVE 24 · C-TRIG LAW 5 (owner item 7) — THE CONDITION IS GONE, and this REMOVES A + # SHIPPED CAPABILITY. Watched `fields` is now the whole of this trigger's configuration: + # "a record was updated" is an event, and asking it to also be a filter was the overlap + # with `event_field` the owner asked to end. Stated loudly in the contract AND here so + # nobody restores it as a bug fix. + # ⚠ Same migration shape as law 4: a stored `when` stops being read, so `_row_gate` + # naturally returns "no gate" for it — the write itself becomes the event — rather than + # this needing a second removal anywhere. + if key == "enters_view": + out["viewId"] = _s(raw.get("viewId") if "viewId" in raw else prev.get("viewId"), + 80).strip() + if key == "form_submitted": + # Blank = any form on that database. Naming one narrows to it, which is what a table + # carrying an intake form AND a correction form needs. + out["formToken"] = _s(raw.get("formToken") if "formToken" in raw + else prev.get("formToken"), 64).strip() + if key == "webhook": + # The token is MINTED here, once, and survives every later patch — rotating it on + # every Save would silently break the external caller the URL was given to. + out["token"] = _s(prev.get("token"), 64) or _secrets_token() + # ⭐ WAVE 24 — DEBT D-41: the request BODY, mapped onto record fields by config. + # ⚠ BOTH HALVES ARE OPTIONAL, and that is what keeps this additive: a webhook with no + # map behaves exactly as it did — it fires the flow and reads nothing — so the live + # webhook automations are untouched. `webhook` deliberately stays OUT of + # `TRIGGER_TABLE_KEYS`: joining it would make a table REQUIRED for `configured`, and + # every existing webhook trigger would go unconfigured and stop firing. + table = _s(raw.get("table") if "table" in raw else prev.get("table"), 60).strip() + if table and not table.startswith(UT_PREFIX): + return None, ("a webhook writes into a blank database (ut_*). " + f"{table!r} is not one") + out["table"] = table + fmap, ferr = clean_body_map(raw.get("fieldMap") if "fieldMap" in raw + else prev.get("fieldMap")) + if ferr: + return None, ferr + out["fieldMap"] = fmap + if key == "email": + out["query"] = _s(raw.get("query") if "query" in raw else prev.get("query"), + 200).strip() or "in:inbox is:unread" + out["configured"] = _trigger_configured(out) + return out, None + + +#: D-41 ceilings. 40 mapped cells is `MAX_ACTION_VALUES` doubled — a webhook payload is somebody +#: else's schema and is legitimately wider than an action's hand-written value list. +MAX_BODY_FIELDS = 40 +MAX_BODY_DEPTH = 5 + + +def clean_body_map(raw): + """D-41: `{"": ""}` for a webhook. Returns `(map, error)`. + + Paths are DOTTED into nested objects (`customer.email`). ⛔ NO ARRAY INDEXING in v1, stated + rather than half-supported: `items.0.sku` would read as working for the first element and + silently write nothing the day a payload arrives with the list empty, which is the shape of + bug this module keeps paying for. A path that resolves to nothing writes nothing. + """ + if raw in (None, ""): + return {}, None + if not isinstance(raw, dict): + return None, "the webhook field map must be an object of {body path: field key}" + if len(raw) > MAX_BODY_FIELDS: + return None, f"a webhook maps at most {MAX_BODY_FIELDS} values onto a record" + out = {} + for path, field in raw.items(): + p = _s(path, 200).strip() + if not p: + return None, "a webhook mapping has an empty body path" + if len(p.split(".")) > MAX_BODY_DEPTH: + return None, (f"{p!r} reaches more than {MAX_BODY_DEPTH} levels into the payload. " + f"map a shallower value") + fk = re.sub(r"[^a-z0-9_]+", "_", _s(field, 60).strip().lower()).strip("_") + if not fk: + return None, f"the value at {p!r} is not mapped to a field" + out[p] = fk[:60] + return out, None + + +def body_value(body, path): + """One dotted path into a decoded JSON body, or None. Scalars only — a mapped value that is + an object or a list answers None rather than a stringified `{...}` in a cell, because the + Row contract is scalar and a serialised dict in a grid cell is unreadable and unfilterable.""" + cur = body + for part in str(path or "").split("."): + if not isinstance(cur, dict): + return None + cur = cur.get(part) + return cur if isinstance(cur, (str, int, float, bool)) else None + + +def webhook_row(rt, defn, body): + """D-41: write ONE record from a webhook payload. Returns `(row_id, mapped_count, note)`. + + ⚠ `note` EXISTS BECAUSE THE CAP WAS SILENT. A table at its row ceiling returned the same + `("", 0)` as "no map configured" and the door answered a cheerful 200 — indistinguishable, + from the only side the caller is on, from a payload whose paths did not resolve. That is the + D-11 class (a table that quietly stops growing), and the caller here is a machine that will + keep posting. The note rides the 200: the flow still fires, and the answer says why no row + was written. + + ⛔ THROUGH THE ENGINE'S OWN WRITER, so it emits no row events — the structural loop + prevention law (A2(2)). A sibling automation watching this table does NOT fire on a + webhook-written row, exactly as it does not fire on a scrape's rows. The webhook's OWN flow + fires, because `hook_fire` fires it explicitly, which is the difference between "this + trigger fired" and "a write happened". + """ + trg = (defn or {}).get("trigger") or {} + table, fmap = str(trg.get("table") or ""), dict(trg.get("fieldMap") or {}) + if not table or not fmap or not isinstance(body, dict): + return "", 0, "" # no map configured — nothing to report + t = ut_get(rt, table) + if t is None: + return "", 0, f"{table} no longer exists, so nothing was written" + values = {} + for path, fkey in fmap.items(): + v = body_value(body, path) + if v is not None: + values[fkey] = _s(v, 500) if isinstance(v, str) else v + if not values: + return "", 0, ("none of the mapped paths resolved to a value in this payload. " + "check the paths against what you are sending") + rows = dict((t.get("rows") or {})) + if len(rows) >= row_cap(table): + return "", 0, (f"{table} is at its {row_cap(table)}-row limit, so no record was " + f"written (the flow still ran)") + rid = str(max([int(r) for r in rows if str(r).isdigit()] or [0]) + 1) + rows[rid] = values + ut_write_rows(rt, table, rows) + return rid, len(values), "" + + +def _trigger_configured(trg, config=None): + """Is this trigger complete enough to fire? One reader, because "configured" is asserted in + three places (the wire, the graph node, the hooks) and three copies of a boolean is how a + surface says "ready" about a trigger the engine skips. + + ⭐ WAVE 24 (A2) — `config` is OPTIONAL and only `ig_profile_match` reads it, because that is + the one trigger whose configuration lives in the DEFINITION's config (the discovery filters) + rather than on the trigger. `clean_trigger` calls this without it and so answers + conservatively (False); `clean_definition` calls it again with the validated config and + refines. Conservative-then-refined is the fail-closed order — the reverse would flash + "ready" on a trigger with nothing to search for. + """ + key = str((trg or {}).get("key") or "") + # ⭐ WAVE 29 — BOTH discovery triggers, and they answer identically: a corpus search with no + # filter is not a search, it is a request for the whole index. Named as a pair rather than + # `or`-ed onto the Instagram line so a third network joins by adding a key, not by editing a + # boolean expression. + if key in ("ig_profile_match", "tiktok_profile_match"): + return bool((config or {}).get("predicates")) + if key in TRIGGER_TABLE_KEYS and not trg.get("table"): + return False + if key == "event_field": + # C-TRIG law 4: the CONDITION is now the whole of it. No condition = "fire on anything, + # ever" — which is not a trigger, it is a description of the table. + # ⚠ STATED CONSEQUENCE OF THE MIGRATION: a live automation that narrowed by FIELD alone + # and carried no condition becomes `configured: false` on its next clean. It stops + # firing, and it SAYS SO — the graph node reads "Finish setting this trigger up before it + # can fire" and `configured` rides the wire. Visibly unfinished, never silently inert. + return bool(trg.get("when")) + if key == "enters_view": + return bool(trg.get("viewId")) + return True + + +def _secrets_token(): + import secrets as _sec + return _sec.token_urlsafe(24) + + +# --- the circuit breaker (A2(2)) — process memory, like _RUNNING: a counter that outlives the +# process would keep punishing an automation for a storm that ended with the restart. +_FIRES = {} +_FIRES_LOCK = threading.Lock() + + +def _breaker_trips(tenant, auto_id, now=None): + now = now if now is not None else time.time() + key = (tenant, str(auto_id)) + with _FIRES_LOCK: + log = [t for t in _FIRES.get(key, []) if now - t < FIRE_WINDOW_SECONDS] + log.append(now) + _FIRES[key] = log + return len(log) > FIRE_LIMIT + + +def _pause_trigger(rt, auto_id, note): + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + d = cur.get(str(auto_id)) + if d is not None: + trg = d.get("trigger") + if isinstance(trg, dict): + trg["paused"] = True + d["statusNote"] = _s(note, 200) + return cur + _store_update(rt, _up, flush="sync") + + +def trigger_fire(rt, tenant, auto_id, log=print, rows=None): + """One trigger firing — breaker first, then the ordinary async run. False when it did not + start (breaker, or already running — both are answers, not errors).""" + if _breaker_trips(tenant, auto_id): + note = (f"auto-paused: more than {FIRE_LIMIT} trigger fires in " + f"{FIRE_WINDOW_SECONDS // 60} minutes. Something is writing this trigger's " + f"subject in a loop") + _pause_trigger(rt, auto_id, note) + log(f"[aios-auto] breaker: {auto_id} {note}") + return False + return run_async(rt, tenant, auto_id, username="automation", log=log, rows=rows) + + +# --- the settle buffer (A2(1)) — per (tenant, automation), coalescing a burst into ONE +# evaluation. With EVENT_SETTLE_SECONDS == 0 the evaluation is INLINE (deterministic for gates). +_SETTLE = {} +_SETTLE_LOCK = threading.Lock() + + +def _settle_buffer(rt, tenant, auto_id, row_id, field="", after=None, log=print): + """Buffer ONE touched row for a coalesced evaluation, carrying the written cell. + + ⚠ WAVE 23 — THE WRITTEN VALUE IS STILL LOAD-BEARING, and an earlier draft of this wave + dropped it on the theory that `_settle_eval` could just read the row. It cannot, and the + reason is worth stating because it is invisible from this file: an ordinary ut cell typed at + the grid door lands in the editor's PER-USER OVERLAY stratum + (`grid_events.overlay_patch` → `table_store.patch_overlay`), not in the `user_tables` + definition rows. Only stage-field writes go through `patch_cells` to the shared rows. So for + the common case the value that just changed exists ONLY in this event, and a definition-row + read sees the pre-write value — the trigger would evaluate stale and never fire. + `_settle_eval` therefore MERGES: the definition row underneath (which is what lets a C4 tree + read the record's other columns) with the written cells on top. + """ + key = (tenant, str(auto_id)) + cell = {str(field): after} if field else {} + if EVENT_SETTLE_SECONDS <= 0: + with _SETTLE_LOCK: + buf = _SETTLE.setdefault(key, {"rows": {}}) + buf["rows"].setdefault(str(row_id), {}).update(cell) + _settle_eval(rt, tenant, auto_id, log=log) + return + with _SETTLE_LOCK: + buf = _SETTLE.setdefault(key, {"rows": {}}) + buf["rows"].setdefault(str(row_id), {}).update(cell) + timer = buf.get("timer") + if timer is not None: + timer.cancel() # the burst continues — push the window out + timer = threading.Timer(EVENT_SETTLE_SECONDS, _settle_eval, + args=(rt, tenant, auto_id), kwargs={"log": log}) + timer.daemon = True + buf["timer"] = timer + timer.start() + + +def view_filter(rt, table_key, view_id): + """`(tree, fields, problem)` for one saved view on a user table — the substrate the + `enters_view` trigger tests membership against (C3-v2 / owner R2). + + Personal strata first (`find_view`), then the shared bucket, because a view somebody shared + is exactly the kind an automation gets pointed at. A view that has been deleted answers a + PROBLEM rather than an empty tree: an empty tree matches everything, so degrading to one + would turn "when a record enters Overdue" into "on every write", which is the widening this + module refuses everywhere else. + """ + key = str(table_key or "") + vid = str(view_id or "").strip() + if not key or not vid: + return None, [], "the trigger names no view" + try: + import core.table_store as table_store + tops = table_store.make(f"{key}_table_workspace", st=rt) + found = tops.find_view(vid) + view = (found[1] if found else None) or tops.shared_view(vid) + except Exception as e: # noqa: BLE001 + return None, [], f"the view could not be read ({type(e).__name__})" + if not isinstance(view, dict): + return None, [], f"view {vid!r} no longer exists on {key}" + cfg = view.get("config") or {} + tree = {"nodes": cfg.get("filters") or [], "conj": cfg.get("filterConj") or "and"} + return tree, list((ut_get(rt, key) or {}).get("fields") or []), "" + + +def _row_gate(rt, defn, trg): + """The MATCH gate for a row-event trigger: `(row -> bool) | None`, plus a problem string. + + None means the trigger has NO match gate — the write itself is the event (wave 22's "the + field changed", and `record_updated` over any field). A problem means the gate cannot be + built, and the caller must then fire NOTHING: a gate we cannot evaluate is not a gate that + passes. + """ + key = trg.get("key") + if key == "enters_view": + tree, fields, problem = view_filter(rt, trg.get("table"), trg.get("viewId")) + if problem: + return None, problem + import harness.filter_eval as filter_eval + return (lambda row: filter_eval.matches(tree, row, fields)), "" + when = trg.get("when") + if when: + return (lambda row: lane_match(when, row)), "" + return None, "" + + +def _settle_eval(rt, tenant, auto_id, log=print): + """Evaluate one settled burst: edge over per-record armed state, flood hold, then fire.""" + with _SETTLE_LOCK: + buf = _SETTLE.pop((tenant, str(auto_id)), None) + written = dict((buf or {}).get("rows") or {}) + touched = list(written) + if not touched: + return + d = all_definitions(rt).get(str(auto_id)) + trg = (d or {}).get("trigger") or {} + if trg.get("key") not in ("event_field", "record_updated", "enters_view") \ + or trg.get("paused") or not trg.get("enabled", True) \ + or not trg.get("configured", True): + return + gate, problem = _row_gate(rt, d, trg) + if problem: + # LOUD, once, and it does not fire. A trigger pointed at a deleted view is a broken + # automation, not a quiet no-op — the note is what the surface shows instead of "On". + if (d.get("statusNote") or "") != problem: + _pause_trigger(rt, auto_id, problem) + log(f"[aios-auto] trigger gate: {auto_id} {problem}") + return + fired = [] + if gate is not None: + rows = (ut_get(rt, trg.get("table") or "") or {}).get("rows") or {} + disarmed = set((d.get("state") or {}).get("eventDisarmed") or []) + nxt = set(disarmed) + for rid in touched: + # The merge (see `_settle_buffer`): the shared definition row underneath so a C4 + # tree can read the record's other columns, the just-written cells on top because + # for an ordinary ut column this event is the ONLY place that value exists yet. + if gate({**(rows.get(rid) or rows.get(str(rid)) or {}), + **(written.get(rid) or {})}): + if rid not in disarmed: + fired.append(rid) # false→true THIS evaluation: the edge + nxt.add(rid) + else: + nxt.discard(rid) # left the state — re-armed (Airtable's rule) + if nxt != disarmed: + set_state(rt, auto_id, {"eventDisarmed": sorted(nxt)[:5000]}) + if not fired: + return + else: + fired = list(touched) # "the field changed" — the burst is the edge + if len(fired) > FLOOD_LIMIT: + _commit_run(rt, auto_id, "partial", + f"the trigger matched {len(fired)} records in one evaluation. More than " + f"the {FLOOD_LIMIT}-record flood hold, so nothing ran. Press Run now to " + f"process them deliberately", {"held": len(fired)}, True) + return + trigger_fire(rt, tenant, auto_id, log=log, rows=fired) + + +def _seed_event_state(rt, defn): + """A2(1)'s enable rule: records ALREADY matching when the trigger is set start DISARMED, so + turning the trigger on fires nothing — the first fire needs a real false→true transition. + Evaluated over the definition rows (the machine-written truth this trigger class watches). + + Wave 23: one seeder for all three gated triggers, reading the SAME `_row_gate` the evaluation + reads. Two implementations of "does this row match" is how a seed disagrees with the edge it + is supposed to arm, and the symptom would be a flood of fires the moment somebody enables it. + """ + trg = (defn or {}).get("trigger") or {} + if trg.get("key") not in ("event_field", "record_updated", "enters_view") \ + or not trg.get("configured"): + return + gate, problem = _row_gate(rt, defn, trg) + if gate is None or problem: + return # no match gate ⇒ nothing to arm; a broken gate seeds nothing + rows = (ut_get(rt, trg.get("table") or "") or {}).get("rows") or {} + matching = sorted(str(rid) for rid, row in rows.items() if gate(row or {})) + set_state(rt, defn.get("id"), {"eventDisarmed": matching[:5000]}) + + +def grid_hook(evt): + """THE listener the human write doors emit into (registered onto + `core.user_tables.ROW_HOOKS` by `routes_automation` at import — the one place that may + import both sides). Never raises into a write path; a broken trigger listener must not + break typing into a cell.""" + try: + st = evt.get("st") + if st is None: + return + tenant = str(getattr(st, "key", "") or "royal-imports") + table = str(evt.get("table") or "") + kind = str(evt.get("type") or "") + # ⭐⭐ WAVE 31 · T35 (D-134) — `cached=True`, and this is the ONLY caller that passes it. + # This function runs once per ROW EVENT, so a 20,000-row import used to perform 20,000 + # whole-document deep copies of the automations bucket, under the store lock, to re-read a + # trigger set that had not changed. See `all_definitions` for why the memo is safe here + # and nowhere else. + defs = all_definitions(st, cached=True) + field = str(evt.get("field") or "") + rid = str(evt.get("rowId") or "") + for aid, d in defs.items(): + trg = (d or {}).get("trigger") or {} + key = str(trg.get("key") or "") + if trg.get("paused") or not trg.get("enabled", True) \ + or not trg.get("configured", True) \ + or key not in TRIGGER_ROW_KEYS \ + or str(trg.get("table") or "") != table: + continue + if kind == "record_created" and key == "record_created": + hw = _ig_int((d.get("state") or {}).get("rcHighwater")) or 0 + ridn = int(rid) if rid.isdigit() else None + if ridn is None or ridn <= hw: + continue # once per record EVER (A2(4)) — undo-proof + set_state(st, aid, {"rcHighwater": ridn}) + # ⛔⛔ W31-T35 — MIRROR THE WRITE INTO THE MEMO'S OWN COPY, IN THE SAME STATEMENT. + # This is the hazard a definitions memo creates and the reason D-134 is not a + # one-line change: this branch READS `rcHighwater` and WRITES it, so within one + # import burst the second row would compare against the highwater the FIRST row + # set — and read the pre-write value out of the memo, fire again, and break + # A2(4)'s *"once per record EVER — undo-proof"*. `set_state` goes through + # `_store_update`, which drops the memo, but `defs` is the object already in hand + # for the rest of THIS event; keeping the two in step is what makes the memo safe + # rather than merely fast [[read-path-cannot-witness-write-path]]. + (d.setdefault("state", {}))["rcHighwater"] = ridn + trigger_fire(st, tenant, aid, rows=[rid]) + elif kind == "event_field" and key == "event_field": + # ⭐ WAVE 24 · law 4 — the per-FIELD narrowing is gone with the stored key. Every + # write on the watched table settles, and the CONDITION decides whether it fires + # (`_row_gate`). The old `not trg.get("field") or str(...) == field` test would + # now always take its first arm anyway; leaving a read of a key the validator no + # longer writes is the drift seat this migration exists to close. + _settle_buffer(st, tenant, aid, rid, field, evt.get("after")) + elif kind == "event_field" and key == "record_updated" and ( + not (trg.get("fields") or []) or field in (trg.get("fields") or [])): + _settle_buffer(st, tenant, aid, rid, field, evt.get("after")) + elif key == "enters_view": + # BOTH kinds feed it: a row can enter a view by being edited into its filter or + # by being CREATED already inside it. Listening only to edits would silently miss + # every new record — the half of the definition a reader assumes is covered. + _settle_buffer(st, tenant, aid, rid, field, evt.get("after")) + except Exception as e: # noqa: BLE001 + print(f"[aios-auto] trigger hook failed: {type(e).__name__}: {e}") + + +def form_fired(rt, table_key, row_id, values=None, form_token=""): + """⭐ THE FROZEN SIGNATURE session D calls from the public form door (contract C9/W23-W7). + + One submitted form row → every `form_submitted` automation watching that database fires. + Returns the list of automation ids that started, so the door can log what it set off (and so + the gate can assert it, rather than asserting a side effect nobody can see). + + ⛔ THIS IS A HUMAN DOOR, deliberately: an anonymous submission is a person filling in a form, + so it fires triggers exactly like typing into a cell does. The loop-prevention law is not + weakened by that — the engine's own writers still never reach here (only `routes_forms` calls + it), so an automation cannot create a form row and re-fire itself. + + `values` is accepted and unused today: the row is already written when this is called, and + the flow reads it from the table. It stays in the signature because the caller HAS it and a + later refire policy ("only when field X was submitted") needs it — a parameter added later + would mean changing D's call site in a wave that does not own it. + """ + started, table = [], str(table_key or "") + if not table: + return started + tenant = str(getattr(rt, "key", "") or "royal-imports") + for aid, d in all_definitions(rt).items(): + trg = (d or {}).get("trigger") or {} + if trg.get("key") != "form_submitted" or trg.get("paused") \ + or not trg.get("enabled", True) or not trg.get("configured", True): + continue + if str(trg.get("table") or "") != table: + continue + want = str(trg.get("formToken") or "") + if want and not hmac.compare_digest(want, str(form_token or "")): + continue # this automation watches a DIFFERENT form on that table + if trigger_fire(rt, tenant, aid, rows=[str(row_id)] if row_id else None): + started.append(aid) + return started + + +def hook_fire(rt, tenant, auto_id, token, body=None): + """The webhook trigger's decision, separated from FastAPI so the gate can drive it. + Returns `(status, payload)` — 404 unknown, 403 wrong/missing token or wrong trigger kind, + 409 already running, 200 started. + + ⭐ WAVE 24 (D-41): `body` is the decoded JSON payload, or None when the caller sent none or + sent something that is not JSON. It is written onto a record BEFORE the flow fires — the + flow's actions walk the record the webhook just created, which is the whole point of mapping + it. `body=None` is the pre-wave behaviour exactly, so an existing caller is unaffected. + """ + import hmac as _hmac + defn = all_definitions(rt).get(str(auto_id)) + if defn is None: + return 404, {"error": "unknown_automation"} + trg = defn.get("trigger") or {} + want = str(trg.get("token") or "") + if trg.get("key") != "webhook" or not want: + return 403, {"error": "no_webhook", "message": + "this automation has no webhook trigger"} + if trg.get("paused") or not trg.get("enabled", True): + return 403, {"error": "webhook_off", "message": "the webhook trigger is turned off"} + if not _hmac.compare_digest(want, str(token or "")): + return 403, {"error": "bad_token", "message": "that token is not valid"} + # D-41: map the payload onto a record FIRST, so the flow that fires next walks it. + row_id, mapped, note = webhook_row(rt, defn, body) + started = trigger_fire(rt, tenant, auto_id, rows=[row_id] if row_id else None) + out = {"started": bool(started), "at": _iso(), + # Answered even when zero, so a caller wiring a map up can see whether their paths + # resolved. Silence here would make "my JSON is not landing" undebuggable from the + # outside, which is the only side the caller is on. + "rowId": row_id or None, "mapped": mapped} + if note: + out["note"] = note # a 200 that wrote no row SAYS which reason it was + return 200, out + + +def email_poll(rt, tenant, auto_id, defn, log=print, _list=None, _read=None): + """The email trigger's tick half (C3): poll the CREATOR's Gmail through C5's seam, write a + row per NEW matching message, fire the flow. `_list`/`_read` are injection points so the + gate drives this without a network; production leaves them None. + + Fail-closed and QUIET when unconnected: the statusNote says so ONCE (not a run entry per + tick — 96 identical failures a day is a klaxon, not a status). Bounded everywhere: at most + `EMAIL_MAX_PER_POLL` new messages per tick, the flood hold above that, one coalesced write. + """ + trg = defn.get("trigger") or {} + if trg.get("key") != "email" or trg.get("paused") or not trg.get("enabled", True): + return None + import oauth_connect + creator = str(defn.get("createdBy") or "").strip() or "admin" + token, err = oauth_connect.google_creds(rt, creator) + if err: + if (defn.get("statusNote") or "") != err: + def _note(cur): + cur = cur if isinstance(cur, dict) else {} + dd = cur.get(str(auto_id)) + if dd is not None: + dd["statusNote"] = _s(err, 200) + return cur + _store_update(rt, _note, flush="sync") + return None + lister = _list or (lambda q, n: oauth_connect.gmail_list(token, q, n)) + reader = _read or (lambda mid: oauth_connect.gmail_message(token, mid)) + ids, lerr = lister(trg.get("query") or "", EMAIL_MAX_PER_POLL + FLOOD_LIMIT) + if lerr: + return _commit_run(rt, auto_id, "partial", f"the Gmail poll did not answer. {lerr}", + {}, True) + seen = set((defn.get("state") or {}).get("emailSeen") or []) + fresh = [m for m in ids if m not in seen] + if not fresh: + return None # nothing new is not a run — no history spam + if len(fresh) > FLOOD_LIMIT: + return _commit_run(rt, auto_id, "partial", + f"{len(fresh)} new emails matched in one poll. More than the " + f"{FLOOD_LIMIT}-record flood hold, so nothing was written. Narrow " + f"the query, or press Run now after adjusting it", + {"held": len(fresh)}, True) + fresh = fresh[:EMAIL_MAX_PER_POLL] + rows_in, notes = [], [] + for mid in fresh: + row, rerr = reader(mid) + if row: + rows_in.append(row) + elif rerr: + notes.append(rerr) + table_key = (defn.get("config") or {}).get("targetTable") or "" + if not table_key: + return _commit_run(rt, auto_id, "error", + "the email trigger has nowhere to write. The automation names no " + "target database", {}, False) + ut_ensure(rt, (defn.get("config") or {}).get("targetLabel") or defn.get("name") or "Inbox", + EMAIL_FIELDS, username=str(defn.get("createdBy") or "automation"), key=table_key) + existing = dict((ut_get(rt, table_key) or {}).get("rows") or {}) + merged, counts = upsert_rows(existing, rows_in, "email_id", cap=row_cap(table_key)) + ut_write_rows(rt, table_key, merged) + new_seen = (list(seen) + fresh)[-EMAIL_SEEN_CAP:] + set_state(rt, auto_id, {"emailSeen": new_seen}) + summary = (f"{len(fresh)} new email{'' if len(fresh) == 1 else 's'} matched. " + f"{counts['inserted']} row{'' if counts['inserted'] == 1 else 's'} written") + if notes: + summary += f". {notes[0][:100]}" + entry = _commit_run(rt, auto_id, "ok" if not notes else "partial", summary, counts, + True, affected=list(merged)[:200]) + trigger_fire(rt, tenant, auto_id, log=log) + return entry + + +def compose_sentence(defn): + """The one-sentence server-composed summary (airtable-brief rec 7): rendered from the + definition so it cannot lie about what runs.""" + cfg = defn.get("config") or {} + trg = defn.get("trigger") or {} + sched = defn.get("schedule") or {} + kind = defn.get("kind") + if trg.get("key") == "event_field": + # Law 4: no watched field any more, so the sentence stops naming one. It said + # "When None changes on ut_x" the moment the key stopped being stored. + head = f"When a record in {trg.get('table')} matches conditions" + elif trg.get("key") == "record_updated": + head = f"When a record in {trg.get('table')} is updated" + elif trg.get("key") == "ig_profile_match": + head = "When an Instagram profile fits the criteria" + elif trg.get("key") == "tiktok_profile_match": + head = "When a TikTok profile fits the criteria" + elif trg.get("key") == "record_created": + head = f"When a record is created in {trg.get('table')}" + elif trg.get("key") == "webhook": + head = "When the webhook is called" + elif trg.get("key") == "email": + head = f"When an email matches {trg.get('query')}" + elif sched.get("enabled"): + head = f"{_cron_label(sched.get('cron'))}" + else: + head = "When you press Run now" + if kind == "scrape_db": + host = urlparse(str(cfg.get("url") or "")).hostname or "the page" + body = f"read {host} and upsert rows into {cfg.get('targetTable') or 'a new database'}" + elif kind == "field_instagram": + # ⚠ NO RUNG CLAUSE (R5). There is one way to capture a profile now, so ", exact counts + # first" / ", anonymous only" described a choice that no longer exists. + body = f"capture Instagram profiles for {cfg.get('targetTable') or 'the database'}" + elif kind in DISCOVERY_KINDS: + # ⭐⭐ WAVE 30 · T05 — the arm covers both kinds, and the network is NAMED from the kind + # rather than hard-coded into the sentence. Instagram-only, a TikTok search fell into the + # `plain` branch below and introduced itself as *"do nothing yet — this automation has no + # actions"*: a flatly false sentence, on the one surface whose stated promise is that it + # cannot lie about what runs. (The `plain` branch's own comment records the mirror-image + # incident — it USED to be this arm, and described every plain automation as an Instagram + # search for 0 profiles. The same two branches have now mis-described each other's + # automations in both directions, which is why neither may be a fallthrough.) + _dplatform, _dtable, _, _ = discovery_facts(kind) + body = (f"search {_dplatform} for up to {cfg.get('recordsLimit') or 0} profiles into " + f"{cfg.get('targetTable') or _dtable}") + else: + # ⭐ WAVE 24 — `plain` describes itself from its FLOW, because the flow is all it has. + # ⛔ THIS BRANCH USED TO BE `discover_instagram`'s, so before the arm above existed every + # plain automation would have introduced itself as "search Instagram for up to 0 profiles + # into ut_ig_candidates" — a sentence composed from a definition that says none of it, + # on the one surface whose whole promise is that it "cannot lie about what runs". + n = sum(1 for _ in walk_actions((defn.get("flow") or {}).get("actions"))) + tbl = cfg.get("targetTable") or trg.get("table") or "" + body = ((f"run {n} action{'' if n == 1 else 's'}" + (f" on {tbl}" if tbl else "")) + if n else "do nothing yet. This automation has no actions") + lanes = cfg.get("lanes") or [] + tail = f", then route each record across {len(lanes)} lanes" if lanes else "" + return f"{head}, {body}{tail}." + + +def run_now(rt, tenant, auto_id, username="automation", log=print, rows=None): + """Execute one automation SYNCHRONOUSLY. The route wraps this in a thread; the tick calls it + directly. Returns the run entry, or None when it was already running (the 409).""" + defn = all_definitions(rt).get(str(auto_id)) + if defn is None: + return None + # ⛔⛔ WAVE 32 · T45 (owner item 10) — AN UNCONFIGURED ACTION BLOCKS THE RUN, HERE, WHERE EVERY + # DOOR PASSES. The route checks too so a person gets a 400 rather than a silent no-op, but the + # tick and the webhook do not go through the route; a client-only block is not a block (D-112). + # ⚠ BEFORE `_claim`, deliberately: claiming and then refusing would leave the automation + # marked running until the release, i.e. a refusal that also produces a phantom 409 for the + # next honest attempt. + _refusal = run_refusal(defn) + if _refusal: + log(f"[aios-auto] refused: {_refusal}") + return None + if not _claim(tenant, auto_id): + return None + # ⛔ THE TABLES AN AUTOMATION MAKES BELONG TO THE AUTOMATION'S CREATOR, not to whoever + # happened to press Run — and above all not to the scheduler, which is not a person and + # cannot own anything (see `ut_ensure`). Without this the owner of a database was decided by + # whether a human or a cron got to the first run first. + owner = str(defn.get("createdBy") or "").strip() + if owner and username in MACHINE_OWNERS: + username = owner + try: + _step(tenant, auto_id, "running") + # A deferred metric snapshot is paid work already in Bright Data's queue. Collect it + # first and do not start another profile scrape while it is outstanding: re-running the + # action would buy duplicate engagement reads and reintroduce the timeout this handoff + # exists to remove. The normal scheduler calls this path too via `pending_collect_ids`. + # ⭐ 2026-08-09 — THE PROFILE HANDOFF IS COLLECTED FIRST, for the same reason and one + # rung earlier: a profile snapshot the vendor is still building is paid work, and + # starting a fresh scrape for the same handle would buy the identical row a second time. + # Ahead of the metric collector because the profile IS the thing the run was asked for; + # the engagement batches hang off it. + if _pending_profile_tasks(defn): + state, summary, counts, affected, steps = collect_pending_profile_snapshots( + rt, defn, username=username, log=log, + step=lambda text: _step(tenant, auto_id, text)) + return _commit_run(rt, auto_id, state, summary, + {k: v for k, v in counts.items() if k != RUN_NOTES_KEY}, + state != "error", affected, steps, + notes=counts.get(RUN_NOTES_KEY)) + if _pending_metric_tasks(defn): + state, summary, counts, affected, steps = collect_pending_metric_snapshots( + rt, defn, username=username, log=log, + step=lambda text: _step(tenant, auto_id, text)) + return _commit_run(rt, auto_id, state, summary, + {k: v for k, v in counts.items() if k != RUN_NOTES_KEY}, + state != "error", affected, steps, + notes=counts.get(RUN_NOTES_KEY)) + runner = RUNNERS.get(defn.get("kind")) + if runner is None: + return _commit_run(rt, auto_id, "error", + f"unknown automation kind {defn.get('kind')!r}", {}, False) + try: + # ⭐ WAVE 24 (item 6, on D's measurement) — THE LIVE STEP, closed over this run. + # `status.step` was already on the wire and D's half renders it; measured against the + # code, it was set exactly ONCE ("running") and never again, so the word would have + # been identical whether a run was mid-vendor-wait or genuinely hung. Rendering a + # constant as a progress indicator is worse than rendering nothing: it looks like an + # answer. The runners move it now, and the 120 s Bright Data wait counts out loud. + state, summary, counts, affected, steps = runner( + rt, defn, username=username, log=log, + step=lambda text: _step(tenant, auto_id, text), rows=rows) + except Refused as e: + return _commit_run(rt, auto_id, "error", f"refused: {e}", {}, False) + except Exception as e: # noqa: BLE001 + log(f"[aios-auto] run {auto_id} failed: {type(e).__name__}: {e}") + return _commit_run(rt, auto_id, "error", + f"{type(e).__name__}: {str(e)[:200]}", {}, False) + # ⭐ WAVE 23 (C4/C5) — THE FLOW RUNS HERE, after the machine steps and before the run is + # committed, over the records this run actually touched. ONE call site rather than three + # inside the runners: every kind gets actions and endings for free, and a fourth runner + # cannot forget to opt in. + # + # ⚠ Its absence was the wave's most expensive near-miss: actions were stored, validated, + # wired to the wire and covered by twelve gate checks that all called `apply_actions` + # DIRECTLY — so the whole feature was green and unreachable. A person would have built a + # flow, pressed Run now, and watched nothing happen. The gate now drives `run_now`. + # + # A failing action must not fail the RUN: the machine steps already wrote their rows and + # reporting that as an error would misdescribe what happened. It degrades to `partial` + # with the reason in the summary — the cap_note discipline. + # ⚠ BOUND BEFORE THE `try`. The `except` below falls through to the same `_commit_run`, + # which now reads this name — an assignment only on the success path would turn any + # action failure into a NameError inside the handler that exists to prevent exactly that. + # ⚠ THE RUNNER PRODUCES NOTES TOO, and its are the ones that survive a run which walked + # NOTHING — the branch where every candidate was a known-dead handle, i.e. exactly the + # run a person stares at wondering why the automation stopped doing anything. + run_notes = list((counts or {}).pop(RUN_NOTES_KEY, None) or []) + try: + a_counts = apply_actions(rt, defn, _flow_table(defn), affected or [], + username=username, log=log, + step=lambda text: _step(tenant, auto_id, text)) + # ⭐⭐ D-103 — POPPED BEFORE THE MERGE. The per-record reasons ride inside `counts` so + # the runner contract keeps its shape, and they must leave before the merge or they + # would be a "count" everywhere downstream. + run_notes += list(a_counts.pop(RUN_NOTES_KEY, None) or []) + counts = {**(counts or {}), **{k: v for k, v in a_counts.items() if v}} + if a_counts.get("enrichMetricBatchesPending"): + batches = int(a_counts["enrichMetricBatchesPending"]) + state = "partial" if state != "error" else state + summary += (f". {batches} post-engagement batch" + f"{'' if batches == 1 else 'es'} still building; Views and other " + "metrics will be collected automatically without another paid read") + if a_counts.get("enrichUnbound"): + # ⛔ D-79(2): AND THE FIX GOES IN THE SUMMARY, not only in the log. The run is + # `partial` because it genuinely did part of its job — it walked the records — and + # the sentence names the ONE thing that has to change, in the two places a person + # can change it. The old behaviour was `ok` with an empty table. + state = "partial" if state != "error" else state + summary += (". The Instagram step did not run: this database has no profile " + "column. Name one on the step, or mark a text column as the " + "Instagram profile" + + _unbound_hint(rt, _flow_table(defn))) + if a_counts.get("ttEnrichUnbound"): + # ⛔ WAVE 30 · T08 — ITS OWN SENTENCE, not the one above with a word swapped by a + # variable. A flow may carry BOTH steps, and the fix a person has to apply is + # per-column: naming an Instagram profile column does nothing for a TikTok step, + # so a single sentence covering "the enrich step" would send them to the wrong + # place half the time. Both may appear on one run, which is correct. + state = "partial" if state != "error" else state + summary += (". The TikTok step did not run: this database has no TikTok profile " + "column. Name one on the step, or mark a text column as a TikTok " + "profile" + + _unbound_hint(rt, _flow_table(defn))) + if a_counts.get("ttEnrichBlocked"): + state = "partial" if state != "error" else state + tt_note = next((n for n in run_notes if "(TikTok): " in n), "") + summary += (f". {int(a_counts['ttEnrichBlocked'])} TikTok profile read(s) were " + "blocked" + (f". {_s(tt_note, 220)}" if tt_note else "")) + if a_counts.get("enrichProfileBatchesPending"): + # ⭐ The paid profile the vendor is still building. Said out loud so a run that + # looks like a failure is read as the handoff it is — the tick finishes it. + n = int(a_counts["enrichProfileBatchesPending"]) + state = "partial" if state != "error" else state + summary += (f". {n} profile read{'' if n == 1 else 's'} took longer than the " + "wait allows and will be collected automatically, at no extra cost") + if a_counts.get("enrichBlocked"): + state = "partial" if state != "error" else state + # ⭐⭐ D-103 — THE REASON IS IN THE SENTENCE, not only behind a click. "1 profile + # read(s) were blocked" is the exact string the owner read three mornings running + # before asking "wtf is going on"; it names a quantity and withholds the one + # thing that would let anybody act. The first note is the vendor's own words. + # ⚠ ...and the Instagram selector SKIPS the tagged TikTok lines for the same + # reason. Two sentences quoting each other's vendor reason is worse than one. + blocked_note = next((n for n in run_notes + if ": " in n and "(TikTok): " not in n), "") + summary += (f". {int(a_counts['enrichBlocked'])} profile read(s) were blocked" + + (f". {_s(blocked_note, 220)}" if blocked_note else "")) + # ⭐ WAVE 25 · C5 — A FULL TARGET IS A `partial` RUN THAT SAYS SO. D-11 made this the + # law for the runners' OWN writes (`cap_note`), and `create_record` never joined: it + # logged the cap and rolled up `ok`, so a flow that had silently stopped writing + # looked exactly like one that had nothing to write. Same rule, same sentence shape. + if a_counts.get("createCapped"): + state = "partial" if state != "error" else state + summary = (f"{summary}. {a_counts['createCapped']} row(s) NOT created: a target " + f"database is at its row cap") + except Exception as e: # noqa: BLE001 + log(f"[aios-auto] actions on {auto_id} failed: {type(e).__name__}: {e}") + state = "partial" if state != "error" else state + summary = f"{summary}. The actions did not finish ({type(e).__name__})" + return _commit_run(rt, auto_id, state, summary, counts, state != "error", affected, + steps, notes=run_notes) + finally: + _release(tenant, auto_id) + + def run_async(rt, tenant, auto_id, username="automation", log=print, rows=None): """Dispatch one run without keeping production work resident in the web process. @@ -14513,95 +14885,95 @@ def run_async(rt, tenant, auto_id, username="automation", log=print, rows=None): except ImportError: pass th = threading.Thread(target=run_now, args=(rt, tenant, auto_id, username, log, rows), - daemon=True, name=f"automation-{tenant}-{auto_id}") - th.start() - return True - - -# --------------------------------------------------------------------------------------------- -# THE TICK + the in-process scheduler -# --------------------------------------------------------------------------------------------- - + daemon=True, name=f"automation-{tenant}-{auto_id}") + th.start() + return True + + +# --------------------------------------------------------------------------------------------- +# THE TICK + the in-process scheduler +# --------------------------------------------------------------------------------------------- + def pending_collect_ids(rt, definitions=None): - """⭐ 2026-08-06 — automations holding a snapshot the vendor is still building. - - ⛔ THE DEFECT THIS CLOSES, reported live: *"why is the SMALL 10 records test search taking - forever, its not populating"*. A corpus search takes ~20 minutes, so the run hands off and - stores `pendingSnapshot` with the summary *"The next run picks up the results"* — which is - true only if there IS a next run. The owner's automation is MANUAL (`schedule.enabled` false), - so `due_ids` never returns it, nothing ever collected it, and the results the account had - already been charged for sat ready at the vendor forever. The sentence was not wrong; it was - describing a run nobody had scheduled. - - A pending snapshot is unfinished work the tenant has already paid for, so the tick finishes it - regardless of schedule. Independent of `due_ids` on purpose — a schedule says *start something - new*, this says *collect what is already running*, and folding the second into the first would - make an unscheduled automation's paid result depend on someone remembering to press a button. - """ - out = [] + """⭐ 2026-08-06 — automations holding a snapshot the vendor is still building. + + ⛔ THE DEFECT THIS CLOSES, reported live: *"why is the SMALL 10 records test search taking + forever, its not populating"*. A corpus search takes ~20 minutes, so the run hands off and + stores `pendingSnapshot` with the summary *"The next run picks up the results"* — which is + true only if there IS a next run. The owner's automation is MANUAL (`schedule.enabled` false), + so `due_ids` never returns it, nothing ever collected it, and the results the account had + already been charged for sat ready at the vendor forever. The sentence was not wrong; it was + describing a run nobody had scheduled. + + A pending snapshot is unfinished work the tenant has already paid for, so the tick finishes it + regardless of schedule. Independent of `due_ids` on purpose — a schedule says *start something + new*, this says *collect what is already running*, and folding the second into the first would + make an unscheduled automation's paid result depend on someone remembering to press a button. + """ + out = [] definitions = all_definitions(rt) if definitions is None else definitions for aid, d in definitions.items(): - if not isinstance(d, dict): - continue - if (d.get("trigger") or {}).get("paused"): - continue - # ⭐⭐ WAVE 30 · T05 — BOTH discovery kinds, and this one is a MONEY defect rather than a - # cosmetic one. It tested `== "discover_instagram"`, so a TikTok corpus search stored its - # `pendingSnapshot`, told the person *"The next run picks up the results"*, and was then - # never returned by this function — the tick collected nothing, forever. That is EXACTLY - # the live incident quoted in the docstring above (*"why is the SMALL 10 records test - # search taking forever, its not populating"*), reproduced for the second platform by the - # wave that added it: a result the tenant has already been charged for, stranded. - # ⚠ NOT ON THE SCOUT'S LIST OF FIVE. Found by reading this function for a different - # ticket, which is the argument for `DISCOVERY_KINDS` in one line — the sites that test a - # kind string are not enumerable by memory, and this one is three thousand lines from the - # others. Pressing Run again does still collect (the runner's own branch reads the same - # field), so the money was recoverable BY HAND and only ever silently lost on a schedule. - pending_discovery = (d.get("kind") in DISCOVERY_KINDS - and str((d.get("state") or {}).get("pendingSnapshot") or "").strip()) - # ⭐ 2026-08-09 — PROFILE handoffs join the other two. Without this line the profile - # deferral would be stored and never collected, which is the same defect it fixes wearing - # a queue: `due_ids` only returns SCHEDULED automations, and the automation this was - # measured on is `trigger: manual`. A capability written down but never walked is what - # this whole change is about, so it must not be reintroduced one function later. - if pending_discovery or _pending_metric_tasks(d) or _pending_profile_tasks(d): - out.append(aid) - return sorted(out) - - + if not isinstance(d, dict): + continue + if (d.get("trigger") or {}).get("paused"): + continue + # ⭐⭐ WAVE 30 · T05 — BOTH discovery kinds, and this one is a MONEY defect rather than a + # cosmetic one. It tested `== "discover_instagram"`, so a TikTok corpus search stored its + # `pendingSnapshot`, told the person *"The next run picks up the results"*, and was then + # never returned by this function — the tick collected nothing, forever. That is EXACTLY + # the live incident quoted in the docstring above (*"why is the SMALL 10 records test + # search taking forever, its not populating"*), reproduced for the second platform by the + # wave that added it: a result the tenant has already been charged for, stranded. + # ⚠ NOT ON THE SCOUT'S LIST OF FIVE. Found by reading this function for a different + # ticket, which is the argument for `DISCOVERY_KINDS` in one line — the sites that test a + # kind string are not enumerable by memory, and this one is three thousand lines from the + # others. Pressing Run again does still collect (the runner's own branch reads the same + # field), so the money was recoverable BY HAND and only ever silently lost on a schedule. + pending_discovery = (d.get("kind") in DISCOVERY_KINDS + and str((d.get("state") or {}).get("pendingSnapshot") or "").strip()) + # ⭐ 2026-08-09 — PROFILE handoffs join the other two. Without this line the profile + # deferral would be stored and never collected, which is the same defect it fixes wearing + # a queue: `due_ids` only returns SCHEDULED automations, and the automation this was + # measured on is `trigger: manual`. A capability written down but never walked is what + # this whole change is about, so it must not be reintroduced one function later. + if pending_discovery or _pending_metric_tasks(d) or _pending_profile_tasks(d): + out.append(aid) + return sorted(out) + + def due_ids(rt, now=None, definitions=None): """Which of this tenant's automations a tick at `now` should start. Pure over the store.""" definitions = all_definitions(rt) if definitions is None else definitions return sorted(aid for aid, d in definitions.items() if is_due(d, now)) - - + + def tick(rt, tenant, now=None, log=print): - """Fire every due automation for ONE tenant + poll every email trigger (C3). Returns the - ids started. The email polls are bounded and fail-quiet per automation — one broken - mailbox connection must not stop the tenant's schedules.""" + """Fire every due automation for ONE tenant + poll every email trigger (C3). Returns the + ids started. The email polls are bounded and fail-quiet per automation — one broken + mailbox connection must not stop the tenant's schedules.""" started = [] definitions = tick_definitions(rt) - # ⭐ COLLECT-FIRST (2026-08-06). A snapshot the vendor has finished building is a result the - # tenant has already been charged for; it is collected whether or not this automation is on a - # schedule. `_claim` makes the union safe — an id in both lists starts once. + # ⭐ COLLECT-FIRST (2026-08-06). A snapshot the vendor has finished building is a result the + # tenant has already been charged for; it is collected whether or not this automation is on a + # schedule. `_claim` makes the union safe — an id in both lists starts once. for aid in dict.fromkeys(list(pending_collect_ids(rt, definitions=definitions)) + list(due_ids(rt, now, definitions=definitions))): - if run_async(rt, tenant, aid, username="scheduler", log=log): - started.append(aid) + if run_async(rt, tenant, aid, username="scheduler", log=log): + started.append(aid) for aid, d in definitions.items(): - if (d.get("trigger") or {}).get("key") == "email": - try: - if email_poll(rt, tenant, aid, d, log=log) is not None: - started.append(aid) - except Exception as e: # noqa: BLE001 - log(f"[aios-auto] email poll {tenant}/{aid} failed: {type(e).__name__}: {e}") + if (d.get("trigger") or {}).get("key") == "email": + try: + if email_poll(rt, tenant, aid, d, log=log) is not None: + started.append(aid) + except Exception as e: # noqa: BLE001 + log(f"[aios-auto] email poll {tenant}/{aid} failed: {type(e).__name__}: {e}") # Derived cells are maintenance, not part of the scheduler's HTTP acknowledgement. The # background lane reads the large workspace at most once per UTC day, plus once after a real # automation run marks it dirty. An idle wake-up therefore stays rows-free, and Lambda never # times out waiting for a 40 MB tenant document to cross the public network. _start_derived_refresh(rt, tenant, now=now, log=log) - if started: - log(f"[aios-auto] tick {tenant}: started {', '.join(started)}") + if started: + log(f"[aios-auto] tick {tenant}: started {', '.join(started)}") return started @@ -14788,8 +15160,8 @@ def scheduler_status(): def tick_all(now=None, log=print): - """Every registered tenant. Fail-quiet per tenant: one tenant's broken store must not stop - the others' schedules.""" + """Every registered tenant. Fail-quiet per tenant: one tenant's broken store must not stop + the others' schedules.""" from harness import runtime as _rt out = {} slugs = _scheduler_tenants(_rt, now=now) @@ -14807,46 +15179,46 @@ def tick_all(now=None, log=print): _SCHEDULER_METRICS["lastTickAt"] = _iso() _SCHEDULER_METRICS["lastTenantCount"] = len(slugs) return out - - -#: How often the in-process scheduler wakes. A minute is the cron resolution; anything finer -#: would be a busy loop against a vocabulary that cannot express it. -TICK_SECONDS = int(os.environ.get("AIOS_AUTOMATION_TICK_SECONDS") or 60) -_SCHEDULER = [None] - - -def scheduler_loop(log=print): - """The resync-daemon pattern (`api/main.py:313`): sleep FIRST, then work. - - Sleeping first is deliberate and load-bearing for the gate battery — `verify_api` and - `verify_seam` import this module through `main.py`, run against fake stores in seconds and - exit. A loop that ticked on entry would fire inside them. - """ - while True: - time.sleep(TICK_SECONDS) - try: - tick_all(log=log) - except Exception as e: # noqa: BLE001 - log(f"[aios-auto] scheduler tick failed: {type(e).__name__}: {e}") - - + + +#: How often the in-process scheduler wakes. A minute is the cron resolution; anything finer +#: would be a busy loop against a vocabulary that cannot express it. +TICK_SECONDS = int(os.environ.get("AIOS_AUTOMATION_TICK_SECONDS") or 60) +_SCHEDULER = [None] + + +def scheduler_loop(log=print): + """The resync-daemon pattern (`api/main.py:313`): sleep FIRST, then work. + + Sleeping first is deliberate and load-bearing for the gate battery — `verify_api` and + `verify_seam` import this module through `main.py`, run against fake stores in seconds and + exit. A loop that ticked on entry would fire inside them. + """ + while True: + time.sleep(TICK_SECONDS) + try: + tick_all(log=log) + except Exception as e: # noqa: BLE001 + log(f"[aios-auto] scheduler tick failed: {type(e).__name__}: {e}") + + def start_scheduler(log=print): - """Start the loop once per process. **OPT-IN: `AIOS_AUTOMATIONS=1`.** - - ⚠ AMENDED 2026-08-03 (D), and the reason is a measurement rather than a preference. The wave - brief specified this thread DEFAULT-ON for the API. Then `verify_api.py` was timed: **196 - seconds**, i.e. more than three 60 s tick intervals. A default-on scheduler therefore fires - two or three times inside a gate that imports `main.py` — and while a tick over a fake store - finds nothing due, `tick_all` reaches `runtime.get_runtime()`, which BUILDS tenants and - mutates the LRU cache that `verify_api`'s own isolation assertions read. A background thread - that can move a gate's subject is a flaky suite waiting to happen. - - So it takes the convention `main.py:353` already established for exactly this hazard — - `AIOS_PREWARM=1` — for exactly the reason stated there: env-gated rather than a startup - event, so importing this module in a gate can never fire anything. The DEPLOY sets it; the - external EventBridge tick (R5) does not depend on it either way, since that POSTs the - endpoint rather than riding this thread. - """ + """Start the loop once per process. **OPT-IN: `AIOS_AUTOMATIONS=1`.** + + ⚠ AMENDED 2026-08-03 (D), and the reason is a measurement rather than a preference. The wave + brief specified this thread DEFAULT-ON for the API. Then `verify_api.py` was timed: **196 + seconds**, i.e. more than three 60 s tick intervals. A default-on scheduler therefore fires + two or three times inside a gate that imports `main.py` — and while a tick over a fake store + finds nothing due, `tick_all` reaches `runtime.get_runtime()`, which BUILDS tenants and + mutates the LRU cache that `verify_api`'s own isolation assertions read. A background thread + that can move a gate's subject is a flaky suite waiting to happen. + + So it takes the convention `main.py:353` already established for exactly this hazard — + `AIOS_PREWARM=1` — for exactly the reason stated there: env-gated rather than a startup + event, so importing this module in a gate can never fire anything. The DEPLOY sets it; the + external EventBridge tick (R5) does not depend on it either way, since that POSTs the + endpoint rather than riding this thread. + """ if os.environ.get("AIOS_AUTOMATIONS") != "1": return False # PostgreSQL production has one durable wake-up: EventBridge -> Lambda -> protected endpoint. @@ -14856,9 +15228,9 @@ def start_scheduler(log=print): log("[aios-auto] in-process scheduler refused on PostgreSQL; use the external tick") return False if _SCHEDULER[0] is not None: - return False - th = threading.Thread(target=scheduler_loop, kwargs={"log": log}, - daemon=True, name="automation-scheduler") - _SCHEDULER[0] = th - th.start() - return True + return False + th = threading.Thread(target=scheduler_loop, kwargs={"log": log}, + daemon=True, name="automation-scheduler") + _SCHEDULER[0] = th + th.start() + return True