fsanyoto commited on
Commit
0adeb0c
Β·
verified Β·
1 Parent(s): bf8519f

Deploy AIOS web (React glide grid + FastAPI slice)

Browse files
RELEASES.json CHANGED
@@ -1,5 +1,5 @@
1
  {
2
- "current": "0cf3d4c",
3
  "releases": [
4
  {
5
  "version": "v24",
 
1
  {
2
+ "current": "e43869a",
3
  "releases": [
4
  {
5
  "version": "v24",
VERSION CHANGED
@@ -1 +1 @@
1
- 0cf3d4c
 
1
+ e43869a
api/automation_engine.py CHANGED
@@ -1233,9 +1233,9 @@ def clean_config(kind, raw, previous=None):
1233
 
1234
  ⚠ THE NODE-SWITCH FLAGS FALL BACK TO `previous` WHEN THE KEY IS ABSENT, and that is a rail
1235
  rather than a nicety. `patch` replaces the whole config, and the canvas's config panels do not
1236
- edit `tier` / `noFallback` / `dryRun` β€” those are node SWITCHES. So a plain "Save" from a
1237
- panel that never knew about them would silently reset the paid rung to `anonymous` and turn
1238
- off the dry run: a save that quietly changes what the automation costs. Absent β‡’ keep;
1239
  present β‡’ take it, including `false`.
1240
  """
1241
  raw = raw if isinstance(raw, dict) else {}
@@ -1292,13 +1292,17 @@ def clean_config(kind, raw, previous=None):
1292
  fkey = _s(raw.get("fieldKey"), 80).strip()
1293
  if not fkey:
1294
  return None, "pick the automation column this run writes into"
1295
- # ⚠ UNKNOWN TIERS FALL BACK TO `anonymous`, NOT to the paid rung. A typo must never be
1296
- # the thing that starts spending money. An ABSENT tier keeps the one already stored (see
1297
- # the `flag` note above), and a stored `hiker` is carried FORWARD to the new vendor
1298
- # rather than silently demoted β€” see `TIER_ALIASES`.
1299
- tier = clean_tier(raw["tier"]) if "tier" in raw else ""
1300
- if not tier and "tier" not in raw:
1301
- tier = clean_tier(prev.get("tier"))
 
 
 
 
1302
  # C5: absent β‡’ keep whatever is stored, like every other switch in this branch β€” a panel
1303
  # that does not edit the post count must not reset it to the default on Save. And an
1304
  # INHERITED value is clamped rather than refused; see `clean_max_posts`.
@@ -1309,8 +1313,6 @@ def clean_config(kind, raw, previous=None):
1309
  return None, perr
1310
  return {"targetTable": table, "fieldKey": fkey,
1311
  "urlField": _s(raw.get("urlField"), 80).strip(),
1312
- "tier": tier or "anonymous",
1313
- "noFallback": flag("noFallback"),
1314
  # β›” THE ONE FLAG THAT MULTIPLIES THE BILL BY THE POST COUNT. Off unless asked
1315
  # for: the Profiles row carries post IDENTITY for free but NO engagement
1316
  # (measured 2026-08-05), so likes/comments cost one extra vendor record PER POST.
@@ -1515,8 +1517,7 @@ def _ensure_enrich_step(flow_raw):
1515
  # with only step 1 lands this at the end, which IS step 2.
1516
  seeded.insert(1, {
1517
  "id": "act_enrich", "kind": "enrich_instagram", "enabled": True, "when": None,
1518
- "config": {"tier": "anonymous", "noFallback": False, "postMetrics": False,
1519
- "commentMetrics": False,
1520
  "dryRun": False, "maxPosts": DEFAULT_POSTS_PER_PULL,
1521
  "fromView": "", "sortField": DEFAULT_ENRICH_SORT, "sortDir": "desc",
1522
  "limit": DEFAULT_ENRICH_LIMIT, "skipRecent": True,
@@ -2593,7 +2594,7 @@ from connectors_ig import ( # noqa: E402
2593
  _first, _ig_int,
2594
  bd_call, bd_filter_rows, bd_filter_start, bd_filter_status, bd_ready,
2595
  bd_snapshot_progress, depth_refusal,
2596
- ig_handle, pull_profile,
2597
  )
2598
 
2599
 
@@ -4620,22 +4621,24 @@ def preset_cells(res, pulled):
4620
 
4621
  def run_field_instagram(rt, defn, username="automation", log=print, step=_no_step,
4622
  rows=None):
4623
- """Automation #2 (R7 + R1's hybrid): for every row of a blank database that carries a profile
4624
- URL, pull the public profile β€” the PAID rung first when the tier says so, the anonymous
4625
- ladder underneath it β€” write a status string into the automation column, and append a
4626
- timestamped row to each of the three IG tables."""
 
 
 
4627
  cfg = defn.get("config") or {}
4628
  table_key, fkey = cfg.get("targetTable"), cfg.get("fieldKey")
4629
- tier = clean_tier(cfg.get("tier")) or "anonymous"
4630
- paid = tier == "brightdata"
4631
- fallback = not cfg.get("noFallback")
4632
  post_metrics = bool(cfg.get("postMetrics"))
4633
  comment_metrics = bool(cfg.get("commentMetrics"))
4634
  dry = bool(cfg.get("dryRun"))
4635
- steps = {"trigger": "ok", "source": "idle", "capture": "idle",
4636
- "capture_paid": "idle" if paid else "skipped",
4637
- "capture_anon": "idle" if (not paid or fallback) else "skipped",
4638
- "capture_metrics": "idle" if post_metrics else "skipped",
 
 
4639
  "write": "idle"}
4640
  t = ut_get(rt, table_key)
4641
  if not t:
@@ -4673,7 +4676,6 @@ def run_field_instagram(rt, defn, username="automation", log=print, step=_no_ste
4673
  # `MAX_UT_IG_ROWS` makes the same loop hundreds of millions of dict copies, i.e. an automation
4674
  # that no longer finishes. **Raising a cap and batching its writer are ONE change.** (W19-C.)
4675
  in_snaps, in_posts, in_psnaps, in_comments = [], [], [], []
4676
- paid_tried = paid_answered = anon_tried = anon_answered = False
4677
  targets = [(rid, str(r.get(url_field, "") or "").strip())
4678
  for rid, r in rows.items() if str(r.get(url_field, "") or "").strip()]
4679
  for i, (rid, url) in enumerate(targets):
@@ -4685,24 +4687,17 @@ def run_field_instagram(rt, defn, username="automation", log=print, step=_no_ste
4685
  step(f"Capturing profile {i + 1} of {len(targets)}")
4686
  counts["profiles"] += 1
4687
  res = pull_profile(url, max_posts=cfg.get("maxPosts") or DEFAULT_POSTS_PER_PULL, log=log,
4688
- tier=tier, fallback=fallback, post_metrics=post_metrics,
4689
- comment_metrics=comment_metrics,
4690
  pending_metrics=(pending_metrics if not dry else None))
4691
  pulled = _iso()
4692
  via = res.get("via") or ""
4693
  read_ok = res["state"] in ("ok", "partial")
4694
- # WHICH RUNG ANSWERED β€” tracked per profile so the canvas's rung dots are a measurement.
4695
- if paid:
4696
- paid_tried = True
4697
- if via == "brightdata" and read_ok:
4698
- paid_answered = True
4699
- counts["paid"] += 1
4700
- elif fallback:
4701
- anon_tried = True
4702
- anon_answered = anon_answered or read_ok
4703
- else:
4704
- anon_tried = True
4705
- anon_answered = anon_answered or read_ok
4706
  if read_ok:
4707
  counts["ok" if res["state"] == "ok" else "partial"] += 1
4708
  prof = res["profile"]
@@ -4803,21 +4798,24 @@ def run_field_instagram(rt, defn, username="automation", log=print, step=_no_ste
4803
  else:
4804
  cap_state = "partial" if (counts["blocked"] or counts["error"]
4805
  or counts["partial"]) else "ok"
4806
- steps["capture"] = cap_state
4807
- steps["capture_paid"] = ("ok" if paid_answered
4808
- else "blocked" if paid_tried else "skipped")
4809
- steps["capture_anon"] = ("ok" if anon_answered
4810
- else "blocked" if anon_tried else "skipped")
4811
  # ⚠ MEASURED, LIKE EVERY OTHER DOT: it is `ok` only if an engagement row was actually
4812
  # appended. "It was switched on" is not the same fact as "it answered", and painting the
4813
  # second from the first is the fabrication `node_status` refuses to make.
4814
  # ⚠ AND `blocked` IS A CLAIM ABOUT THE VENDOR, so it needs something to have been ASKED. A
4815
  # run that found no posts to enrich did not have a rung refuse it β€” nothing was requested β€”
4816
  # so that reads `skipped`, the same word an off switch earns.
4817
- steps["capture_metrics"] = ("ok" if counts["metrics"]
4818
- else "partial" if counts.get("metric_batches_pending")
4819
- else "blocked" if (post_metrics and counts["posts"])
4820
- else "skipped")
 
 
 
4821
 
4822
  summary = (f"{read}/{counts['profiles']} profiles read, {counts['posts']} posts "
4823
  f"({counts['new_posts']} new)")
@@ -5351,17 +5349,23 @@ NODE_STATES = ("idle", "ok", "partial", "error", "blocked", "skipped")
5351
  #: deliberate answer rather than an omission: a "Fetch the page" step that can be turned off is
5352
  #: not an automation with a disabled step, it is a broken automation with a lie on it. The ones
5353
  #: here are all REAL β€” each changes what the next run does:
5354
- #: schedule the trigger fires on its cron, or only by hand
5355
- #: paid the paid rung is reached for, or it is not (this is the one that spends money)
5356
- #: fallback the anonymous ladder catches a refused paid rung, or the run says `blocked`
5357
- #: instead of quietly writing an approximate number
5358
- #: postMetrics likes/comments per post are bought, or the engagement series does not grow β€”
5359
- #: the expensive one, because it costs a vendor record PER POST rather than per
5360
- #: profile (measured: the profile row carries post identity and no engagement)
5361
- #: write DRY RUN β€” read everything, compute the counts, write nothing anywhere
5362
- NODE_TOGGLES = {"trigger": "schedule", "capture_paid": "paid",
5363
- "capture_anon": "fallback", "capture_metrics": "postMetrics",
5364
- "write": "write"}
 
 
 
 
 
 
5365
 
5366
 
5367
  def _cron_label(cron):
@@ -5445,56 +5449,46 @@ def graph(defn):
5445
  edges = []
5446
 
5447
  if defn.get("kind") == "field_instagram":
5448
- paid = clean_tier(cfg.get("tier")) == "brightdata"
5449
- fb = not cfg.get("noFallback")
5450
- anon_on = (not paid) or fb
 
 
 
 
 
 
5451
  metrics_on = bool(cfg.get("postMetrics"))
 
5452
  max_posts = cfg.get("maxPosts") or DEFAULT_POSTS_PER_PULL
5453
  nodes += [
5454
  node("source", "source", "Profile set", cfg.get("targetTable") or "No database", 1,
5455
  panel="source",
5456
  detail=f"URL column: {cfg.get('urlField')}" if cfg.get("urlField")
5457
  else "URL column: the one the field is bound to"),
5458
- node("capture", "branch", "Capture",
5459
- "Exact counts, then estimates" if paid else "Estimated counts only", 2, panel="capture",
5460
- detail=f"Up to {max_posts} posts per profile"),
5461
- node("capture_paid", "capture", "Exact counts",
5462
- "Exact followers and top posts" if paid else "Off",
5463
- 3, row=-1, panel="capture", on=paid,
5464
- detail="" if bd_ready() else "Not set up yet"),
5465
- node("capture_anon", "capture", "Estimated counts",
5466
- "Free, rounded numbers" if anon_on else "Off",
5467
- 3, row=1, panel="capture", on=anon_on,
5468
- detail="Rounded to the nearest thousand"),
5469
  # ⚠ ITS OWN NODE BECAUSE IT IS ITS OWN BILL. The profile row carries post IDENTITY
5470
  # and no engagement (measured), so likes/comments are a SECOND vendor call per post.
5471
  # A switch that multiplies a run's cost by the post count deserves to be visible on
5472
  # the canvas rather than buried in a config panel.
5473
- node("capture_metrics", "capture", "Post engagement",
5474
- "Likes and comments per post" if metrics_on
5475
- else "Off β€” no likes/comments are bought",
5476
- 4, panel="capture", on=metrics_on,
5477
- detail=(f"~{max_posts}x the records of a profile-only run"
5478
- if metrics_on else
5479
  "ut_ig_post_snapshots only grows while this is on")),
 
 
 
 
 
5480
  node("write", "write", "Write",
5481
- cfg.get("targetTable") or "No database", 5, panel="write",
5482
  on=not cfg.get("dryRun"),
5483
  detail="+ ut_ig_snapshots Β· ut_ig_posts Β· ut_ig_post_snapshots"),
5484
  ]
5485
  edges = [{"from": "trigger", "to": "source", "label": ""},
5486
- {"from": "source", "to": "capture", "label": ""},
5487
- {"from": "capture", "to": "capture_paid", "label": "paid"},
5488
- {"from": "capture", "to": "capture_anon", "label": "fallback"},
5489
- {"from": "capture_paid", "to": "capture_metrics", "label": ""},
5490
- {"from": "capture_anon", "to": "capture_metrics", "label": ""},
5491
- {"from": "capture_metrics", "to": "write", "label": ""}]
5492
- # When there is no paid rung the anonymous one is the ONLY rung, so it must not carry a
5493
- # switch that would leave the automation with nothing to run.
5494
- if not paid:
5495
- for n in nodes:
5496
- if n["id"] == "capture_anon":
5497
- n["toggle"] = ""
5498
  elif defn.get("kind") == "discover_instagram":
5499
  limit = int(cfg.get("recordsLimit") or 0)
5500
  est = discover_estimate(limit)
@@ -5583,13 +5577,10 @@ def toggle_node(rt, auto_id, node_id):
5583
  return patch(rt, auto_id, {"trigger": trg})
5584
  if which == "schedule":
5585
  sched["enabled"] = not sched.get("enabled")
5586
- elif which == "paid":
5587
- cfg["tier"] = ("anonymous" if clean_tier(cfg.get("tier")) == "brightdata"
5588
- else "brightdata")
5589
- elif which == "fallback":
5590
- cfg["noFallback"] = not cfg.get("noFallback")
5591
  elif which == "postMetrics":
5592
  cfg["postMetrics"] = not cfg.get("postMetrics")
 
 
5593
  elif which == "write":
5594
  cfg["dryRun"] = not cfg.get("dryRun")
5595
  return patch(rt, auto_id, {"config": cfg, "schedule": sched})
@@ -6137,10 +6128,10 @@ def _clean_action_config(kind, cfg, depth, seen, count):
6137
  "limit": min(e_limit, MAX_ENRICH_PER_RUN),
6138
  "skipRecent": bool(cfg.get("skipRecent")),
6139
  "skipRecentDays": max(1, e_days),
6140
- # An unknown tier falls back to `anonymous`, never to the paid rung β€” a typo must
6141
- # not be the thing that starts spending money (the `clean_config` rule, verbatim).
6142
- "tier": clean_tier(cfg.get("tier")) or "anonymous",
6143
- "noFallback": bool(cfg.get("noFallback")),
6144
  # β›” THE FLAG THAT MULTIPLIES THE BILL BY THE POST COUNT. Off unless asked for.
6145
  "postMetrics": bool(cfg.get("postMetrics")),
6146
  # The separate Comments dataset is more granular than a post read and stays off
@@ -6292,13 +6283,21 @@ DEFAULT_ENRICH_SORT = "first_found"
6292
  #: an underscore β€” `COUNT_LABELS` on the client is an allow-list and cannot pick it up by
6293
  #: accident, and `_commit_run`'s numeric filter is the second net under it.
6294
  RUN_NOTES_KEY = "_notes"
6295
- #: How long a handle a VENDOR SAID DOES NOT EXIST is left alone before it is tried again.
6296
- #: ⚠ NOT permanent, and not the success cooldown either. Permanent would be wrong β€” a handle can
6297
- #: be renamed back or a suspension lifted β€” but re-buying a dead account every morning is what
6298
- #: the owner reported ("blocked AGAIN"), so the floor is one paid attempt per handle per month
6299
- #: instead of one per day. Editing the handle changes the key and retries immediately, which is
6300
- #: the behaviour someone correcting a typo expects.
6301
- NOT_FOUND_RETRY_DAYS = 30
 
 
 
 
 
 
 
 
6302
  #: A queued profile snapshot the vendor never finishes is dropped after this, with a note. An
6303
  #: unbounded pending list is the forever-loop this whole change exists to remove, wearing the
6304
  #: opposite mask ([[gate-answers-the-wrong-question]]).
@@ -6375,6 +6374,48 @@ def _gone_key(row, handle):
6375
  return f"{plat}:{str(handle or '').strip().lstrip('@').lower()}"
6376
 
6377
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6378
  def enrich_selection(rt, table_key, cfg, profile_key, today=None, gone=None):
6379
  """⭐ 2026-08-07 (owner ruling) β€” WHICH records this enrich step spends on, in order.
6380
 
@@ -6512,12 +6553,12 @@ def enrich_selection(rt, table_key, cfg, profile_key, today=None, gone=None):
6512
  if not raw_handle:
6513
  blank_handle += 1
6514
  continue
6515
- verdict = (gone or {}).get(_gone_key(r, raw_handle))
6516
- if isinstance(verdict, dict):
6517
- since_gone = _days_since(verdict.get("at"), today=today)
6518
- if since_gone is None or since_gone < NOT_FOUND_RETRY_DAYS:
6519
- skipped_gone.append(raw_handle)
6520
- continue
6521
  if cooling:
6522
  since = _days_since(r.get("enriched_at"), today=today)
6523
  if since is not None and since < days:
@@ -6532,9 +6573,12 @@ def enrich_selection(rt, table_key, cfg, profile_key, today=None, gone=None):
6532
  notes.append(f"{cooled} skipped as enriched in the last {days} days")
6533
  if skipped_gone:
6534
  shown = ", ".join(f"@{h}" for h in skipped_gone[:5])
 
 
 
6535
  notes.append(f"{len(skipped_gone)} skipped because Instagram has no such account "
6536
  f"({shown}{', …' if len(skipped_gone) > 5 else ''}) β€” delete the row or "
6537
- f"correct the handle; it is retried after {NOT_FOUND_RETRY_DAYS} days")
6538
  if blank_handle:
6539
  notes.append(f"{blank_handle} skipped with no handle")
6540
  if len(picked) < quota and (cooled or blank_handle or skipped_gone or ordered):
@@ -6770,8 +6814,6 @@ def apply_actions(rt, defn, table_key, row_ids, username="automation", log=print
6770
  res = pull_profile(handle_raw,
6771
  max_posts=cfg.get("maxPosts") or DEFAULT_POSTS_PER_PULL,
6772
  log=log,
6773
- tier=clean_tier(cfg.get("tier")) or "anonymous",
6774
- fallback=not cfg.get("noFallback"),
6775
  post_metrics=bool(cfg.get("postMetrics")),
6776
  comment_metrics=bool(cfg.get("commentMetrics")),
6777
  pending_metrics=(enrich["pending"]
@@ -6886,8 +6928,6 @@ def migrate_field_instagram(rt, defn):
6886
  max_posts = 24
6887
  act = {"id": "act_enrich", "kind": "enrich_instagram", "enabled": True, "when": None,
6888
  "config": {"profileField": bound,
6889
- "tier": clean_tier(cfg.get("tier")) or "anonymous",
6890
- "noFallback": bool(cfg.get("noFallback")),
6891
  "postMetrics": bool(cfg.get("postMetrics")),
6892
  "commentMetrics": bool(cfg.get("commentMetrics")),
6893
  "dryRun": bool(cfg.get("dryRun")),
@@ -7177,7 +7217,7 @@ def collect_pending_profile_snapshots(rt, defn, username="automation", log=print
7177
  """
7178
  tasks = _pending_profile_tasks(defn)
7179
  if not tasks:
7180
- return ("ok", "No pending profile reads.", {}, [], {"capture_paid": "idle"})
7181
  step(f"Collecting {len(tasks)} deferred profile read{'' if len(tasks) == 1 else 's'}")
7182
  remaining, patches, affected, notes = [], {}, [], []
7183
  acc = {"snaps": [], "posts": [], "psnaps": [], "comments": [], "pending": [],
@@ -7196,7 +7236,8 @@ def collect_pending_profile_snapshots(rt, defn, username="automation", log=print
7196
  acc["notes"].append(note)
7197
  # ⭐ `failed` = the vendor finished, collected nothing, and blamed the TARGET. On a
7198
  # profile request that means the account could not be reached at all, so the verdict
7199
- # is remembered and the selection stops re-buying it (`NOT_FOUND_RETRY_DAYS`).
 
7200
  # ⚠ `done`-with-zero is NOT remembered: "we found no matches" is a statement about
7201
  # the query, and turning it into "this account does not exist" would silently retire
7202
  # live handles.
@@ -7268,7 +7309,7 @@ def collect_pending_profile_snapshots(rt, defn, username="automation", log=print
7268
  f"; {waiting} still building" if waiting else ""])
7269
  state = "ok" if ready and not closed else "partial"
7270
  return (state, head + tail, counts, affected,
7271
- {"capture_paid": "ok" if ready else "partial", "write": "ok" if ready else "idle"})
7272
 
7273
 
7274
  def _write_collected_metric_rows(rt, defn, username, idents, snapshots, comments, log):
@@ -7318,10 +7359,10 @@ def collect_pending_metric_snapshots(rt, defn, username="automation", log=print,
7318
  """Collect deferred paid Post/Reel/Comment snapshots; never launches a new scrape request."""
7319
  tasks = _pending_metric_tasks(defn)
7320
  if not tasks:
7321
- return ("ok", "No pending post-engagement snapshots.", {}, [], {"capture_metrics": "idle"})
7322
  step(f"Collecting {len(tasks)} post-engagement batch{'' if len(tasks) == 1 else 'es'}")
7323
  remaining, idents, snapshots, comments = [], [], [], []
7324
- ready, waiting, closed, closed_notes = 0, 0, 0, []
7325
  for task in tasks:
7326
  # ⭐⭐ 2026-08-09 β€” ASK THE STATUS DOCUMENT, NOT THE ROWS. `building` used to be
7327
  # `not rows or …`, so a snapshot the vendor had FINISHED with zero records was re-queued
@@ -7333,7 +7374,7 @@ def collect_pending_metric_snapshots(rt, defn, username="automation", log=print,
7333
  # β›” CLOSED, NOT RE-QUEUED. The vendor is finished and there is nothing to collect;
7334
  # keeping the entry would be a pending task that can never resolve.
7335
  closed += 1
7336
- closed_notes.append(f"{task['influencer']}: {_s(empty_note, 160)}")
7337
  continue
7338
  payload, note = bd_call(f"{BD_PATH_SNAPSHOT}/{task['snapshotId']}", {"format": "json"})
7339
  rows = _bd_rows(payload) if not note else []
@@ -7357,6 +7398,23 @@ def collect_pending_metric_snapshots(rt, defn, username="automation", log=print,
7357
  post["influencer_key"] = task["influencer"]
7358
  mapped_posts.append(post)
7359
  if mapped_posts:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
7360
  _unused, post_rows, metric_rows, embedded = capture_rows(
7361
  {"state": "ok", "profile": {"username": task["influencer"]},
7362
  "posts": mapped_posts, "comments": [], "via": "brightdata:deferred"}, pulled)
@@ -7381,8 +7439,8 @@ def collect_pending_metric_snapshots(rt, defn, username="automation", log=print,
7381
  # other call sites disagree about the shape of a run β€” and `_commit_run` already drops
7382
  # non-numeric count values, so a pop that is ever missed degrades to today's behaviour rather
7383
  # than to a crash.
7384
- if closed_notes:
7385
- counts[RUN_NOTES_KEY] = closed_notes
7386
  # ⚠ THE EMPTY ONES ARE NAMED, not silently dropped. A batch that finished with no records is
7387
  # a real outcome the tenant paid for and it must read as an answer, not as a disappearance.
7388
  tail = (f"; {closed} finished with nothing to collect" if closed else "")
@@ -7390,9 +7448,9 @@ def collect_pending_metric_snapshots(rt, defn, username="automation", log=print,
7390
  return ("partial",
7391
  f"{ready} post-engagement batch{'' if ready == 1 else 'es'} collected; "
7392
  f"{waiting} still building and will be collected automatically{tail}",
7393
- counts, [], {"capture_metrics": "partial", "write": "ok"})
7394
  return ("ok", f"{ready} post-engagement batch{'' if ready == 1 else 'es'} collected{tail}",
7395
- counts, [], {"capture_metrics": "ok", "write": "ok"})
7396
 
7397
 
7398
  def ai_decide(rt, defn, act, row, row_id="", log=print):
@@ -7781,8 +7839,25 @@ def _ut():
7781
 
7782
 
7783
  #: The fns by family β€” how a value is folded, and what a blank means in each.
7784
- _ROLLUP_NUM_FNS = frozenset({"sum", "average", "min", "max"})
7785
  _ROLLUP_BOOL_FNS = frozenset({"and", "or", "xor"})
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
7786
  _ROLLUP_TEXT_FNS = frozenset({"concatenate", "arrayjoin", "arraycompact", "arrayunique"})
7787
  #: What `arrayjoin` puts between values. Airtable uses ", "; `concatenate` uses nothing.
7788
  _ROLLUP_JOIN = ", "
@@ -7826,6 +7901,46 @@ def _sort_key(value, ftype):
7826
  return (raw.lower(),)
7827
 
7828
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
7829
  def _rollup_fold(fn, values):
7830
  """`values` (raw cells, in the order the window kept them) β†’ the aggregate, as a STRING.
7831
 
@@ -7853,8 +7968,24 @@ def _rollup_fold(fn, values):
7853
  nums = [n for n in (_lane_num(v) for v in values) if n is not None]
7854
  if not nums:
7855
  return ""
7856
- out = (sum(nums) if fn == "sum" else min(nums) if fn == "min"
7857
- else max(nums) if fn == "max" else sum(nums) / len(nums))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
7858
  return f"{out:.0f}" if float(out).is_integer() else f"{out:.2f}"
7859
  if fn in _ROLLUP_BOOL_FNS:
7860
  # A checkbox cell is '1'/'' in this product, so truth is "non-blank and not a zero".
@@ -7864,7 +7995,11 @@ def _rollup_fold(fn, values):
7864
  hit = (all(flags) if fn == "and" else any(flags) if fn == "or"
7865
  else sum(1 for f in flags if f) % 2 == 1)
7866
  return "1" if hit else ""
7867
- # --- the text family
 
 
 
 
7868
  vals = [str(v).strip() for v in values if str(v or "").strip() != ""]
7869
  if fn == "arrayunique":
7870
  seen, uniq = set(), []
@@ -7912,8 +8047,17 @@ def _linked_rows_by_join(linked, on_key):
7912
  return idx
7913
 
7914
 
7915
- def _rollup_condition_matches(row, condition, field_types):
7916
- """Evaluate one Airtable-style linked-record condition against a candidate row."""
 
 
 
 
 
 
 
 
 
7917
  field = str((condition or {}).get("field") or "")
7918
  op = str((condition or {}).get("op") or "")
7919
  raw = (row or {}).get(field)
@@ -7922,6 +8066,18 @@ def _rollup_condition_matches(row, condition, field_types):
7922
  return text == ""
7923
  if op == "is_not_empty":
7924
  return text != ""
 
 
 
 
 
 
 
 
 
 
 
 
7925
  wanted = str((condition or {}).get("value") or "").strip()
7926
  if op == "contains":
7927
  return wanted.casefold() in text.casefold()
@@ -8021,16 +8177,21 @@ def compute_relation_cells(rt, table_key, tables=None):
8021
  # ⚠ A rollup whose link field does not exist (renamed, deleted) resolves to NOTHING
8022
  # and therefore to a blank cell β€” never to a stale number. A column that keeps
8023
  # printing yesterday's answer after its input is gone is the worst of the options.
8024
- conditions = list(bag.get("conditions") or [])
8025
- if conditions:
8026
- matches = lambda pair: [
8027
- _rollup_condition_matches(pair[1], condition,
8028
- linked_types.get(lk_key) or {})
8029
- for condition in conditions]
8030
- if str(bag.get("conditionConj") or "and") == "or":
8031
- hits = [pair for pair in hits if any(matches(pair))]
8032
- else:
8033
- hits = [pair for pair in hits if all(matches(pair))]
 
 
 
 
 
8034
  sort_by = str(bag.get("sortBy") or "")
8035
  if sort_by:
8036
  ftype = (linked_types.get(lk_key) or {}).get(sort_by, "text")
@@ -8060,6 +8221,30 @@ def compute_relation_cells(rt, table_key, tables=None):
8060
  limit = int(bag.get("limit") or 0)
8061
  if limit:
8062
  hits = hits[:limit]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8063
  src = str(bag.get("field") or "")
8064
  want = _rollup_fold(str(bag.get("fn") or ""),
8065
  [r.get(src) for _i, r in hits] if src else [1] * len(hits))
@@ -9068,9 +9253,9 @@ def compose_sentence(defn):
9068
  host = urlparse(str(cfg.get("url") or "")).hostname or "the page"
9069
  body = f"read {host} and upsert rows into {cfg.get('targetTable') or 'a new database'}"
9070
  elif kind == "field_instagram":
9071
- body = (f"capture Instagram profiles for {cfg.get('targetTable') or 'the database'}"
9072
- + (", exact counts first" if clean_tier(cfg.get("tier")) == "brightdata"
9073
- else ", anonymous only"))
9074
  elif kind == "discover_instagram":
9075
  body = (f"search Instagram for up to {cfg.get('recordsLimit') or 0} profiles into "
9076
  f"{cfg.get('targetTable') or DISCOVER_TABLE}")
 
1233
 
1234
  ⚠ THE NODE-SWITCH FLAGS FALL BACK TO `previous` WHEN THE KEY IS ABSENT, and that is a rail
1235
  rather than a nicety. `patch` replaces the whole config, and the canvas's config panels do not
1236
+ edit `postMetrics` / `commentMetrics` / `dryRun` β€” those are node SWITCHES. So a plain "Save"
1237
+ from a panel that never knew about them would silently turn off the dry run, or turn ON a
1238
+ per-post purchase: a save that quietly changes what the automation costs. Absent β‡’ keep;
1239
  present β‡’ take it, including `false`.
1240
  """
1241
  raw = raw if isinstance(raw, dict) else {}
 
1292
  fkey = _s(raw.get("fieldKey"), 80).strip()
1293
  if not fkey:
1294
  return None, "pick the automation column this run writes into"
1295
+ # β›”β›” `tier` AND `noFallback` ARE ACCEPTED AND IGNORED (wave 28 / R5, contract C2).
1296
+ # They are read from nothing and written to nothing: a stored definition carrying either
1297
+ # still SAVES β€” it simply loses them on the next write β€” and neither is ever a reason to
1298
+ # refuse. That asymmetry is D-65's law and it is not squeamishness: refusing an unknown
1299
+ # key would 400 every automation a tenant stored before this wave, forever, on a screen
1300
+ # that gives them no way to remove it. Dropping a retired key is a migration; refusing it
1301
+ # is an outage.
1302
+ # ⚠ `clean_tier`/`TIERS`/`bd_ready` KEEP THEIR NAMES. They are VENDOR vocabulary
1303
+ # (`brightdata` is a provider, and `verify_automation` fences the name), not the retired
1304
+ # USER concept β€” renaming them would be a second, unrelated change wearing this one's
1305
+ # justification.
1306
  # C5: absent β‡’ keep whatever is stored, like every other switch in this branch β€” a panel
1307
  # that does not edit the post count must not reset it to the default on Save. And an
1308
  # INHERITED value is clamped rather than refused; see `clean_max_posts`.
 
1313
  return None, perr
1314
  return {"targetTable": table, "fieldKey": fkey,
1315
  "urlField": _s(raw.get("urlField"), 80).strip(),
 
 
1316
  # β›” THE ONE FLAG THAT MULTIPLIES THE BILL BY THE POST COUNT. Off unless asked
1317
  # for: the Profiles row carries post IDENTITY for free but NO engagement
1318
  # (measured 2026-08-05), so likes/comments cost one extra vendor record PER POST.
 
1517
  # with only step 1 lands this at the end, which IS step 2.
1518
  seeded.insert(1, {
1519
  "id": "act_enrich", "kind": "enrich_instagram", "enabled": True, "when": None,
1520
+ "config": {"postMetrics": False, "commentMetrics": False,
 
1521
  "dryRun": False, "maxPosts": DEFAULT_POSTS_PER_PULL,
1522
  "fromView": "", "sortField": DEFAULT_ENRICH_SORT, "sortDir": "desc",
1523
  "limit": DEFAULT_ENRICH_LIMIT, "skipRecent": True,
 
2594
  _first, _ig_int,
2595
  bd_call, bd_filter_rows, bd_filter_start, bd_filter_status, bd_ready,
2596
  bd_snapshot_progress, depth_refusal,
2597
+ ig_handle, pull_profile, top_up_views,
2598
  )
2599
 
2600
 
 
4621
 
4622
  def run_field_instagram(rt, defn, username="automation", log=print, step=_no_step,
4623
  rows=None):
4624
+ """Automation #2 (R7): for every row of a database that carries a profile URL, pull the public
4625
+ profile through the paid capability chain, write a status string into the automation column,
4626
+ and append a timestamped row to each of the three IG tables.
4627
+
4628
+ ⭐ WAVE 28 / R5 β€” there is no tier and no rung choice here any more. `pull_profile` routes per
4629
+ CAPABILITY and reports `blocked` rather than downgrading to an approximate row, so the two
4630
+ "which rung answered" counters this function used to keep have nothing left to distinguish."""
4631
  cfg = defn.get("config") or {}
4632
  table_key, fkey = cfg.get("targetTable"), cfg.get("fieldKey")
 
 
 
4633
  post_metrics = bool(cfg.get("postMetrics"))
4634
  comment_metrics = bool(cfg.get("commentMetrics"))
4635
  dry = bool(cfg.get("dryRun"))
4636
+ # ⚠ THE STEP KEYS ARE THE CANVAS NODE IDS (contract C3) and the two must move together β€” a
4637
+ # status written under a node id `graph()` no longer emits is a dot nothing renders, which is
4638
+ # indistinguishable from a step that never ran.
4639
+ steps = {"trigger": "ok", "source": "idle",
4640
+ "capture_posts": "idle" if post_metrics else "skipped",
4641
+ "capture_comments": "idle" if comment_metrics else "skipped",
4642
  "write": "idle"}
4643
  t = ut_get(rt, table_key)
4644
  if not t:
 
4676
  # `MAX_UT_IG_ROWS` makes the same loop hundreds of millions of dict copies, i.e. an automation
4677
  # that no longer finishes. **Raising a cap and batching its writer are ONE change.** (W19-C.)
4678
  in_snaps, in_posts, in_psnaps, in_comments = [], [], [], []
 
4679
  targets = [(rid, str(r.get(url_field, "") or "").strip())
4680
  for rid, r in rows.items() if str(r.get(url_field, "") or "").strip()]
4681
  for i, (rid, url) in enumerate(targets):
 
4687
  step(f"Capturing profile {i + 1} of {len(targets)}")
4688
  counts["profiles"] += 1
4689
  res = pull_profile(url, max_posts=cfg.get("maxPosts") or DEFAULT_POSTS_PER_PULL, log=log,
4690
+ post_metrics=post_metrics, comment_metrics=comment_metrics,
 
4691
  pending_metrics=(pending_metrics if not dry else None))
4692
  pulled = _iso()
4693
  via = res.get("via") or ""
4694
  read_ok = res["state"] in ("ok", "partial")
4695
+ # ⚠ `counts["paid"]` SURVIVES R5 AND IT IS NOT THE RETIRED TIER. It counts profiles a PAID
4696
+ # vendor actually answered, which is the run's spend report β€” the thing the owner reads to
4697
+ # reconcile a bill. What died is the free rung it used to be contrasted with, so the test
4698
+ # is now simply "did a vendor answer" rather than "did the vendor we were told to try".
4699
+ if read_ok and via in ("brightdata", "apify"):
4700
+ counts["paid"] += 1
 
 
 
 
 
 
4701
  if read_ok:
4702
  counts["ok" if res["state"] == "ok" else "partial"] += 1
4703
  prof = res["profile"]
 
4798
  else:
4799
  cap_state = "partial" if (counts["blocked"] or counts["error"]
4800
  or counts["partial"]) else "ok"
4801
+ # ⭐ C3 β€” THE CAPTURE FORK IS GONE, SO ITS OUTCOME LANDS ON `source`. There is one way to
4802
+ # read a profile now, and the node that names the profile set is the honest owner of "did
4803
+ # reading them work". The old `capture`/`capture_paid`/`capture_anon` trio described a branch
4804
+ # that no longer exists in behaviour OR on screen.
4805
+ steps["source"] = cap_state if targets else steps["source"]
4806
  # ⚠ MEASURED, LIKE EVERY OTHER DOT: it is `ok` only if an engagement row was actually
4807
  # appended. "It was switched on" is not the same fact as "it answered", and painting the
4808
  # second from the first is the fabrication `node_status` refuses to make.
4809
  # ⚠ AND `blocked` IS A CLAIM ABOUT THE VENDOR, so it needs something to have been ASKED. A
4810
  # run that found no posts to enrich did not have a rung refuse it β€” nothing was requested β€”
4811
  # so that reads `skipped`, the same word an off switch earns.
4812
+ steps["capture_posts"] = ("ok" if counts["metrics"]
4813
+ else "partial" if counts.get("metric_batches_pending")
4814
+ else "blocked" if (post_metrics and counts["posts"])
4815
+ else "skipped")
4816
+ steps["capture_comments"] = ("ok" if counts.get("comments")
4817
+ else "blocked" if (comment_metrics and counts["posts"])
4818
+ else "skipped")
4819
 
4820
  summary = (f"{read}/{counts['profiles']} profiles read, {counts['posts']} posts "
4821
  f"({counts['new_posts']} new)")
 
5349
  #: deliberate answer rather than an omission: a "Fetch the page" step that can be turned off is
5350
  #: not an automation with a disabled step, it is a broken automation with a lie on it. The ones
5351
  #: here are all REAL β€” each changes what the next run does:
5352
+ #: schedule the trigger fires on its cron, or only by hand
5353
+ #: postMetrics likes/comments per post are bought, or the engagement series does not grow β€”
5354
+ #: it costs a vendor record PER POST rather than per profile (measured: the
5355
+ #: profile row carries post identity and no engagement)
5356
+ #: commentMetrics the Comments dataset is bought, which can bill many rows per post
5357
+ #: write DRY RUN β€” read everything, compute the counts, write nothing anywhere
5358
+ #:
5359
+ #: ⭐⭐ WAVE 28 / R5+R6 (contract C3) β€” `paid` AND `fallback` ARE GONE, and what replaced them is
5360
+ #: the point of the ruling: the money switches used to be "which rung do we try" (a question about
5361
+ #: our plumbing), and they are now "what do you want captured" (a question about the user's data).
5362
+ #: Profile is always captured and has no switch β€” an automation that fetches nothing is not an
5363
+ #: automation with a step turned off, it is a broken one with a lie on it.
5364
+ #: ⚠ `trigger: "schedule"` IS NOT PART OF THAT COLLAPSE. It is the trigger card's own on/off and
5365
+ #: has its own branch in `toggle_node` (event triggers flip themselves, not the cron); C3's "the
5366
+ #: three toggles" names the CAPTURE ladder it is reshaping.
5367
+ NODE_TOGGLES = {"trigger": "schedule", "capture_posts": "postMetrics",
5368
+ "capture_comments": "commentMetrics", "write": "write"}
5369
 
5370
 
5371
  def _cron_label(cron):
 
5449
  edges = []
5450
 
5451
  if defn.get("kind") == "field_instagram":
5452
+ # ⭐⭐ WAVE 28 / CONTRACT C3 β€” FOUR NODES, THREE OF THEM SWITCHES OVER WHAT IS CAPTURED.
5453
+ # This branch used to draw a FORK: `Capture` splitting into `Exact counts` (paid) and
5454
+ # `Estimated counts` (the free anonymous ladder), rejoining at `Post engagement`. R5
5455
+ # deleted the ladder, so the fork had one arm; keeping it would have drawn a decision the
5456
+ # engine no longer makes, with a switch (`fallback`) flipping a config key the cleaners
5457
+ # now discard. A canvas that offers a choice the runtime ignores is worse than no canvas.
5458
+ # ⚠ `Profile set` STILL CARRIES NO SWITCH, and now that is the whole ruling rather than an
5459
+ # implementation detail: the profile is always captured (R6), Posts and Comments are the
5460
+ # opt-ins, and both are OFF until asked for.
5461
  metrics_on = bool(cfg.get("postMetrics"))
5462
+ comments_on = bool(cfg.get("commentMetrics"))
5463
  max_posts = cfg.get("maxPosts") or DEFAULT_POSTS_PER_PULL
5464
  nodes += [
5465
  node("source", "source", "Profile set", cfg.get("targetTable") or "No database", 1,
5466
  panel="source",
5467
  detail=f"URL column: {cfg.get('urlField')}" if cfg.get("urlField")
5468
  else "URL column: the one the field is bound to"),
 
 
 
 
 
 
 
 
 
 
 
5469
  # ⚠ ITS OWN NODE BECAUSE IT IS ITS OWN BILL. The profile row carries post IDENTITY
5470
  # and no engagement (measured), so likes/comments are a SECOND vendor call per post.
5471
  # A switch that multiplies a run's cost by the post count deserves to be visible on
5472
  # the canvas rather than buried in a config panel.
5473
+ node("capture_posts", "capture", "Post data",
5474
+ "Likes and comments per post" if metrics_on else "Off",
5475
+ 2, panel="capture", on=metrics_on,
5476
+ detail=(f"Up to {max_posts} posts per profile" if metrics_on else
 
 
5477
  "ut_ig_post_snapshots only grows while this is on")),
5478
+ node("capture_comments", "capture", "Comment data",
5479
+ "Comments on those posts" if comments_on else "Off",
5480
+ 3, panel="capture", on=comments_on,
5481
+ detail=("The full comments dataset β€” many rows per post" if comments_on else
5482
+ "Comments already embedded in a paid post row are kept either way")),
5483
  node("write", "write", "Write",
5484
+ cfg.get("targetTable") or "No database", 4, panel="write",
5485
  on=not cfg.get("dryRun"),
5486
  detail="+ ut_ig_snapshots Β· ut_ig_posts Β· ut_ig_post_snapshots"),
5487
  ]
5488
  edges = [{"from": "trigger", "to": "source", "label": ""},
5489
+ {"from": "source", "to": "capture_posts", "label": ""},
5490
+ {"from": "capture_posts", "to": "capture_comments", "label": ""},
5491
+ {"from": "capture_comments", "to": "write", "label": ""}]
 
 
 
 
 
 
 
 
 
5492
  elif defn.get("kind") == "discover_instagram":
5493
  limit = int(cfg.get("recordsLimit") or 0)
5494
  est = discover_estimate(limit)
 
5577
  return patch(rt, auto_id, {"trigger": trg})
5578
  if which == "schedule":
5579
  sched["enabled"] = not sched.get("enabled")
 
 
 
 
 
5580
  elif which == "postMetrics":
5581
  cfg["postMetrics"] = not cfg.get("postMetrics")
5582
+ elif which == "commentMetrics":
5583
+ cfg["commentMetrics"] = not cfg.get("commentMetrics")
5584
  elif which == "write":
5585
  cfg["dryRun"] = not cfg.get("dryRun")
5586
  return patch(rt, auto_id, {"config": cfg, "schedule": sched})
 
6128
  "limit": min(e_limit, MAX_ENRICH_PER_RUN),
6129
  "skipRecent": bool(cfg.get("skipRecent")),
6130
  "skipRecentDays": max(1, e_days),
6131
+ # β›” `tier`/`noFallback` are ACCEPTED AND IGNORED here for the same reason as in
6132
+ # `clean_config` (R5 / C2): a stored action carrying them still saves and simply
6133
+ # loses them, because refusing a retired key 400s every definition written before
6134
+ # this wave (D-65).
6135
  # β›” THE FLAG THAT MULTIPLIES THE BILL BY THE POST COUNT. Off unless asked for.
6136
  "postMetrics": bool(cfg.get("postMetrics")),
6137
  # The separate Comments dataset is more granular than a post read and stays off
 
6283
  #: an underscore β€” `COUNT_LABELS` on the client is an allow-list and cannot pick it up by
6284
  #: accident, and `_commit_run`'s numeric filter is the second net under it.
6285
  RUN_NOTES_KEY = "_notes"
6286
+ #: ⭐⭐ WAVE 28 / OWNER RULING R9 β€” `NOT_FOUND_RETRY_DAYS` IS RETIRED. A handle a VENDOR SAID DOES
6287
+ #: NOT EXIST is now a TOMBSTONE: never auto-retried, at any interval.
6288
+ #: β›” THE 30-DAY BACKOFF WAS THE DEFENSIBLE ANSWER AND IT WAS STILL WRONG, which is the sentence
6289
+ #: worth keeping. Its argument was that a handle can be renamed back or a suspension lifted, so
6290
+ #: the verdict should expire. But nothing about US changes on day 31 β€” the only new information
6291
+ #: is a guess that the world moved β€” so the timer buys one paid vendor call per dead handle per
6292
+ #: month, forever, on a row nobody is looking at, and reports it as "1 blocked". A tenant with a
6293
+ #: hundred stale handles pays a hundred times a month to be told the same thing.
6294
+ #: ⭐ WHAT RE-ARMS IT IS A HUMAN, and there are two doors:
6295
+ #: 1. the KEY is `(platform, handle)` β€” so correcting a typo re-arms IMMEDIATELY, with no
6296
+ #: wiring at all, because the corrected handle simply is not the one we recorded; and
6297
+ #: 2. `clear_gone()` β€” called when the profile CELL is written, so re-typing the SAME handle
6298
+ #: (a person saying "try it again") also re-arms.
6299
+ #: ⚠ The run keeps NAMING the skipped handles every time, not only on the run that discovered
6300
+ #: them: a tombstone the owner cannot see is just a disappearance.
6301
  #: A queued profile snapshot the vendor never finishes is dropped after this, with a note. An
6302
  #: unbounded pending list is the forever-loop this whole change exists to remove, wearing the
6303
  #: opposite mask ([[gate-answers-the-wrong-question]]).
 
6374
  return f"{plat}:{str(handle or '').strip().lstrip('@').lower()}"
6375
 
6376
 
6377
+ def clear_gone(rt, table_key, handle, row=None):
6378
+ """R9's re-arm door: forget the "this account does not exist" verdict for ONE handle.
6379
+
6380
+ Returns the number of automations whose memory was changed β€” 0 is the ordinary answer and is
6381
+ not an error, because most cell edits are not on a dead handle.
6382
+
6383
+ ⭐⭐ WHY THIS IS A PUBLIC FUNCTION IN THE ENGINE RATHER THAN A LOOKUP IN THE ROW WRITER.
6384
+ The verdict lives on the AUTOMATION (`state.enrichNotFound`), not on the row β€” it has to,
6385
+ because it is a fact about what a run learned and paid for. But the thing that re-arms it is a
6386
+ ROW event, and the row writer must not need to know the shape of an automation's state to
6387
+ trigger it. So the seam is one call with the three things the writer already has, and every
6388
+ walk of `all_definitions` stays on this side of the fence.
6389
+ ⚠ IT MATCHES THE SAME WAY THE SKIP DOES. `_gone_key` is the one implementation of "which
6390
+ handle is this", so a verdict can never be recorded under a key this cannot find
6391
+ ([[one-evaluator-per-question]]).
6392
+
6393
+ β›” WITHOUT ITS CALLER THIS IS INERT, AND THAT IS RECORDED RATHER THAN ASSUMED. R9's first
6394
+ door β€” correcting a typo β€” works with no wiring at all, because the key IS the handle and a
6395
+ different handle is simply not the one we recorded. This second door only opens when the row
6396
+ write path calls it, which lives in `routes_tables.patch_row` (another session's fence).
6397
+ Until that line lands, re-typing the SAME dead handle stays skipped
6398
+ ([[flag-shipped-without-its-writer]] β€” named on purpose, so it is not discovered later).
6399
+ """
6400
+ key = _gone_key(row or {}, handle)
6401
+ if not str(handle or "").strip():
6402
+ return 0
6403
+ changed = 0
6404
+ for auto_id, defn in (all_definitions(rt) or {}).items():
6405
+ if str((defn.get("config") or {}).get("targetTable") or "") != str(table_key):
6406
+ continue
6407
+ known = dict(((defn.get("state") or {}).get("enrichNotFound")) or {})
6408
+ if key not in known:
6409
+ continue
6410
+ known.pop(key, None)
6411
+ # ⚠ `or None` β€” an empty dict must clear the key rather than store `{}`, which is what
6412
+ # every other state writer here does and what keeps a definition from growing a graveyard
6413
+ # of empty maps.
6414
+ set_state(rt, str(auto_id), {"enrichNotFound": known or None})
6415
+ changed += 1
6416
+ return changed
6417
+
6418
+
6419
  def enrich_selection(rt, table_key, cfg, profile_key, today=None, gone=None):
6420
  """⭐ 2026-08-07 (owner ruling) β€” WHICH records this enrich step spends on, in order.
6421
 
 
6553
  if not raw_handle:
6554
  blank_handle += 1
6555
  continue
6556
+ # β›” R9 β€” NO EXPIRY. There is deliberately no date arithmetic here any more: a verdict is
6557
+ # a verdict until a human edits the cell. An `at` stamp is still STORED (it is what the
6558
+ # owner reads to know when we last paid to be told this), it is simply not a clock.
6559
+ if isinstance((gone or {}).get(_gone_key(r, raw_handle)), dict):
6560
+ skipped_gone.append(raw_handle)
6561
+ continue
6562
  if cooling:
6563
  since = _days_since(r.get("enriched_at"), today=today)
6564
  if since is not None and since < days:
 
6573
  notes.append(f"{cooled} skipped as enriched in the last {days} days")
6574
  if skipped_gone:
6575
  shown = ", ".join(f"@{h}" for h in skipped_gone[:5])
6576
+ # ⚠ THE SENTENCE IS THE FEATURE. It must name the handles AND the two things a person can
6577
+ # do, because nothing else will ever retry them β€” under R9 this note is the only path
6578
+ # back from a tombstone, so a vaguer version would strand the row permanently.
6579
  notes.append(f"{len(skipped_gone)} skipped because Instagram has no such account "
6580
  f"({shown}{', …' if len(skipped_gone) > 5 else ''}) β€” delete the row or "
6581
+ f"correct the handle; they are not retried automatically")
6582
  if blank_handle:
6583
  notes.append(f"{blank_handle} skipped with no handle")
6584
  if len(picked) < quota and (cooled or blank_handle or skipped_gone or ordered):
 
6814
  res = pull_profile(handle_raw,
6815
  max_posts=cfg.get("maxPosts") or DEFAULT_POSTS_PER_PULL,
6816
  log=log,
 
 
6817
  post_metrics=bool(cfg.get("postMetrics")),
6818
  comment_metrics=bool(cfg.get("commentMetrics")),
6819
  pending_metrics=(enrich["pending"]
 
6928
  max_posts = 24
6929
  act = {"id": "act_enrich", "kind": "enrich_instagram", "enabled": True, "when": None,
6930
  "config": {"profileField": bound,
 
 
6931
  "postMetrics": bool(cfg.get("postMetrics")),
6932
  "commentMetrics": bool(cfg.get("commentMetrics")),
6933
  "dryRun": bool(cfg.get("dryRun")),
 
7217
  """
7218
  tasks = _pending_profile_tasks(defn)
7219
  if not tasks:
7220
+ return ("ok", "No pending profile reads.", {}, [], {"source": "idle"})
7221
  step(f"Collecting {len(tasks)} deferred profile read{'' if len(tasks) == 1 else 's'}")
7222
  remaining, patches, affected, notes = [], {}, [], []
7223
  acc = {"snaps": [], "posts": [], "psnaps": [], "comments": [], "pending": [],
 
7236
  acc["notes"].append(note)
7237
  # ⭐ `failed` = the vendor finished, collected nothing, and blamed the TARGET. On a
7238
  # profile request that means the account could not be reached at all, so the verdict
7239
+ # is remembered and the selection stops re-buying it (R9: permanently, until a
7240
+ # human edits the handle cell β€” see `clear_gone`).
7241
  # ⚠ `done`-with-zero is NOT remembered: "we found no matches" is a statement about
7242
  # the query, and turning it into "this account does not exist" would silently retire
7243
  # live handles.
 
7309
  f"; {waiting} still building" if waiting else ""])
7310
  state = "ok" if ready and not closed else "partial"
7311
  return (state, head + tail, counts, affected,
7312
+ {"source": "ok" if ready else "partial", "write": "ok" if ready else "idle"})
7313
 
7314
 
7315
  def _write_collected_metric_rows(rt, defn, username, idents, snapshots, comments, log):
 
7359
  """Collect deferred paid Post/Reel/Comment snapshots; never launches a new scrape request."""
7360
  tasks = _pending_metric_tasks(defn)
7361
  if not tasks:
7362
+ return ("ok", "No pending post-engagement snapshots.", {}, [], {"capture_posts": "idle"})
7363
  step(f"Collecting {len(tasks)} post-engagement batch{'' if len(tasks) == 1 else 'es'}")
7364
  remaining, idents, snapshots, comments = [], [], [], []
7365
+ ready, waiting, closed, run_notes = 0, 0, 0, []
7366
  for task in tasks:
7367
  # ⭐⭐ 2026-08-09 β€” ASK THE STATUS DOCUMENT, NOT THE ROWS. `building` used to be
7368
  # `not rows or …`, so a snapshot the vendor had FINISHED with zero records was re-queued
 
7374
  # β›” CLOSED, NOT RE-QUEUED. The vendor is finished and there is nothing to collect;
7375
  # keeping the entry would be a pending task that can never resolve.
7376
  closed += 1
7377
+ run_notes.append(f"{task['influencer']}: {_s(empty_note, 160)}")
7378
  continue
7379
  payload, note = bd_call(f"{BD_PATH_SNAPSHOT}/{task['snapshotId']}", {"format": "json"})
7380
  rows = _bd_rows(payload) if not note else []
 
7398
  post["influencer_key"] = task["influencer"]
7399
  mapped_posts.append(post)
7400
  if mapped_posts:
7401
+ # ⭐⭐ 2026-08-09 β€” THE VIEWS TOP-UP RUNS HERE TOO, and its absence is why the
7402
+ # owner's `theresalearns` run filled every column except Views.
7403
+ #
7404
+ # β›” Bright Data is DECLARED INCAPABLE of `ig_post_views` (`providers.py`), so a
7405
+ # Posts row physically cannot carry a view count β€” MEASURED on the stored
7406
+ # payloads: 12 rows, `content_type: "Reel"`, and no view/play key in any of them.
7407
+ # Views only ever comes from the Apify capability. That top-up lived INSIDE
7408
+ # `pull_profile_bd`, so it ran only when the Posts scrape answered within the wait
7409
+ # budget; when the batch deferred β€” which is routine, and what happened here β€” the
7410
+ # rows came back through THIS function and Apify was never asked.
7411
+ # β‡’ `top_up_views` is now one function with two callers rather than a copy, so
7412
+ # the inline and deferred paths cannot answer this differently again.
7413
+ # ⚠ It mutates `mapped_posts` in place and must run BEFORE `capture_rows`, which
7414
+ # is what freezes the values into the post + snapshot rows.
7415
+ v_note = top_up_views(mapped_posts, log=log)
7416
+ if v_note:
7417
+ run_notes.append(f"@{task['influencer']}: {v_note}")
7418
  _unused, post_rows, metric_rows, embedded = capture_rows(
7419
  {"state": "ok", "profile": {"username": task["influencer"]},
7420
  "posts": mapped_posts, "comments": [], "via": "brightdata:deferred"}, pulled)
 
7439
  # other call sites disagree about the shape of a run β€” and `_commit_run` already drops
7440
  # non-numeric count values, so a pop that is ever missed degrades to today's behaviour rather
7441
  # than to a crash.
7442
+ if run_notes:
7443
+ counts[RUN_NOTES_KEY] = run_notes
7444
  # ⚠ THE EMPTY ONES ARE NAMED, not silently dropped. A batch that finished with no records is
7445
  # a real outcome the tenant paid for and it must read as an answer, not as a disappearance.
7446
  tail = (f"; {closed} finished with nothing to collect" if closed else "")
 
7448
  return ("partial",
7449
  f"{ready} post-engagement batch{'' if ready == 1 else 'es'} collected; "
7450
  f"{waiting} still building and will be collected automatically{tail}",
7451
+ counts, [], {"capture_posts": "partial", "write": "ok"})
7452
  return ("ok", f"{ready} post-engagement batch{'' if ready == 1 else 'es'} collected{tail}",
7453
+ counts, [], {"capture_posts": "ok", "write": "ok"})
7454
 
7455
 
7456
  def ai_decide(rt, defn, act, row, row_id="", log=print):
 
7839
 
7840
 
7841
  #: The fns by family β€” how a value is folded, and what a blank means in each.
7842
+ _ROLLUP_NUM_FNS = frozenset({"sum", "average", "stdev", "min", "max"})
7843
  _ROLLUP_BOOL_FNS = frozenset({"and", "or", "xor"})
7844
+ #: The count/order family β€” folded by their own arms above the lanes.
7845
+ _ROLLUP_SEQ_FNS = frozenset({"countall", "counta", "count", "latest"})
7846
+ #: The text family. β›” IT IS A NAMED SET NOW AND IT USED TO BE THE FALL-THROUGH, which is how
7847
+ #: wave 28 nearly shipped a wrong number that looked like data: `stdev` validated and STORED
7848
+ #: (`core.user_tables.ROLLUP_FNS`) a commit before this function learned it, and an fn no arm
7849
+ #: claims fell past both lanes into the join below β€” so a "Std deviation" column rendered
7850
+ #: `"137684, 19561, 8123"`. Filled, plausible, and not a statistic. An unrecognised fn now
7851
+ #: returns `""` ([[gate-answers-the-wrong-question]]: blank is the honest answer to a question
7852
+ #: nothing can answer; a comma-joined list is a different question's answer wearing this label).
7853
+ _ROLLUP_TEXT_FNS = frozenset({"concatenate", "arrayjoin", "arraycompact", "arrayunique"})
7854
+ #: ⭐⭐ WHAT THIS FOLD ACTUALLY IMPLEMENTS, DERIVED FROM THE ARMS RATHER THAN RESTATED.
7855
+ #: `verify_automation` asserts this is IDENTICAL to `core.user_tables.ROLLUP_FNS` name for name
7856
+ #: (contract C1's parity leg), so the validator can never again accept a function the fold cannot
7857
+ #: compute β€” in EITHER direction. A hand-listed copy in the gate would have gone green on the
7858
+ #: defect above, because the defect was that the two lists already disagreed.
7859
+ ROLLUP_FOLD_FNS = frozenset(_ROLLUP_NUM_FNS | _ROLLUP_BOOL_FNS | _ROLLUP_SEQ_FNS
7860
+ | _ROLLUP_TEXT_FNS)
7861
  _ROLLUP_TEXT_FNS = frozenset({"concatenate", "arrayjoin", "arraycompact", "arrayunique"})
7862
  #: What `arrayjoin` puts between values. Airtable uses ", "; `concatenate` uses nothing.
7863
  _ROLLUP_JOIN = ", "
 
7901
  return (raw.lower(),)
7902
 
7903
 
7904
+ def _sample_stdev(nums):
7905
+ """SAMPLE standard deviation (n-1) of `nums`, or None under two values.
7906
+
7907
+ ⭐ ONE IMPLEMENTATION, TWO READERS, and that is the point of lifting four lines into a
7908
+ function: `_rollup_fold` RENDERS this number into a "Std deviation" column and
7909
+ `_rollup_ref_threshold` COMPARES rows against it inside a `sigmas` condition. A second copy
7910
+ would let a column and the filter beside it disagree about the same word on the same set β€”
7911
+ the exact drift `top_up_views` was extracted to prevent one module over
7912
+ ([[one-evaluator-per-question]]).
7913
+ β›” None means UNANSWERABLE and no caller may read it as 0.
7914
+ """
7915
+ if len(nums) < 2:
7916
+ return None
7917
+ mean = sum(nums) / len(nums)
7918
+ return (sum((n - mean) ** 2 for n in nums) / (len(nums) - 1)) ** 0.5
7919
+
7920
+
7921
+ def _rollup_ref_threshold(rows, field, sigmas):
7922
+ """`mean + sigmas*stdev` of `field` over `rows` β€” a statistic OF THE SCOPED SET (contract C1).
7923
+
7924
+ β›” `rows` MUST already be the scoped window: ranked by `sortBy`, deduped, and cut by `limit`,
7925
+ and NOT yet filtered by the conditions this threshold feeds. That order is the contract
7926
+ (`core.user_tables.ROLLUP_REF_OPS`'s note says so in as many words) and both other orders
7927
+ produce a plausible number: computing it before `limit` answers "2 sigma of everything this
7928
+ account ever posted" under a column that says "of the last 10", and computing it after the
7929
+ filter makes the threshold depend on the rows it is choosing β€” a definition that chases
7930
+ itself.
7931
+
7932
+ Returns None when the window cannot answer β€” fewer than two numeric values in `field`.
7933
+ β›” THE CALLER MUST DROP THE ROW, NOT KEEP IT. "Beyond 2 sigma of one post" is not a question
7934
+ with a permissive answer; letting an unanswerable leaf pass everything would silently turn
7935
+ "the outliers" into "all of them", which is this module's worst failure mode wearing a filter.
7936
+ """
7937
+ nums = [n for n in (_lane_num((r or {}).get(field)) for r in rows) if n is not None]
7938
+ sd = _sample_stdev(nums)
7939
+ if sd is None:
7940
+ return None
7941
+ return (sum(nums) / len(nums)) + float(sigmas) * sd
7942
+
7943
+
7944
  def _rollup_fold(fn, values):
7945
  """`values` (raw cells, in the order the window kept them) β†’ the aggregate, as a STRING.
7946
 
 
7968
  nums = [n for n in (_lane_num(v) for v in values) if n is not None]
7969
  if not nums:
7970
  return ""
7971
+ if fn == "stdev":
7972
+ # ⭐⭐ SAMPLE standard deviation (n-1), and the divisor is a ruling, not a preference
7973
+ # (C1 / `core.user_tables.ROLLUP_FNS`'s note): a rollup folds the rows that happen to
7974
+ # be LINKED, which is a sample of an account's posting history and not its entirety.
7975
+ # ⚠ Fixture to check a refactor against: [2,4,4,4,5,5,7,9] -> 2.14. The POPULATION
7976
+ # form gives 2.00 on the same input, so a test that ever reads 2.00 has silently
7977
+ # switched divisors.
7978
+ # β›” FEWER THAN TWO VALUES IS "" AND NEVER "0". n-1 = 0 would divide by zero, but the
7979
+ # honest reason is upstream of the arithmetic: one measurement has no spread to
7980
+ # report, and a 0 in a "Std deviation" column reads as PERFECT CONSISTENCY β€” the
7981
+ # single most confident thing this column can say, asserted from a single row. Same
7982
+ # law as the blank `sum`, and it bites harder here.
7983
+ out = _sample_stdev(nums)
7984
+ if out is None:
7985
+ return ""
7986
+ else:
7987
+ out = (sum(nums) if fn == "sum" else min(nums) if fn == "min"
7988
+ else max(nums) if fn == "max" else sum(nums) / len(nums))
7989
  return f"{out:.0f}" if float(out).is_integer() else f"{out:.2f}"
7990
  if fn in _ROLLUP_BOOL_FNS:
7991
  # A checkbox cell is '1'/'' in this product, so truth is "non-blank and not a zero".
 
7995
  hit = (all(flags) if fn == "and" else any(flags) if fn == "or"
7996
  else sum(1 for f in flags if f) % 2 == 1)
7997
  return "1" if hit else ""
7998
+ # --- the text family. β›” CLAIMED BY NAME, NEVER BY FALL-THROUGH β€” see `_ROLLUP_TEXT_FNS`.
7999
+ # An fn no arm above recognises returns "" rather than a comma-joined dump of every value,
8000
+ # which is the shape a not-yet-implemented aggregate wore for one commit of wave 28.
8001
+ if fn not in _ROLLUP_TEXT_FNS:
8002
+ return ""
8003
  vals = [str(v).strip() for v in values if str(v or "").strip() != ""]
8004
  if fn == "arrayunique":
8005
  seen, uniq = set(), []
 
8047
  return idx
8048
 
8049
 
8050
+ def _rollup_condition_matches(row, condition, field_types, ref_value=None):
8051
+ """Evaluate one Airtable-style linked-record condition against a candidate row.
8052
+
8053
+ `ref_value` is the threshold a `ref: {sigmas}` leaf compares against, already computed by the
8054
+ caller over the SCOPED set (`_rollup_ref_threshold`). β›” It is passed IN rather than computed
8055
+ here because this function sees one row and the statistic is a property of the whole window β€”
8056
+ a version that reached for the set from inside would be recomputing the same mean once per
8057
+ row, and would have to be handed the window anyway.
8058
+ ⚠ `None` means the window could not answer, and the leaf then matches NOTHING. See the
8059
+ threshold helper for why the permissive reading is the dangerous one.
8060
+ """
8061
  field = str((condition or {}).get("field") or "")
8062
  op = str((condition or {}).get("op") or "")
8063
  raw = (row or {}).get(field)
 
8066
  return text == ""
8067
  if op == "is_not_empty":
8068
  return text != ""
8069
+ if (condition or {}).get("ref") is not None:
8070
+ # β›” NUMERIC LANE ONLY, BOTH SIDES. The validator already restricts `ref` to the ordering
8071
+ # ops, and a row whose cell is blank or unparseable has no position relative to a computed
8072
+ # threshold β€” it is not "below" it. Dropping it is the same partition law `_sort_key`
8073
+ # follows: unrankable is not a rank ([[sentinel-in-a-sort-key]]).
8074
+ left_num = _lane_num(text)
8075
+ if ref_value is None or left_num is None:
8076
+ return False
8077
+ return ((op == "gt" and left_num > ref_value)
8078
+ or (op == "gte" and left_num >= ref_value)
8079
+ or (op == "lt" and left_num < ref_value)
8080
+ or (op == "lte" and left_num <= ref_value))
8081
  wanted = str((condition or {}).get("value") or "").strip()
8082
  if op == "contains":
8083
  return wanted.casefold() in text.casefold()
 
8177
  # ⚠ A rollup whose link field does not exist (renamed, deleted) resolves to NOTHING
8178
  # and therefore to a blank cell β€” never to a stale number. A column that keeps
8179
  # printing yesterday's answer after its input is gone is the worst of the options.
8180
+ # ⭐⭐ WAVE 28 / CONTRACT C1 β€” SCOPE FIRST, FILTER SECOND, AND THE TWO USED TO BE THE
8181
+ # OTHER WAY ROUND. The conditions block stood HERE, above the ranking, so
8182
+ # "last 10 posts where views > X" meant *the 10 most recent of the posts over X*
8183
+ # rather than *the ones over X among the last 10* β€” two different windows wearing one
8184
+ # sentence. Harmless while a threshold was a literal; incoherent the moment a
8185
+ # threshold is a statistic OF the window, because the set being described and the set
8186
+ # doing the describing would be different sets.
8187
+ # β›” THIS ORDER IS THE CONTRACT, not an implementation choice: `core.user_tables`'s
8188
+ # `ROLLUP_REF_OPS` note states it ("the scope picks the window, THEN the threshold is
8189
+ # computed over that window, THEN the conditions filter it") and the validator half
8190
+ # was written against it.
8191
+ # ⚠ IT IS A BEHAVIOUR CHANGE FOR EXACTLY ONE SHAPE: a stored rollup carrying BOTH
8192
+ # `conditions` AND `limit`. No shipped preset does (measured across `odoo_relational`
8193
+ # and the IG presets β€” the one preset with conditions, `_OPEN_ONLY`, is a `countall`
8194
+ # with no limit), so the blast radius is user-built rollups only.
8195
  sort_by = str(bag.get("sortBy") or "")
8196
  if sort_by:
8197
  ftype = (linked_types.get(lk_key) or {}).get(sort_by, "text")
 
8221
  limit = int(bag.get("limit") or 0)
8222
  if limit:
8223
  hits = hits[:limit]
8224
+ # --- the window is now FIXED, so a set-statistic threshold has a set to be about.
8225
+ conditions = list(bag.get("conditions") or [])
8226
+ if conditions:
8227
+ # ⚠ RESOLVED ONCE PER LEAF, NOT ONCE PER ROW. The threshold is a property of the
8228
+ # window; computing it inside `matches` would recompute the same mean for every
8229
+ # candidate and β€” worse β€” would invite computing it over a set that the filter is
8230
+ # already shrinking underneath it.
8231
+ # ⚠ `.get("sigmas", 0.0)`, never `... or 0.0` β€” a legitimate `sigmas: 0` ("beyond
8232
+ # the mean") is falsy, and the `or` spelling would silently rewrite it to the same
8233
+ # number by accident. It reads identically and is right for the wrong reason,
8234
+ # which is how it survives a review.
8235
+ refs = [_rollup_ref_threshold(
8236
+ [r for _i, r in hits], str(c.get("field") or ""),
8237
+ (c.get("ref") or {}).get("sigmas", 0.0))
8238
+ if isinstance(c, dict) and c.get("ref") is not None else None
8239
+ for c in conditions]
8240
+ matches = lambda pair: [
8241
+ _rollup_condition_matches(pair[1], condition,
8242
+ linked_types.get(lk_key) or {}, ref_value=ref)
8243
+ for condition, ref in zip(conditions, refs)]
8244
+ if str(bag.get("conditionConj") or "and") == "or":
8245
+ hits = [pair for pair in hits if any(matches(pair))]
8246
+ else:
8247
+ hits = [pair for pair in hits if all(matches(pair))]
8248
  src = str(bag.get("field") or "")
8249
  want = _rollup_fold(str(bag.get("fn") or ""),
8250
  [r.get(src) for _i, r in hits] if src else [1] * len(hits))
 
9253
  host = urlparse(str(cfg.get("url") or "")).hostname or "the page"
9254
  body = f"read {host} and upsert rows into {cfg.get('targetTable') or 'a new database'}"
9255
  elif kind == "field_instagram":
9256
+ # ⚠ NO RUNG CLAUSE (R5). There is one way to capture a profile now, so ", exact counts
9257
+ # first" / ", anonymous only" described a choice that no longer exists.
9258
+ body = f"capture Instagram profiles for {cfg.get('targetTable') or 'the database'}"
9259
  elif kind == "discover_instagram":
9260
  body = (f"search Instagram for up to {cfg.get('recordsLimit') or 0} profiles into "
9261
  f"{cfg.get('targetTable') or DISCOVER_TABLE}")
api/connectors_ig.py CHANGED
@@ -34,8 +34,8 @@ a caller reached first. The server imports the engine first and would have worke
34
  probe doing `import connectors_ig` would have got a half-built engine and an AttributeError from
35
  inside a vendor call. **An intermittent-by-import-order failure is the worst shape available**, so
36
  every reach-back is a `from automation_engine import …` INSIDE the function that needs it, where
37
- the module is always fully built. There are exactly eight β€” `_ig_get`, `_bd_why`, `bd_call`,
38
- `apify_posts`, `_tag_metric_deferrals`, `pull_profile_bd`, `pull_profile`, `_paginate` β€” each
39
  marked `# lazy β€” see the module header`, and they reach for four kinds of thing:
40
 
41
  fetch / fetch_json / Refused the SSRF-guarded HTTP rail (the scrape runner shares it)
@@ -54,7 +54,6 @@ allowed to know.
54
  """
55
  from __future__ import annotations
56
 
57
- import datetime as _dt
58
  import hashlib
59
  import json
60
  import os
@@ -68,11 +67,16 @@ import providers
68
 
69
 
70
  # ---------------------------------------------------------------------------------------------
71
- # THE ANONYMOUS LADDER'S READERS ($0, no key of any kind)
72
  # ---------------------------------------------------------------------------------------------
73
-
74
- IG_APP_ID = "936619743392459" # the public web client id the browser itself sends
75
- IG_MAX_RETRIES = 2
 
 
 
 
 
76
 
77
  def ig_handle(url):
78
  """The handle out of a profile URL (or a bare handle). '' when it is not one."""
@@ -90,25 +94,6 @@ def ig_handle(url):
90
  return re.sub(r"[^A-Za-z0-9._]", "", first)[:40]
91
 
92
 
93
- def _ig_get(url, headers=None, timeout=20.0):
94
- """One anonymous GET with 429 backoff. Returns `(status, body_bytes, note)`."""
95
- from automation_engine import PACE_SECONDS, Refused, fetch # lazy β€” see the module header
96
- delay = PACE_SECONDS
97
- for attempt in range(IG_MAX_RETRIES + 1):
98
- try:
99
- status, _final, body = fetch(url, timeout=timeout, headers=headers, max_kb=4096)
100
- except Refused:
101
- raise
102
- except Exception as e: # noqa: BLE001
103
- return 0, b"", f"{type(e).__name__}: {str(e)[:120]}"
104
- if status == 429 and attempt < IG_MAX_RETRIES:
105
- time.sleep(delay)
106
- delay *= 2 # exponential, never a tight retry
107
- continue
108
- return status, body, ""
109
- return 429, b"", "rate limited after retries"
110
-
111
-
112
  def _ig_int(v):
113
  try:
114
  return int(v)
@@ -131,104 +116,6 @@ def _ig_zero_is_blank(v):
131
  return None if n == 0 else n
132
 
133
 
134
- def _profile_from_web_api(node, handle):
135
- return {
136
- "username": node.get("username") or handle,
137
- "full_name": node.get("full_name") or "",
138
- "bio": node.get("biography") or "",
139
- "followers": _ig_int((node.get("edge_followed_by") or {}).get("count")),
140
- "following": _ig_int((node.get("edge_follow") or {}).get("count")),
141
- "posts_count": _ig_int((node.get("edge_owner_to_timeline_media") or {}).get("count")),
142
- "verified": "1" if node.get("is_verified") else "",
143
- }
144
-
145
-
146
- def _posts_from_edges(edges):
147
- out = []
148
- for e in edges or []:
149
- n = (e or {}).get("node") or {}
150
- code = n.get("shortcode")
151
- if not code:
152
- continue
153
- caption = ""
154
- for ce in ((n.get("edge_media_to_caption") or {}).get("edges") or []):
155
- caption = ((ce or {}).get("node") or {}).get("text") or ""
156
- break
157
- ts = n.get("taken_at_timestamp")
158
- out.append({
159
- "shortcode": code,
160
- "posted_at": (_dt.datetime.fromtimestamp(ts).strftime("%Y-%m-%d %H:%M")
161
- if isinstance(ts, (int, float)) else ""),
162
- "type": ("video" if n.get("is_video") else
163
- "carousel" if n.get("__typename") == "GraphSidecar" else "image"),
164
- "caption": caption,
165
- "url": f"https://www.instagram.com/p/{code}/",
166
- "likes": _ig_int((n.get("edge_liked_by") or n.get("edge_media_preview_like")
167
- or {}).get("count")),
168
- "comments": _ig_int((n.get("edge_media_to_comment") or {}).get("count")),
169
- })
170
- return out
171
-
172
-
173
- _OG_COUNTS = re.compile(
174
- r"([\d.,KMkm]+)\s+Followers?,\s*([\d.,KMkm]+)\s+Following,\s*([\d.,KMkm]+)\s+Posts?")
175
-
176
-
177
- def _loose_count(txt):
178
- """'1.2M' / '12,345' -> int. IG's og:description abbreviates, so the snapshot is honest
179
- about being approximate rather than pretending to a precision it does not have."""
180
- t = str(txt or "").strip().replace(",", "")
181
- mult = 1
182
- if t[-1:].lower() == "k":
183
- mult, t = 1_000, t[:-1]
184
- elif t[-1:].lower() == "m":
185
- mult, t = 1_000_000, t[:-1]
186
- try:
187
- return int(float(t) * mult)
188
- except Exception:
189
- return None
190
-
191
- # ---------------------------------------------------------------------------------------------
192
- # BRIGHT DATA β€” THE PAID RUNG (owner ruling R1 / wave-20 D-21, contract C4)
193
- # ---------------------------------------------------------------------------------------------
194
- # WHY A VENDOR AT ALL, stated once so nobody re-opens it: the anonymous ladder above is honest but
195
- # it is ALSO geographically fenced β€” Instagram blocks datacenter egress, so from the HF Space or a
196
- # Lambda the ladder degrades to `blocked` and the whole automation becomes a thing that only works
197
- # on the owner's desk. The vendor takes the egress risk, returns EXACT counts (not
198
- # og:description's "204K") and reaches post-level data the anonymous surface has closed.
199
- #
200
- # ⚠ WHY THIS VENDOR AND NOT THE PREVIOUS ONE. HikerAPI was crypto-only with no legal entity β€”
201
- # no invoice, no counterparty, nothing that can go on a subprocessor list at the EU/enterprise
202
- # gates. Bright Data is nameable, litigated Meta and won on logged-off public scraping, and runs
203
- # KYC. That is a POSTURE purchase, not a feature one; the price is ~2.5Γ— and it is the right
204
- # trade. Full comparison: `.claude/wiki/research/instagram-capture.md` Β§2a.
205
- #
206
- # β›” THE FOUR RAILS TRANSFER UNCHANGED (contract C4 β€” they were written vendor-agnostic):
207
- # 1. **FAIL-CLOSED on the key.** No `AIOS_BRIGHTDATA_KEY` β‡’ this rung refuses with an honest
208
- # blocked ATTEMPT and the anonymous ladder still runs. Never a crash, never a silent skip.
209
- # 2. **The key is never logged, never in a URL, never in an exception.** It travels in the
210
- # `Authorization: Bearer` header only, and every error below quotes a STATUS, never the
211
- # request. A secret that reaches a log line has leaked to everyone who can read logs.
212
- # 3. **The base URL is guarded like any other.** `AIOS_BRIGHTDATA_BASE` is env-provided, so it
213
- # goes through the same SSRF rail β€” an env var pointed at the metadata endpoint would
214
- # otherwise be a credential-bearing request to it.
215
- # 4. **A rung failure is a RUNG failure.** `bd_call` swallows its own `Refused`/transport
216
- # errors, because `run_now` turns a `Refused` into a whole-run error β€” and a mistyped base
217
- # URL must not kill a run whose anonymous rung would have answered.
218
- # 5. **Exact β‡’ NO `approx` TAG.** That tag is the anonymous rung's rounding disclosure.
219
- #
220
- # ⚠ THE KEY IS READ LAZILY, INSIDE THE CALL. A module-level snapshot would be taken at import,
221
- # which makes the "key absent β‡’ blocked, not crash" gate untestable (nothing could flip it between
222
- # checks) and would also mean a deploy that sets the secret after import silently has no key.
223
- #
224
- # β›” TWO SNAPSHOT NAMESPACES THAT 404 EACH OTHER (measured; the trap that costs an hour):
225
- # the SCRAPER `/datasets/v3/trigger` + `/v3/scrape` -> `sd_…` read at `/datasets/v3/…`
226
- # the CORPUS `/datasets/filter` (⚠ NO `/v3/`) -> `snap_…` read at `/datasets/…`
227
- # Cross them and you get a flat "404 Snapshot does not exist" about a snapshot that is alive.
228
-
229
- #: Pinned from live probes against `api.brightdata.com` (2026-08-04 and 2026-08-05) β€” see
230
- #: `.claude/wiki/research/instagram-capture.md` Β§2c/Β§2d for the recipe and its provenance tags.
231
- #: They live in ONE named block so a vendor rename is a one-line fix rather than a hunt.
232
  BD_BASE_DEFAULT = "https://api.brightdata.com"
233
  BD_DS_PROFILES = "gd_l1vikfch901nx3by4" # Instagram – Profiles. 36 fields, 620M records
234
  BD_DS_POSTS = "gd_lk5ns7kz21pck8jpis" # Instagram – Posts. 43 fields
@@ -1213,6 +1100,83 @@ def _tag_metric_deferrals(items, start, kind, influencer_key):
1213
  item["requestedAt"] = _iso()
1214
 
1215
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1216
  def pull_profile_bd(url, max_posts=None, post_metrics=False,
1217
  comment_metrics=False, log=print, pending_metrics=None,
1218
  pending_profile=None):
@@ -1380,61 +1344,7 @@ def pull_profile_bd(url, max_posts=None, post_metrics=False,
1380
  elif metric_notes:
1381
  note = f"some post metrics were unavailable: {'; '.join(metric_notes)}"
1382
 
1383
- # ⭐⭐ THE VIEWS TOP-UP β€” capability routing, not provider failover (owner ruling
1384
- # 2026-08-08, `providers.py`). Bright Data has already answered profile + likes + comments
1385
- # above and is declared INCAPABLE of `ig_post_views`, so the chain for that ONE capability
1386
- # resolves to Apify and only the video permalinks are re-bought. Re-running the whole
1387
- # profile on the second vendor would pay twice for the 90% that already worked, which at
1388
- # thousands of tenants x twelve posts is the entire bill.
1389
- # ⚠ VIDEO ONLY. A carousel or image has no view count, so sending it would buy a record
1390
- # that can only come back blank.
1391
- video_urls = [p["url"] for p in posts
1392
- if p.get("type") == "video" and p.get("url")]
1393
- if video_urls:
1394
- def _views_work(prov, _urls=tuple(video_urls)):
1395
- if prov.key == "apify":
1396
- return apify_posts(list(_urls))
1397
- # A provider in the chain with no runner here is a configuration error, not a
1398
- # vendor outage β€” say so rather than returning an empty list that reads as "the
1399
- # vendor had nothing".
1400
- return None, f"no {prov.key} runner is wired for ig_post_views"
1401
-
1402
- got_views, attempts = providers.run(
1403
- "ig_post_views", _views_work,
1404
- # β›” THE SATISFIED PREDICATE IS THE FALLBACK TRIGGER. Rows that come back with
1405
- # every `views` blank are a FAILURE for this capability even at HTTP 200 β€” which
1406
- # is exactly how Bright Data behaves, and why a chain that only caught exceptions
1407
- # would never have reached a second provider.
1408
- satisfied=lambda rows: bool(rows) and any(r.get("views") for r in rows),
1409
- log=log)
1410
- for row in (got_views or []):
1411
- target = next((p for p in posts
1412
- if p.get("shortcode") == row.get("shortcode")), None)
1413
- # β›” TAKE ONLY THE CAPABILITY THAT WAS ASKED FOR. This provider also returns
1414
- # likes/comments/caption, and letting them land would silently switch the source
1415
- # of columns Bright Data already answered β€” the schema would be stable but the
1416
- # PROVENANCE would flip halfway through a row.
1417
- if target and row.get("views") not in (None, ""):
1418
- target["views"] = row["views"]
1419
- # ⭐ LOCATION RIDES ALONG FOR FREE, AND ONLY INTO A GAP (2026-08-08).
1420
- # MEASURED: Bright Data's `location_details` is RICHER when present β€” it carries
1421
- # real coordinates (`lat -6.9246, lng 106.9292, name "Sukabumi"`) which Apify does
1422
- # not return at all β€” but it is populated on only 44 of 227 posts, and on
1423
- # `DblAkEbv0ry` it returned nothing while Apify returned "Jakarta, Indonesia".
1424
- # So Bright Data stays the source and this fills only what it left BLANK.
1425
- # β›” THE COST ARGUMENT IS WHY IT IS HERE AND NOT ITS OWN CAPABILITY: this Apify
1426
- # record has ALREADY been bought for the view count and carries `locationName` in
1427
- # the same payload. Routing location as a separate capability would buy a second
1428
- # record for a field that is already sitting in this response.
1429
- if target and not str(target.get("tagged_location") or "").strip() and str(row.get("tagged_location") or "").strip():
1430
- target["tagged_location"] = row["tagged_location"]
1431
- used = next((a.provider for a in attempts if a.ok), "")
1432
- if used:
1433
- note = "; ".join(x for x in (note, f"view counts via {used}") if x)
1434
- elif attempts:
1435
- note = "; ".join(x for x in (
1436
- note, "no provider could supply view counts: "
1437
- + "; ".join(f"{a.provider} {a.note}" for a in attempts)) if x)
1438
 
1439
  # ⭐⭐ DEBT D-82 (owner ruling R12: *"price, then wire the Posts-dataset call so posts_count
1440
  # fills with the true value"*). The Profiles dataset's `posts_count: 0` is discarded as the
@@ -1744,32 +1654,36 @@ def bd_filter_rows(snapshot_id):
1744
  return rows, ""
1745
 
1746
 
1747
- def pull_profile(url, max_posts=None, log=print, tier="anonymous",
1748
- fallback=True,
1749
  post_metrics=False, comment_metrics=False, pending_metrics=None,
1750
  pending_profile=None):
1751
- """Everything readable about one public profile, cheapest-honest rung last.
1752
 
1753
  Returns `{state, profile, posts, via, note}` with state ∈ ok | partial | blocked | error.
1754
- `partial` means the identity was read but the media was not β€” which is the outcome the
1755
- anonymous surface most often gives, and saying so is the point.
1756
-
1757
- **RUNG 0 IS THE PAID ONE (R1's hybrid).** With `tier="brightdata"` the vendor is tried FIRST
1758
- and, when it answers, this returns exact counts with no `approx` tag. When it does not answer
1759
- the attempt is RECORDED and the anonymous ladder runs underneath it β€” that is what
1760
- "fail-closed on the key, and the anonymous rung still works" means in C4.
1761
-
1762
- `fallback=False` turns that safety net off on purpose: it means *"exact numbers or nothing"*.
1763
- An influencer table where some rows are exact and some are `og:description` roundings, with
1764
- no way to tell them apart at a glance, is a table you cannot do arithmetic on β€” so a user who
1765
- needs the guarantee can have it, and gets an honest `blocked` instead of a quiet downgrade.
1766
- (The `source`/`approx` columns tell them apart anyway; this makes it impossible to forget.)
 
 
 
 
 
1767
 
1768
  ⚠ `max_posts=None` means the default β€” see `pull_profile_bd` for why the constant cannot be
1769
  named in the signature any more.
1770
  """
1771
  from automation_engine import (DEFAULT_POSTS_PER_PULL, PACE_SECONDS, # lazy β€” module header
1772
- _s, clean_tier)
1773
  max_posts = DEFAULT_POSTS_PER_PULL if max_posts is None else max_posts
1774
  handle = ig_handle(url)
1775
  if not handle:
@@ -1781,173 +1695,55 @@ def pull_profile(url, max_posts=None, log=print, tier="anonymous",
1781
  # inferred from a mere failure, only from a vendor stating it.
1782
  gone = ""
1783
 
1784
- if clean_tier(tier) == "brightdata":
1785
- paid = pull_profile_bd(url, max_posts=max_posts, post_metrics=post_metrics,
1786
- comment_metrics=comment_metrics, log=log,
1787
- pending_metrics=pending_metrics,
1788
- pending_profile=pending_profile)
1789
- if paid["state"] in ("ok", "partial"):
1790
- return paid
1791
- # ⚠ 200, NOT 90. This string is the ONLY account of what the vendor said, and at 90 the
1792
- # measured sentence truncated to *"…was not ready within"* β€” mid-clause, with the budget
1793
- # and the snapshot id cut off. A reason nobody can read is the reason being discarded
1794
- # with extra steps (D-103).
1795
- attempts.append(f"brightdata:{_s(paid.get('note'), 200)}")
1796
- # ⭐ THE APIFY RUNG (owner report 2026-08-09: "both Bright Data and Apify working in
1797
- # tandem"). `providers.py` has declared `ig_profile -> ("brightdata", "apify")` since
1798
- # 2026-08-08 and nothing ever walked it: Bright Data failing went straight to the
1799
- # anonymous rungs, past a configured provider declared capable of exactly this.
1800
- #
1801
- # ⚠ IT RUNS ONLY AFTER BRIGHT DATA HAS ACTUALLY FAILED, never as a top-up. Bright Data
1802
- # answers profile+likes+comments correctly and far more cheaply; re-buying a profile from
1803
- # a second vendor on every read is the expensive mistake the capability split exists to
1804
- # prevent. This is a fallback, and the `attempts` trail records that it was needed.
1805
- #
1806
- # ⚠ AND IT IS GATED ON `configured()`, not merely on being in the chain β€” an unconfigured
1807
- # provider must read as a rung that was never tried, not as one that refused.
1808
- if providers.PROVIDERS["apify"].can("ig_profile"):
1809
- a_prof, a_note = apify_profile(handle)
1810
- if a_prof and a_prof.get("followers") is not None:
1811
- log(f" apify: profile ok ({a_prof.get('followers')} followers)")
1812
- return {"state": "ok", "profile": a_prof, "posts": [], "comments": [],
1813
- "via": "apify",
1814
- "note": ("Bright Data could not read this profile "
1815
- f"({_s(paid.get('note'), 90)}), so it came from Apify")}
1816
- # ⭐ A VENDOR SAYING *not found* IS THE ANSWER, NOT A FAILED ATTEMPT. Recorded here
1817
- # and carried to whichever return this walk reaches, so the run can say the one thing
1818
- # that ends the loop instead of the fourth variation on "we could not read it".
1819
- if a_note == ACCOUNT_GONE_NOTE:
1820
- gone = a_note
1821
- attempts.append(f"apify:{_s(a_note or 'answered without follower counts', 200)}")
1822
- time.sleep(PACE_SECONDS)
1823
- if not fallback:
1824
- return {"state": "blocked", "profile": {}, "posts": [], "comments": [],
1825
- "via": "brightdata", "gone": bool(gone),
1826
- "note": (gone + " (the anonymous fallback is turned off for this automation)"
1827
- if gone else
1828
- "the paid rungs did not answer and the anonymous fallback is turned "
1829
- f"off for this automation ({'; '.join(attempts) or 'blocked'})")}
1830
  time.sleep(PACE_SECONDS)
1831
 
1832
- # RUNG 1 β€” the web client's own profile endpoint. Public, no cookie; the one that used to
1833
- # carry both the profile AND the first page of media in a single answer.
1834
- api = f"https://www.instagram.com/api/v1/users/web_profile_info/?username={handle}"
1835
- status, body, note = _ig_get(api, headers={
1836
- "X-IG-App-ID": IG_APP_ID, "Accept": "*/*",
1837
- "Referer": f"https://www.instagram.com/{handle}/"})
1838
- attempts.append(f"web_profile_info:{status or note}")
1839
- if 200 <= status < 300 and body:
1840
- try:
1841
- node = ((json.loads(body.decode("utf-8", "replace")) or {})
1842
- .get("data") or {}).get("user") or {}
1843
- except Exception:
1844
- node = {}
1845
- if node:
1846
- profile = _profile_from_web_api(node, handle)
1847
- media = node.get("edge_owner_to_timeline_media") or {}
1848
- posts = _posts_from_edges(media.get("edges"))[:max_posts]
1849
- page = media.get("page_info") or {}
1850
- if page.get("has_next_page") and page.get("end_cursor") and len(posts) < max_posts:
1851
- more, pnote = _paginate(node.get("id"), page.get("end_cursor"),
1852
- max_posts - len(posts), log)
1853
- posts.extend(more)
1854
- if pnote:
1855
- attempts.append(pnote)
1856
- state = "ok" if posts else "partial"
1857
- return {"state": state, "profile": profile, "posts": posts,
1858
- "via": "web_profile_info",
1859
- "note": "" if posts else "profile read; media edges empty ("
1860
- + ", ".join(attempts) + ")"}
1861
-
1862
- time.sleep(PACE_SECONDS)
1863
-
1864
- # RUNG 2 β€” the profile HTML. Its og:description carries the three counts even when every
1865
- # JSON surface refuses, so it is the honest floor: identity without media.
1866
- status, body, note = _ig_get(f"https://www.instagram.com/{handle}/")
1867
- attempts.append(f"html:{status or note}")
1868
- if 200 <= status < 300 and body:
1869
- text = body.decode("utf-8", "replace")
1870
- if "/accounts/login" in text[:4000] and "og:description" not in text:
1871
- return {"state": "blocked", "profile": {}, "posts": [], "via": "html",
1872
- "note": "Instagram served the login wall (" + ", ".join(attempts) + ")"}
1873
- desc = ""
1874
- m = re.search(r'<meta[^>]+property="og:description"[^>]+content="([^"]*)"', text)
1875
- if m:
1876
- desc = m.group(1)
1877
- counts = _OG_COUNTS.search(desc or "")
1878
- title = ""
1879
- mt = re.search(r'<meta[^>]+property="og:title"[^>]+content="([^"]*)"', text)
1880
- if mt:
1881
- title = mt.group(1)
1882
- if counts:
1883
- profile = {
1884
- "username": handle, "full_name": title.split("(")[0].strip(),
1885
- # ⚠ NO BIO. `og:description` on a profile page is
1886
- # "204K Followers, 5,245 Following, 750 Posts - See Instagram photos and videos
1887
- # from Inayma (@inayma)" β€” the tail is Instagram's own boilerplate, not the
1888
- # person's biography. Storing it would fill a Bio column with the same sentence
1889
- # for every influencer, which reads as data and is not. Measured 2026-08-04.
1890
- "bio": "",
1891
- "followers": _loose_count(counts.group(1)),
1892
- "following": _loose_count(counts.group(2)),
1893
- "posts_count": _loose_count(counts.group(3)),
1894
- "verified": "",
1895
- # The page abbreviates ("204K"), so the number is the page's rounding, not a
1896
- # count we read. Recorded on the row rather than left for someone to discover
1897
- # when 204,000 fails to reconcile with anything.
1898
- "approx": "1",
1899
- }
1900
- return {"state": "partial", "profile": profile, "posts": [], "via": "og:description",
1901
- "note": "counts are the page's own abbreviations; media is not anonymously "
1902
- "readable (" + ", ".join(attempts) + ")"}
1903
- # ⭐⭐ THE DEFINITIVE ANSWER WINS OVER THE LAST RUNG'S SHRUG. If a vendor said the account does
1904
- # not exist and the free rungs then also found nothing, the honest report is *"this account
1905
- # does not exist"* β€” not *"nothing anonymously readable"*, which is a statement about US.
1906
- # ⚠ It is checked HERE, after the free rungs have run, and not as an early return: an
1907
- # anonymous read that DOES succeed above proves the vendor wrong and returns `ok`/`partial`
1908
- # on its own. A vendor's not-found is strong evidence, never a reason to stop looking.
1909
- if gone:
1910
- return {"state": "blocked", "profile": {}, "posts": [], "via": "", "gone": True,
1911
- "note": f"{gone}. Nothing else could read it either ({', '.join(attempts)})"}
1912
- if status in (401, 403, 429) or status == 0:
1913
- return {"state": "blocked", "profile": {}, "posts": [], "via": "",
1914
- "note": f"Instagram refused the anonymous read ({', '.join(attempts)})"}
1915
- return {"state": "error", "profile": {}, "posts": [], "via": "",
1916
- "note": f"nothing anonymously readable ({', '.join(attempts)})"}
1917
-
1918
-
1919
- #: The historical anonymous media query. Kept because trying it and RECORDING the refusal is
1920
- #: what makes "as deep as anonymously accessible" (R7) a measurement instead of an assumption.
1921
- IG_MEDIA_QUERY_HASH = "e769aa130647d2354c40ea6a439bfc08"
1922
-
1923
-
1924
- def _paginate(user_id, cursor, want, log=print):
1925
- """Walk further back through the media edges while the anonymous surface allows it."""
1926
- from automation_engine import PACE_SECONDS # lazy β€” see the module header
1927
- out, note = [], ""
1928
- while user_id and cursor and len(out) < want:
1929
- time.sleep(PACE_SECONDS)
1930
- variables = json.dumps({"id": str(user_id), "first": min(50, want - len(out)),
1931
- "after": cursor}, separators=(",", ":"))
1932
- url = (f"https://www.instagram.com/graphql/query/?query_hash={IG_MEDIA_QUERY_HASH}"
1933
- f"&variables={requests.utils.quote(variables)}")
1934
- status, body, err = _ig_get(url, headers={"X-IG-App-ID": IG_APP_ID})
1935
- if not (200 <= status < 300) or not body:
1936
- note = f"graphql:{status or err}"
1937
- break
1938
- try:
1939
- media = (((json.loads(body.decode("utf-8", "replace")) or {}).get("data") or {})
1940
- .get("user") or {}).get("edge_owner_to_timeline_media") or {}
1941
- except Exception:
1942
- note = "graphql:unparseable"
1943
- break
1944
- edges = media.get("edges") or []
1945
- if not edges:
1946
- note = "graphql:empty"
1947
- break
1948
- out.extend(_posts_from_edges(edges))
1949
- page = media.get("page_info") or {}
1950
- if not page.get("has_next_page"):
1951
- break
1952
- cursor = page.get("end_cursor")
1953
- return out[:want], note
 
34
  probe doing `import connectors_ig` would have got a half-built engine and an AttributeError from
35
  inside a vendor call. **An intermittent-by-import-order failure is the worst shape available**, so
36
  every reach-back is a `from automation_engine import …` INSIDE the function that needs it, where
37
+ the module is always fully built. There are exactly seven β€” `_bd_why`, `bd_call`,
38
+ `apify_posts`, `apify_profile`, `_tag_metric_deferrals`, `pull_profile_bd`, `pull_profile` β€” each
39
  marked `# lazy β€” see the module header`, and they reach for four kinds of thing:
40
 
41
  fetch / fetch_json / Refused the SSRF-guarded HTTP rail (the scrape runner shares it)
 
54
  """
55
  from __future__ import annotations
56
 
 
57
  import hashlib
58
  import json
59
  import os
 
67
 
68
 
69
  # ---------------------------------------------------------------------------------------------
70
+ # HANDLE + COUNT READERS (no vendor, no key β€” pure parsing)
71
  # ---------------------------------------------------------------------------------------------
72
+ # β›” THE ANONYMOUS LADDER THAT USED TO LIVE HERE IS DELETED (wave 28, owner ruling R5), and this
73
+ # note is the reason it must not come back on a "we could read this for free" impulse. Its two
74
+ # rungs β€” Instagram's `web_profile_info` endpoint and the profile HTML's `og:description` β€”
75
+ # returned counts the PAGE had already rounded ("204K"), which landed in the same `followers`
76
+ # column as a measured 204,318 under an `approx: "1"` flag nobody sees until after they have
77
+ # averaged it. Free was never the problem; a column you cannot do arithmetic on was.
78
+ # Gone with it: `_ig_get`, `IG_APP_ID`, `IG_MEDIA_QUERY_HASH`, `_paginate`, `_posts_from_edges`,
79
+ # `_profile_from_web_api`, `_OG_COUNTS`, `_loose_count`.
80
 
81
  def ig_handle(url):
82
  """The handle out of a profile URL (or a bare handle). '' when it is not one."""
 
94
  return re.sub(r"[^A-Za-z0-9._]", "", first)[:40]
95
 
96
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
97
  def _ig_int(v):
98
  try:
99
  return int(v)
 
116
  return None if n == 0 else n
117
 
118
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
119
  BD_BASE_DEFAULT = "https://api.brightdata.com"
120
  BD_DS_PROFILES = "gd_l1vikfch901nx3by4" # Instagram – Profiles. 36 fields, 620M records
121
  BD_DS_POSTS = "gd_lk5ns7kz21pck8jpis" # Instagram – Posts. 43 fields
 
1100
  item["requestedAt"] = _iso()
1101
 
1102
 
1103
+ def top_up_views(posts, log=print):
1104
+ """Fill `views` on video posts from the `ig_post_views` capability. MUTATES `posts`; returns
1105
+ a note for the run (`""` when there was nothing to do).
1106
+
1107
+ ⭐⭐ CAPABILITY ROUTING, NOT PROVIDER FAILOVER (owner ruling 2026-08-08, `providers.py`).
1108
+ Bright Data answers profile + likes + comments and is declared INCAPABLE of `ig_post_views`,
1109
+ so the chain for that ONE capability resolves to Apify and only the video permalinks are
1110
+ re-bought. Re-running the whole profile on the second vendor would pay twice for the 90% that
1111
+ already worked, which at thousands of tenants x twelve posts is the entire bill.
1112
+
1113
+ β›”β›” WHY THIS IS A FUNCTION AND NOT INLINE, which is the whole point of the 2026-08-09 fix.
1114
+ It used to live inside `pull_profile_bd`'s post-metrics block, so it ran ONLY when the Posts
1115
+ scrape answered within the wait budget. When that batch DEFERRED β€” routine, and the normal
1116
+ outcome on a busy account β€” the posts came back through
1117
+ `automation_engine.collect_pending_metric_snapshots` instead, which never called it. MEASURED
1118
+ on nurilab's `theresalearns`: profile, 12 posts and 154 comments all landed, and **every
1119
+ Views cell was blank**, because the deferred path never asked Apify at all. One capability,
1120
+ two collection paths, one of them wired: the same shape as the `ig_profile` rung that was
1121
+ declared and never called ([[flag-shipped-without-its-writer]]).
1122
+ β‡’ ONE implementation, TWO callers. A copy in the collector would have been a second thing to
1123
+ keep in step, and the two would drift on the next ruling.
1124
+
1125
+ ⚠ VIDEO ONLY. A carousel or image has no view count, so sending it would buy a record that
1126
+ can only come back blank.
1127
+ """
1128
+ video_urls = [p["url"] for p in (posts or [])
1129
+ if p.get("type") == "video" and p.get("url")]
1130
+ if not video_urls:
1131
+ return ""
1132
+
1133
+ def _views_work(prov, _urls=tuple(video_urls)):
1134
+ if prov.key == "apify":
1135
+ return apify_posts(list(_urls))
1136
+ # A provider in the chain with no runner here is a configuration error, not a vendor
1137
+ # outage β€” say so rather than returning an empty list that reads as "the vendor had
1138
+ # nothing".
1139
+ return None, f"no {prov.key} runner is wired for ig_post_views"
1140
+
1141
+ got_views, attempts = providers.run(
1142
+ "ig_post_views", _views_work,
1143
+ # β›” THE SATISFIED PREDICATE IS THE FALLBACK TRIGGER. Rows that come back with every
1144
+ # `views` blank are a FAILURE for this capability even at HTTP 200 β€” which is exactly how
1145
+ # Bright Data behaves, and why a chain that only caught exceptions would never have
1146
+ # reached a second provider.
1147
+ satisfied=lambda rows: bool(rows) and any(r.get("views") for r in rows),
1148
+ log=log)
1149
+ for row in (got_views or []):
1150
+ target = next((p for p in posts
1151
+ if p.get("shortcode") == row.get("shortcode")), None)
1152
+ # β›” TAKE ONLY THE CAPABILITY THAT WAS ASKED FOR. This provider also returns
1153
+ # likes/comments/caption, and letting them land would silently switch the source of
1154
+ # columns Bright Data already answered β€” the schema would be stable but the PROVENANCE
1155
+ # would flip halfway through a row.
1156
+ if target and row.get("views") not in (None, ""):
1157
+ target["views"] = row["views"]
1158
+ # ⭐ LOCATION RIDES ALONG FOR FREE, AND ONLY INTO A GAP (2026-08-08).
1159
+ # MEASURED: Bright Data's `location_details` is RICHER when present β€” it carries real
1160
+ # coordinates (`lat -6.9246, lng 106.9292, name "Sukabumi"`) which Apify does not return
1161
+ # at all β€” but it is populated on only 44 of 227 posts, and on `DblAkEbv0ry` it returned
1162
+ # nothing while Apify returned "Jakarta, Indonesia". So Bright Data stays the source and
1163
+ # this fills only what it left BLANK.
1164
+ # β›” THE COST ARGUMENT IS WHY IT IS HERE AND NOT ITS OWN CAPABILITY: this Apify record has
1165
+ # ALREADY been bought for the view count and carries `locationName` in the same payload.
1166
+ # Routing location as a separate capability would buy a second record for a field that is
1167
+ # already sitting in this response.
1168
+ if target and not str(target.get("tagged_location") or "").strip() \
1169
+ and str(row.get("tagged_location") or "").strip():
1170
+ target["tagged_location"] = row["tagged_location"]
1171
+ used = next((a.provider for a in attempts if a.ok), "")
1172
+ if used:
1173
+ return f"view counts via {used}"
1174
+ if attempts:
1175
+ return ("no provider could supply view counts: "
1176
+ + "; ".join(f"{a.provider} {a.note}" for a in attempts))
1177
+ return ""
1178
+
1179
+
1180
  def pull_profile_bd(url, max_posts=None, post_metrics=False,
1181
  comment_metrics=False, log=print, pending_metrics=None,
1182
  pending_profile=None):
 
1344
  elif metric_notes:
1345
  note = f"some post metrics were unavailable: {'; '.join(metric_notes)}"
1346
 
1347
+ note = "; ".join(x for x in (note, top_up_views(posts, log=log)) if x)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1348
 
1349
  # ⭐⭐ DEBT D-82 (owner ruling R12: *"price, then wire the Posts-dataset call so posts_count
1350
  # fills with the true value"*). The Profiles dataset's `posts_count: 0` is discarded as the
 
1654
  return rows, ""
1655
 
1656
 
1657
+ def pull_profile(url, max_posts=None, log=print,
 
1658
  post_metrics=False, comment_metrics=False, pending_metrics=None,
1659
  pending_profile=None):
1660
+ """Everything readable about one public profile β€” EXACT NUMBERS OR NOTHING.
1661
 
1662
  Returns `{state, profile, posts, via, note}` with state ∈ ok | partial | blocked | error.
1663
+ `partial` means the identity was read but the media was not.
1664
+
1665
+ ⭐⭐ WAVE 28 / OWNER RULING R5 β€” THE FREE ANONYMOUS LADDER IS RETIRED, AND THE REASON IS WHAT
1666
+ IT PUT IN THE COLUMN, not what it cost. Two rungs used to sit under the paid one: Instagram's
1667
+ own `web_profile_info` endpoint, and the profile HTML's `og:description`, which carries
1668
+ "204K Followers, 5,245 Following, 750 Posts". That last one is the page's OWN ROUNDING, and it
1669
+ landed in the same `followers` column as a measured 204,318, stamped `approx: "1"` β€” a flag
1670
+ that tells you the number is an abbreviation only AFTER you have already averaged it, sorted
1671
+ on it, or filtered a shortlist by it. A table where some rows are measurements and some are
1672
+ the page's shorthand is a table you cannot do arithmetic on.
1673
+ β‡’ A vendor refusal now returns `blocked` and the row KEEPS WHAT IT LAST KNEW, instead of being
1674
+ overwritten with a rounder number. The user-facing tier/fallback choice is gone with the
1675
+ rungs; there is one behaviour and it is this one.
1676
+
1677
+ β›” THIS IS STILL CAPABILITY ROUTING, NOT A SINGLE VENDOR. `ig_profile` is declared on Bright
1678
+ Data AND Apify (`providers.py`), so a profile Bright Data cannot read is asked of Apify before
1679
+ anything reports blocked. What was retired is the FREE approximate rung, not the second
1680
+ opinion β€” the fallback that costs money and answers exactly is the one worth having.
1681
 
1682
  ⚠ `max_posts=None` means the default β€” see `pull_profile_bd` for why the constant cannot be
1683
  named in the signature any more.
1684
  """
1685
  from automation_engine import (DEFAULT_POSTS_PER_PULL, PACE_SECONDS, # lazy β€” module header
1686
+ _s)
1687
  max_posts = DEFAULT_POSTS_PER_PULL if max_posts is None else max_posts
1688
  handle = ig_handle(url)
1689
  if not handle:
 
1695
  # inferred from a mere failure, only from a vendor stating it.
1696
  gone = ""
1697
 
1698
+ paid = pull_profile_bd(url, max_posts=max_posts, post_metrics=post_metrics,
1699
+ comment_metrics=comment_metrics, log=log,
1700
+ pending_metrics=pending_metrics,
1701
+ pending_profile=pending_profile)
1702
+ if paid["state"] in ("ok", "partial"):
1703
+ return paid
1704
+ # ⚠ 200, NOT 90. This string is the ONLY account of what the vendor said, and at 90 the
1705
+ # measured sentence truncated to *"…was not ready within"* β€” mid-clause, with the budget and
1706
+ # the snapshot id cut off. A reason nobody can read is the reason being discarded with extra
1707
+ # steps (D-103).
1708
+ attempts.append(f"brightdata:{_s(paid.get('note'), 200)}")
1709
+ # ⭐ THE APIFY RUNG (owner report 2026-08-09: "both Bright Data and Apify working in tandem").
1710
+ # `providers.py` has declared `ig_profile -> ("brightdata", "apify")` since 2026-08-08 and
1711
+ # nothing ever walked it: Bright Data failing went straight to the anonymous rungs, past a
1712
+ # configured provider declared capable of exactly this.
1713
+ #
1714
+ # ⚠ IT RUNS ONLY AFTER BRIGHT DATA HAS ACTUALLY FAILED, never as a top-up. Bright Data answers
1715
+ # profile+likes+comments correctly and far more cheaply; re-buying a profile from a second
1716
+ # vendor on every read is the expensive mistake the capability split exists to prevent. This
1717
+ # is a fallback, and the `attempts` trail records that it was needed.
1718
+ #
1719
+ # ⚠ AND IT IS GATED ON `configured()`, not merely on being in the chain β€” an unconfigured
1720
+ # provider must read as a rung that was never tried, not as one that refused.
1721
+ if providers.PROVIDERS["apify"].can("ig_profile"):
1722
+ a_prof, a_note = apify_profile(handle)
1723
+ if a_prof and a_prof.get("followers") is not None:
1724
+ log(f" apify: profile ok ({a_prof.get('followers')} followers)")
1725
+ return {"state": "ok", "profile": a_prof, "posts": [], "comments": [],
1726
+ "via": "apify",
1727
+ "note": ("Bright Data could not read this profile "
1728
+ f"({_s(paid.get('note'), 90)}), so it came from Apify")}
1729
+ # ⭐ A VENDOR SAYING *not found* IS THE ANSWER, NOT A FAILED ATTEMPT. Recorded here and
1730
+ # carried to the return below, so the run can say the one thing that ends the loop
1731
+ # instead of the fourth variation on "we could not read it".
1732
+ if a_note == ACCOUNT_GONE_NOTE:
1733
+ gone = a_note
1734
+ attempts.append(f"apify:{_s(a_note or 'answered without follower counts', 200)}")
 
 
 
 
 
 
 
 
 
1735
  time.sleep(PACE_SECONDS)
1736
 
1737
+ # β›”β›” THE END OF THE WALK (R5). There were two more rungs here β€” `web_profile_info` and the
1738
+ # profile HTML's `og:description` β€” and they are DELETED, not disabled. What they returned was
1739
+ # a `partial` carrying `approx: "1"` and counts rounded by Instagram's own page furniture; the
1740
+ # docstring says why that is worse than nothing. `blocked` keeps the row's last real values.
1741
+ # ⚠ `via` STAYS "brightdata" on this path rather than "" β€” it names the chain that was walked,
1742
+ # which is what a person reading a blocked row needs in order to know who to ask.
1743
+ return {"state": "blocked", "profile": {}, "posts": [], "comments": [],
1744
+ "via": "brightdata", "gone": bool(gone),
1745
+ "note": (gone if gone else
1746
+ "no provider could read this profile "
1747
+ f"({'; '.join(attempts) or 'blocked'})")}
1748
+
1749
+
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
api/odoo_relational.py CHANGED
@@ -54,11 +54,29 @@ INVOICES_KEY = "ut_odoo_invoices"
54
  CUSTOMERS_KEY = "ut_odoo_customers"
55
  ORDERS_KEY = "ut_odoo_orders"
56
  PRODUCTS_KEY = "ut_odoo_products"
 
 
 
 
 
 
 
 
 
 
 
57
 
58
  #: The join column the partner-grain tables carry. Derived links resolve through it (`on`/`from`).
59
  JOIN_KEY = "partner_id"
60
  #: The product-grain equivalent.
61
  PRODUCT_JOIN_KEY = "product_id"
 
 
 
 
 
 
 
62
 
63
  #: The oracle's own predicate β€” `modules/ar._open_docs`, copied rather than re-derived so the two
64
  #: cannot drift. It now selects a SUBSET of the invoices table rather than defining it.
@@ -119,6 +137,134 @@ def _refreshed_field():
119
  "default": False, "description": "When this row was last reconciled against Odoo."}
120
 
121
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
122
  def invoice_fields():
123
  """One row per POSTED customer invoice or refund β€” the full history, not just what is open."""
124
  return [_preset(f) for f in (
@@ -155,6 +301,19 @@ def invoice_fields():
155
  {"key": "customer_link", "label": "Customer record", "type": "link", "source": "overlay",
156
  "default": False,
157
  "link": {"table": CUSTOMERS_KEY, "on": JOIN_KEY, "from": JOIN_KEY}},
 
 
 
 
 
 
 
 
 
 
 
 
 
158
  _refreshed_field(),
159
  )]
160
 
@@ -184,6 +343,11 @@ def order_fields():
184
  {"key": "customer_link", "label": "Customer record", "type": "link", "source": "overlay",
185
  "default": False,
186
  "link": {"table": CUSTOMERS_KEY, "on": JOIN_KEY, "from": JOIN_KEY}},
 
 
 
 
 
187
  _refreshed_field(),
188
  )]
189
 
@@ -268,6 +432,12 @@ def customer_fields():
268
  "default": True,
269
  "description": "The customer's assigned agent (res.partner.agent_ids[0] β€” the "
270
  "Customers-module convention)."},
 
 
 
 
 
 
271
  _scope_field(),
272
 
273
  # --- the relations -------------------------------------------------------------------
@@ -277,6 +447,11 @@ def customer_fields():
277
  {"key": "orders", "label": "Orders", "type": "link", "source": "overlay",
278
  "default": True,
279
  "link": {"table": ORDERS_KEY, "on": JOIN_KEY, "from": JOIN_KEY}},
 
 
 
 
 
280
 
281
  # --- link rollups over the invoice history --------------------------------------------
282
  # ⭐ NO CONDITION, and that is measured rather than assumed: a settled document's residual
@@ -437,12 +612,19 @@ def read_invoices(cur, excluded=None, open_only=False):
437
  """
438
  excluded = excluded if excluded is not None else excluded_ids(cur)
439
  where = _AR_WHERE if open_only else _POSTED_DOCS
 
 
 
 
 
 
440
  sql = ("SELECT id, name, partner_id, partner_name, invoice_date, invoice_date_due, "
441
- " amount_untaxed_signed, amount_residual_signed, payment_state, move_type "
 
442
  f"FROM account_move WHERE {where} AND partner_id IS NOT NULL")
443
  out = []
444
  for r in cur.execute(sql).fetchall():
445
- (mid, name, pid, pname, inv_date, due, untaxed, residual, pay_state, mtype) = r
446
  scope = _in_scope(pid, excluded)
447
  if open_only and not scope:
448
  continue
@@ -458,6 +640,7 @@ def read_invoices(cur, excluded=None, open_only=False):
458
  "amount_untaxed": float(untaxed or 0.0),
459
  "payment_state": str(pay_state or ""),
460
  "move_type": str(mtype or ""),
 
461
  "wholesale_scope": scope,
462
  })
463
  return out
@@ -547,9 +730,10 @@ def read_customers(cur, excluded=None):
547
  # ⚠ THE AGENT JOIN IS DROPPED WHOLE when `agent_id` is absent, not merely NULL-ed: the join
548
  # itself names the column, so `_col` on the SELECT list alone would still fail to bind.
549
  agent = ("ag.name" if "agent_id" in have else "NULL")
 
550
  join = ("LEFT JOIN res_partner ag ON ag.id = p.agent_id " if "agent_id" in have else "")
551
  sql = (f"SELECT p.id, p.name, {_col(have, 'p.city')}, {_col(have, 'p.state_name')}, "
552
- f" {_col(have, 'p.country_name')}, {agent} "
553
  "FROM res_partner p "
554
  f"{join}"
555
  "WHERE p.id IN ("
@@ -559,7 +743,7 @@ def read_customers(cur, excluded=None):
559
  f" WHERE {_POSTED_DOCS} AND partner_id IS NOT NULL)")
560
  out = []
561
  for r in cur.execute(sql).fetchall():
562
- (pid, name, city, state, country, agent) = r
563
  out.append({
564
  "_id": str(pid),
565
  "customer": str(name or ""),
@@ -568,11 +752,146 @@ def read_customers(cur, excluded=None):
568
  "state": str(state or ""),
569
  "country": str(country or ""),
570
  "agent": str(agent or ""),
 
571
  "wholesale_scope": _in_scope(pid, excluded),
572
  })
573
  return out
574
 
575
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
576
  def customers_from(invoice_rows):
577
  """The partners carrying the given invoice rows β€” the pre-2026-08-09 population builder.
578
 
@@ -608,6 +927,13 @@ TABLES = (
608
  ("products", PRODUCTS_KEY, "Odoo products", product_fields),
609
  ("invoices", INVOICES_KEY, "Odoo invoices", invoice_fields),
610
  ("orders", ORDERS_KEY, "Odoo orders", order_fields),
 
 
 
 
 
 
 
611
  )
612
 
613
 
@@ -631,6 +957,10 @@ def plan(cur, rt=None):
631
  "products": read_products(cur),
632
  "invoices": read_invoices(cur, excluded=excluded),
633
  "orders": read_orders(cur, excluded=excluded),
 
 
 
 
634
  }
635
  problems = []
636
 
 
54
  CUSTOMERS_KEY = "ut_odoo_customers"
55
  ORDERS_KEY = "ut_odoo_orders"
56
  PRODUCTS_KEY = "ut_odoo_products"
57
+ #: ⭐ WAVE 28 (owner R1): *"ALL of Unique ID in Odoo is a database e.g. Customers/Products/Agents,
58
+ #: etc. Including expenses and GL codes."* Four more DOCUMENT/REGISTRY grains, each measured to
59
+ #: fit far inside `MAX_ROWS` (19 / 192 / 6,538 / 393 against 60,000).
60
+ #: β›” THE LINE GRAINS ARE DELIBERATELY NOT HERE and this is R2, not an omission: 256,810 order
61
+ #: lines, 154,917 expense GL lines and 311,140 commission lines cannot live in a `ut_*` document
62
+ #: at any cap (63.9 MB / 2.57 s per copy for the order lines alone). "Expenses" as a BROWSABLE
63
+ #: LEDGER is a read-through mirror grid; "expenses" as a NUMBER is a rollup on the GL account row.
64
+ AGENTS_KEY = "ut_odoo_agents"
65
+ ACCOUNTS_KEY = "ut_odoo_accounts"
66
+ BILLS_KEY = "ut_odoo_bills"
67
+ VENDORS_KEY = "ut_odoo_vendors"
68
 
69
  #: The join column the partner-grain tables carry. Derived links resolve through it (`on`/`from`).
70
  JOIN_KEY = "partner_id"
71
  #: The product-grain equivalent.
72
  PRODUCT_JOIN_KEY = "product_id"
73
+ #: ⚠ AN AGENT IS A `res.partner`, so its id shares the partner namespace with a customer's β€” but
74
+ #: it is a DIFFERENT COLUMN on the customer row (`agent_id`, the customer's assigned agent) and the
75
+ #: two must never be joined through `JOIN_KEY`, which would link every customer to itself.
76
+ AGENT_JOIN_KEY = "agent_id"
77
+ #: A vendor is also a `res.partner`; same reasoning, its own column.
78
+ VENDOR_JOIN_KEY = "vendor_id"
79
+ ACCOUNT_JOIN_KEY = "account_code"
80
 
81
  #: The oracle's own predicate β€” `modules/ar._open_docs`, copied rather than re-derived so the two
82
  #: cannot drift. It now selects a SUBSET of the invoices table rather than defining it.
 
137
  "default": False, "description": "When this row was last reconciled against Odoo."}
138
 
139
 
140
+ def agent_fields():
141
+ """One row per SALES AGENT, keyed on the `res.partner` id.
142
+
143
+ ⭐ THE POPULATION IS A UNION OF TWO DISAGREEING SOURCES, and the disagreement is the reason it
144
+ is a union rather than a pick. MEASURED 2026-08-09: 16 partners carry commission lines, 17
145
+ carry `res_partner.agent = TRUE`, and the union is 19 β€” so **2 agents earn commission without
146
+ the flag and 3 are flagged with no commission yet**. Either source alone silently drops real
147
+ agents. Same shape as `read_customers`' two document universes, for the same reason.
148
+ """
149
+ return [_preset(f) for f in (
150
+ {"key": "agent", "label": "Agent", "type": "text", "source": "overlay",
151
+ "default": True, "pinned": True},
152
+ {"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay",
153
+ "default": False, "description": "The `res.partner` id. Also this row's id."},
154
+ {"key": AGENT_JOIN_KEY, "label": "Odoo agent id", "type": "int", "source": "overlay",
155
+ "default": False},
156
+ {"key": "flagged", "label": "Flagged in Odoo", "type": "checkbox", "source": "overlay",
157
+ "default": True,
158
+ "description": "Ticked = `res.partner.agent` is set. Unticked agents were found by "
159
+ "their commission lines instead - both are real, which is why this "
160
+ "table is the union of the two."},
161
+ {"key": "commissioned", "label": "Has commission lines", "type": "checkbox",
162
+ "source": "overlay", "default": True},
163
+ # ⭐ THE INVERSE HALF: the customers whose `agent_id` names this agent. MEASURED: 2,093
164
+ # customers carry one and ALL 2,093 resolve to a row in this table (zero dangling).
165
+ {"key": "customers", "label": "Customers", "type": "link", "source": "overlay",
166
+ "default": True,
167
+ "link": {"table": CUSTOMERS_KEY, "on": AGENT_JOIN_KEY, "from": AGENT_JOIN_KEY}},
168
+ _refreshed_field(),
169
+ )]
170
+
171
+
172
+ def account_fields():
173
+ """One row per `account.account` β€” the GL chart, the owner's "GL codes".
174
+
175
+ ⚠ NO LINK COLUMN, and that is a finding rather than an omission. A GL account meets the rest
176
+ of this schema only at LINE grain (963,783 `account_move_line` rows, 154,917 of them on
177
+ expense-type accounts), and a `ut_*` link folds rows that live in the store. The honest
178
+ binding is a read-through rollup naming a governed topic, or the mirror grid (R2) β€” never a
179
+ link into a table that does not exist. Declaring one here would render a permanently blank
180
+ column, which is the exact trap D-87 warns about from the value site.
181
+ """
182
+ return [_preset(f) for f in (
183
+ {"key": "account_code", "label": "Code", "type": "text", "source": "overlay",
184
+ "default": True, "pinned": True},
185
+ {"key": "account_name", "label": "Account", "type": "text", "source": "overlay",
186
+ "default": True},
187
+ {"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay",
188
+ "default": False, "description": "The `account.account` id. Also this row's id."},
189
+ # ⚠ 15 DISTINCT VALUES MEASURED IN THE MIRROR, all declared. A `select` storing a value its
190
+ # options omit is wave-26 item 24: the filter panel answers with a list that cannot match
191
+ # what is stored.
192
+ {"key": "account_type", "label": "Type", "type": "select", "source": "overlay",
193
+ "default": True,
194
+ "options": ["expense", "expense_direct_cost", "expense_depreciation", "income",
195
+ "income_other", "asset_cash", "asset_current", "asset_receivable",
196
+ "asset_fixed", "asset_non_current", "asset_prepayments",
197
+ "liability_current", "liability_payable", "liability_credit_card",
198
+ "liability_non_current", "equity", "equity_unaffected", "off_balance"]},
199
+ {"key": "is_expense", "label": "Expense account", "type": "checkbox", "source": "overlay",
200
+ "default": True,
201
+ "description": "Ticked for the expense family - the same predicate the semantic layer's "
202
+ "gl_lines topic uses, so this column and that topic cannot disagree."},
203
+ _refreshed_field(),
204
+ )]
205
+
206
+
207
+ def vendor_fields():
208
+ """One row per partner we have POSTED a vendor bill to, keyed on the `res.partner` id.
209
+
210
+ ⚠ A VENDOR IS NOT A CUSTOMER TABLE ROW, even though both are `res.partner`. MEASURED: 393
211
+ vendors, of which only 9 also appear in the customer population. Pointing bills at
212
+ `ut_odoo_customers` would have dangled 384 of 393 links β€” the failure would have been a mostly
213
+ empty column, not an error.
214
+ """
215
+ return [_preset(f) for f in (
216
+ {"key": "vendor", "label": "Vendor", "type": "text", "source": "overlay",
217
+ "default": True, "pinned": True},
218
+ {"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay",
219
+ "default": False, "description": "The `res.partner` id. Also this row's id."},
220
+ {"key": VENDOR_JOIN_KEY, "label": "Odoo vendor id", "type": "int", "source": "overlay",
221
+ "default": False},
222
+ {"key": "country", "label": "Country", "type": "text", "source": "overlay",
223
+ "default": True},
224
+ {"key": "bills", "label": "Bills", "type": "link", "source": "overlay", "default": True,
225
+ "link": {"table": BILLS_KEY, "on": VENDOR_JOIN_KEY, "from": VENDOR_JOIN_KEY}},
226
+ _refreshed_field(),
227
+ )]
228
+
229
+
230
+ def bill_fields():
231
+ """One row per POSTED vendor bill or refund β€” the owner's "expenses", at DOCUMENT grain.
232
+
233
+ ⚠ DOCUMENT GRAIN IS A CHOICE AND IT IS THE ONLY ONE THAT FITS: 6,538 bills against 154,917
234
+ expense GL lines. What a person calls "expenses" is both, and they are different tables - the
235
+ bill is what you pay, the line is what it was coded to. This is the payable; the line ledger
236
+ is the read-through mirror grid (R2).
237
+ """
238
+ return [_preset(f) for f in (
239
+ {"key": "bill_no", "label": "Bill", "type": "text", "source": "overlay",
240
+ "default": True, "pinned": True},
241
+ {"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay",
242
+ "default": False, "description": "The `account.move` id. Also this row's id."},
243
+ {"key": "vendor", "label": "Vendor", "type": "text", "source": "overlay", "default": True},
244
+ {"key": VENDOR_JOIN_KEY, "label": "Odoo vendor id", "type": "int", "source": "overlay",
245
+ "default": False},
246
+ {"key": "invoice_date", "label": "Bill date", "type": "date", "source": "overlay",
247
+ "default": True},
248
+ {"key": "due_date", "label": "Due date", "type": "date", "source": "overlay",
249
+ "default": True},
250
+ # ⚠ SIGNED, like the customer side: Odoo's `_signed` fields already carry the refund's
251
+ # direction, so a refund reduces a total without anybody re-deriving a sign here.
252
+ {"key": "amount_untaxed", "label": "Billed $", "type": "currency", "source": "overlay",
253
+ "default": True, "agg": "sum"},
254
+ {"key": "residual", "label": "Outstanding $", "type": "currency", "source": "overlay",
255
+ "default": True, "agg": "sum"},
256
+ {"key": "payment_state", "label": "Payment state", "type": "select", "source": "overlay",
257
+ "default": True,
258
+ "options": ["not_paid", "partial", "in_payment", "paid", "reversed"]},
259
+ {"key": "move_type", "label": "Document", "type": "select", "source": "overlay",
260
+ "default": False, "options": ["in_invoice", "in_refund"]},
261
+ {"key": "vendor_link", "label": "Vendor record", "type": "link", "source": "overlay",
262
+ "default": False,
263
+ "link": {"table": VENDORS_KEY, "on": VENDOR_JOIN_KEY, "from": VENDOR_JOIN_KEY}},
264
+ _refreshed_field(),
265
+ )]
266
+
267
+
268
  def invoice_fields():
269
  """One row per POSTED customer invoice or refund β€” the full history, not just what is open."""
270
  return [_preset(f) for f in (
 
301
  {"key": "customer_link", "label": "Customer record", "type": "link", "source": "overlay",
302
  "default": False,
303
  "link": {"table": CUSTOMERS_KEY, "on": JOIN_KEY, "from": JOIN_KEY}},
304
+ # ⭐ DEBT D-88, closed 2026-08-09. `invoice_origin` carries the ORDER NAME an invoice was
305
+ # raised from, and the mirror did not sync it until this wave β€” so order->invoice was a
306
+ # two-hop join through 963,783 `account_move_line` rows and wave 27 shipped no link at all.
307
+ # ⚠ IT IS A NAME, NOT AN ID, and Odoo writes free text there (a manual invoice can hold
308
+ # anything; a merged one can hold several origins space-separated). The link resolves
309
+ # against `order_no` and finds nothing when the text is not an order name β€” the honest
310
+ # outcome, and the reason this is a join HINT rather than a foreign key.
311
+ {"key": "origin_order", "label": "Source order", "type": "text", "source": "overlay",
312
+ "default": False,
313
+ "description": "Odoo's `invoice_origin` - usually the order name, sometimes blank."},
314
+ {"key": "order_link", "label": "Order record", "type": "link", "source": "overlay",
315
+ "default": False,
316
+ "link": {"table": ORDERS_KEY, "on": "order_no", "from": "origin_order"}},
317
  _refreshed_field(),
318
  )]
319
 
 
343
  {"key": "customer_link", "label": "Customer record", "type": "link", "source": "overlay",
344
  "default": False,
345
  "link": {"table": CUSTOMERS_KEY, "on": JOIN_KEY, "from": JOIN_KEY}},
346
+ # The reciprocal of the invoice's `order_link` (D-88): the invoices raised from THIS
347
+ # order, matched on the order's own name.
348
+ {"key": "invoices", "label": "Invoices", "type": "link", "source": "overlay",
349
+ "default": True,
350
+ "link": {"table": INVOICES_KEY, "on": "origin_order", "from": "order_no"}},
351
  _refreshed_field(),
352
  )]
353
 
 
432
  "default": True,
433
  "description": "The customer's assigned agent (res.partner.agent_ids[0] β€” the "
434
  "Customers-module convention)."},
435
+ # ⭐ WAVE 28 β€” the agent's ID beside its NAME, because a link joins on an id and this
436
+ # table carried only the display string. ⚠ It is `agent_id`, NEVER `partner_id`: both are
437
+ # `res.partner` ids, and joining agents through `JOIN_KEY` would link every customer to
438
+ # itself and look plausible doing it.
439
+ {"key": AGENT_JOIN_KEY, "label": "Odoo agent id", "type": "int", "source": "overlay",
440
+ "default": False},
441
  _scope_field(),
442
 
443
  # --- the relations -------------------------------------------------------------------
 
447
  {"key": "orders", "label": "Orders", "type": "link", "source": "overlay",
448
  "default": True,
449
  "link": {"table": ORDERS_KEY, "on": JOIN_KEY, "from": JOIN_KEY}},
450
+ # MEASURED: 2,093 customers carry an `agent_id` and all 2,093 resolve to a row in the
451
+ # agents table β€” zero dangling, which is why this ships as a link rather than a lookup.
452
+ {"key": "agent_link", "label": "Agent record", "type": "link", "source": "overlay",
453
+ "default": False,
454
+ "link": {"table": AGENTS_KEY, "on": AGENT_JOIN_KEY, "from": AGENT_JOIN_KEY}},
455
 
456
  # --- link rollups over the invoice history --------------------------------------------
457
  # ⭐ NO CONDITION, and that is measured rather than assumed: a settled document's residual
 
612
  """
613
  excluded = excluded if excluded is not None else excluded_ids(cur)
614
  where = _AR_WHERE if open_only else _POSTED_DOCS
615
+ # ⚠ `invoice_origin` (D-88) is read through `_col` DELIBERATELY. It was added to `ENTITIES` in
616
+ # this same wave, so a Space whose mirror is still hydrating from a pre-wave seed snapshot does
617
+ # not have the column yet β€” and DuckDB answers a missing identifier with a Binder error that
618
+ # reaches the operator as a bare 500. This is the exact class `columns()` was written for: the
619
+ # link degrades to blank for one sync cycle instead of refusing the whole spawn.
620
+ have = columns(cur, "account_move")
621
  sql = ("SELECT id, name, partner_id, partner_name, invoice_date, invoice_date_due, "
622
+ " amount_untaxed_signed, amount_residual_signed, payment_state, move_type, "
623
+ f" {_col(have, 'invoice_origin', chr(39) + chr(39))} "
624
  f"FROM account_move WHERE {where} AND partner_id IS NOT NULL")
625
  out = []
626
  for r in cur.execute(sql).fetchall():
627
+ (mid, name, pid, pname, inv_date, due, untaxed, residual, pay_state, mtype, origin) = r
628
  scope = _in_scope(pid, excluded)
629
  if open_only and not scope:
630
  continue
 
640
  "amount_untaxed": float(untaxed or 0.0),
641
  "payment_state": str(pay_state or ""),
642
  "move_type": str(mtype or ""),
643
+ "origin_order": str(origin or "").strip(),
644
  "wholesale_scope": scope,
645
  })
646
  return out
 
730
  # ⚠ THE AGENT JOIN IS DROPPED WHOLE when `agent_id` is absent, not merely NULL-ed: the join
731
  # itself names the column, so `_col` on the SELECT list alone would still fail to bind.
732
  agent = ("ag.name" if "agent_id" in have else "NULL")
733
+ agent_id_col = ("p.agent_id" if "agent_id" in have else "NULL")
734
  join = ("LEFT JOIN res_partner ag ON ag.id = p.agent_id " if "agent_id" in have else "")
735
  sql = (f"SELECT p.id, p.name, {_col(have, 'p.city')}, {_col(have, 'p.state_name')}, "
736
+ f" {_col(have, 'p.country_name')}, {agent}, {agent_id_col} "
737
  "FROM res_partner p "
738
  f"{join}"
739
  "WHERE p.id IN ("
 
743
  f" WHERE {_POSTED_DOCS} AND partner_id IS NOT NULL)")
744
  out = []
745
  for r in cur.execute(sql).fetchall():
746
+ (pid, name, city, state, country, agent, agent_id) = r
747
  out.append({
748
  "_id": str(pid),
749
  "customer": str(name or ""),
 
752
  "state": str(state or ""),
753
  "country": str(country or ""),
754
  "agent": str(agent or ""),
755
+ AGENT_JOIN_KEY: int(agent_id) if agent_id else "",
756
  "wholesale_scope": _in_scope(pid, excluded),
757
  })
758
  return out
759
 
760
 
761
+ def read_agents(cur):
762
+ """[(row dict)] β€” the UNION of both agent sources, keyed on the `res.partner` id.
763
+
764
+ β›” `res_partner.agent` is a BOOLEAN and `datastore.BOOL_FIELDS` lists it for a measured reason:
765
+ Odoo returns False both for "empty" and for "boolean false", so a bool missing from that list
766
+ silently becomes NULL and every row would read "not an agent" indistinguishably from
767
+ "unknown". Read it as a truth value, never as a presence test.
768
+ """
769
+ have = columns(cur, "res_partner")
770
+ if "id" not in have:
771
+ return []
772
+ flagged = "p.agent" if "agent" in have else "FALSE"
773
+ # β›”β›” THE COMMISSION TABLE IS GUARDED AS A **TABLE**, not just as a column, and that
774
+ # distinction is the whole point of this block. `columns()` was written for a missing COLUMN
775
+ # (a backfill that has not run yet); `account_invoice_line_agent` is an OCA module entity that
776
+ # a mirror hydrated from an older seed snapshot may not have AT ALL. A SELECT naming an absent
777
+ # table is a DuckDB Binder error, and this reader runs inside `plan()` β€” so one missing table
778
+ # would fail the WHOLE eight-table spawn and reach the operator as a bare 500. That is
779
+ # precisely D-107's shape, and it would have arrived on the first deploy of this feature.
780
+ # ⚠ DEGRADE, NEVER REFUSE, which is the posture `columns()`'s own docstring sets: without the
781
+ # commission table the population falls back to the FLAGGED partners alone and `commissioned`
782
+ # reads blank for every row β€” fewer agents and an honestly empty column, rather than no spawn.
783
+ has_comm = bool(columns(cur, "account_invoice_line_agent"))
784
+ commissioned = ("(p.id IN (SELECT agent_id FROM account_invoice_line_agent "
785
+ " WHERE agent_id IS NOT NULL))" if has_comm else "FALSE")
786
+ union_leg = (" SELECT agent_id FROM account_invoice_line_agent WHERE agent_id IS NOT NULL "
787
+ " UNION " if has_comm else "")
788
+ sql = (f"SELECT p.id, p.name, {flagged}, {commissioned} AS commissioned "
789
+ "FROM res_partner p WHERE p.id IN ("
790
+ f"{union_leg}SELECT id FROM res_partner WHERE {flagged})")
791
+ out = []
792
+ for (aid, name, flag, comm) in cur.execute(sql).fetchall():
793
+ out.append({
794
+ "_id": str(aid),
795
+ "agent": str(name or ""),
796
+ "odoo_id": int(aid),
797
+ AGENT_JOIN_KEY: int(aid),
798
+ "flagged": "1" if flag else "",
799
+ "commissioned": "1" if comm else "",
800
+ })
801
+ return out
802
+
803
+
804
+ def read_accounts(cur):
805
+ """[(row dict)] β€” the whole GL chart, keyed on the `account.account` id.
806
+
807
+ ⚠ THE EXPENSE PREDICATE IS THE SEMANTIC LAYER'S, copied rather than invented:
808
+ `harness/semantic.py`'s `gl_lines` topic scopes expenses as
809
+ `account_type in ('expense','expense_depreciation')`. A second definition here is how a
810
+ column and a topic start disagreeing about the same word.
811
+ β›” `account.account` has NO `active` column in this Odoo version (a domain naming it 500s), so
812
+ there is nothing to filter and every account is a row.
813
+ """
814
+ have = columns(cur, "account_account")
815
+ if not have:
816
+ return []
817
+ sql = (f"SELECT id, {_col(have, 'code', chr(39) + chr(39))}, "
818
+ f" {_col(have, 'name', chr(39) + chr(39))}, "
819
+ f" {_col(have, 'account_type', chr(39) + chr(39))} FROM account_account")
820
+ out = []
821
+ for (aid, code, name, atype) in cur.execute(sql).fetchall():
822
+ t = str(atype or "")
823
+ out.append({
824
+ "_id": str(aid),
825
+ ACCOUNT_JOIN_KEY: str(code or ""),
826
+ "account_name": str(name or ""),
827
+ "odoo_id": int(aid),
828
+ "account_type": t,
829
+ "is_expense": "1" if t in ("expense", "expense_depreciation") else "",
830
+ })
831
+ return out
832
+
833
+
834
+ _VENDOR_DOCS = "state = 'posted' AND move_type IN ('in_invoice','in_refund')"
835
+
836
+
837
+ def read_bills(cur):
838
+ """[(row dict)] β€” posted vendor bills and refunds, keyed on the `account.move` id."""
839
+ have = columns(cur, "account_move")
840
+ if not have:
841
+ return []
842
+ sql = ("SELECT id, name, partner_id, partner_name, invoice_date, invoice_date_due, "
843
+ f" {_col(have, 'amount_untaxed_signed', '0')}, "
844
+ f" {_col(have, 'amount_residual_signed', '0')}, "
845
+ f" {_col(have, 'payment_state', chr(39) + chr(39))}, move_type "
846
+ f"FROM account_move WHERE {_VENDOR_DOCS} AND partner_id IS NOT NULL")
847
+ out = []
848
+ for r in cur.execute(sql).fetchall():
849
+ (mid, name, pid, pname, when, due, untaxed, residual, pay, mtype) = r
850
+ out.append({
851
+ "_id": str(mid),
852
+ "bill_no": str(name or ""),
853
+ "odoo_id": int(mid),
854
+ "vendor": str(pname or ""),
855
+ VENDOR_JOIN_KEY: int(pid),
856
+ "invoice_date": _as_date(when),
857
+ "due_date": _as_date(due),
858
+ "amount_untaxed": float(untaxed or 0.0),
859
+ "residual": float(residual or 0.0),
860
+ "payment_state": str(pay or ""),
861
+ "move_type": str(mtype or ""),
862
+ })
863
+ return out
864
+
865
+
866
+ def read_vendors(cur):
867
+ """[(row dict)] β€” every partner carrying a posted vendor bill, keyed on the `res.partner` id.
868
+
869
+ ⚠ DERIVED FROM THE BILLS, unlike `read_customers` which is deliberately NOT derived from its
870
+ invoices. The asymmetry is intentional and the reason is what that function's own comment
871
+ says: a customer registry sourced from receivables is what kept most Odoo ids out of the store.
872
+ There is no second document universe for vendors β€” a partner with no bill has no payable
873
+ history to show β€” so the bill IS the population, and MEASURED it dangles nothing (0 bills
874
+ carry a null partner; all 393 vendors resolve in `res_partner`).
875
+ """
876
+ have = columns(cur, "res_partner")
877
+ if not have or not columns(cur, "account_move"):
878
+ return []
879
+ sql = (f"SELECT p.id, p.name, {_col(have, 'p.country_name', chr(39) + chr(39))} "
880
+ "FROM res_partner p WHERE p.id IN "
881
+ f" (SELECT partner_id FROM account_move WHERE {_VENDOR_DOCS} "
882
+ " AND partner_id IS NOT NULL)")
883
+ out = []
884
+ for (pid, name, country) in cur.execute(sql).fetchall():
885
+ out.append({
886
+ "_id": str(pid),
887
+ "vendor": str(name or ""),
888
+ "odoo_id": int(pid),
889
+ VENDOR_JOIN_KEY: int(pid),
890
+ "country": str(country or ""),
891
+ })
892
+ return out
893
+
894
+
895
  def customers_from(invoice_rows):
896
  """The partners carrying the given invoice rows β€” the pre-2026-08-09 population builder.
897
 
 
927
  ("products", PRODUCTS_KEY, "Odoo products", product_fields),
928
  ("invoices", INVOICES_KEY, "Odoo invoices", invoice_fields),
929
  ("orders", ORDERS_KEY, "Odoo orders", order_fields),
930
+ # ⭐ WAVE 28 / R1. Measured populations: 19 / 192 / 6,538 / 393 β€” every one of them two orders
931
+ # of magnitude inside `MAX_ROWS`, which is why the answer to "every unique id is a database"
932
+ # is four more spec rows and four readers rather than a new substrate.
933
+ ("agents", AGENTS_KEY, "Odoo agents", agent_fields),
934
+ ("accounts", ACCOUNTS_KEY, "Odoo GL accounts", account_fields),
935
+ ("vendors", VENDORS_KEY, "Odoo vendors", vendor_fields),
936
+ ("bills", BILLS_KEY, "Odoo vendor bills", bill_fields),
937
  )
938
 
939
 
 
957
  "products": read_products(cur),
958
  "invoices": read_invoices(cur, excluded=excluded),
959
  "orders": read_orders(cur, excluded=excluded),
960
+ "agents": read_agents(cur),
961
+ "accounts": read_accounts(cur),
962
+ "vendors": read_vendors(cur),
963
+ "bills": read_bills(cur),
964
  }
965
  problems = []
966
 
api/routes_tables.py CHANGED
@@ -842,5 +842,29 @@ def patch_row(table_key: str, pid: int, body: dict = Body(default=None),
842
  cleared = sorted(k for k in also if k in _ut().PROFILE_PRESET_KEYS)
843
  if cleared:
844
  out["cleared"] = cleared
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
845
  _refresh_relations(session)
846
  return out
 
842
  cleared = sorted(k for k in also if k in _ut().PROFILE_PRESET_KEYS)
843
  if cleared:
844
  out["cleared"] = cleared
845
+ # ⭐⭐ R9's SECOND RE-ARM DOOR β€” the one call that makes `engine.clear_gone` live (wave 28,
846
+ # amendment A5; SESSION B built and gated the function and correctly declared it INERT until
847
+ # this line existed, citing [[flag-shipped-without-its-writer]]).
848
+ #
849
+ # β›” R9 makes a `not_found` handle a TOMBSTONE, not a 30-day backoff: nothing re-buys a dead
850
+ # account on a timer any more. Door 1 β€” correcting the handle β€” needs no wiring, because the
851
+ # verdict is keyed on `(platform, handle)` and a corrected handle simply is not the verdict we
852
+ # recorded. THIS is door 2: a human re-typing the SAME handle, which is how somebody says "try
853
+ # it again, the account is back". Without this call that person has no way back at all, and
854
+ # the failure costs nothing and raises nothing β€” so no spend-shaped test would ever find it.
855
+ #
856
+ # ⚠ GATED ON `updates`, NOT ON `accepted`: re-typing the identical value is the whole case this
857
+ # door exists for, and a no-op write can be filtered out of `accepted`. What matters is that a
858
+ # human touched the handle cell.
859
+ # ⚠ NOT a bare `except: pass`. A swallowed AttributeError here would be exactly the optional-
860
+ # prop silence this wiring exists to prevent β€” if the engine ever loses `clear_gone`, that must
861
+ # be readable in the log rather than degrade into "the re-arm quietly stopped working".
862
+ _pf = _ut().profile_field(table_key, st=session.runtime)
863
+ if _pf and _pf["key"] in updates:
864
+ try:
865
+ import automation_engine as _engine
866
+ _engine.clear_gone(session.runtime, table_key, stored.get(_pf["key"]))
867
+ except Exception as e: # noqa: BLE001
868
+ print(f"[aios-api] clear_gone failed: {type(e).__name__}: {e}")
869
  _refresh_relations(session)
870
  return out
platform/core/user_tables.py CHANGED
@@ -1047,6 +1047,21 @@ def _clean_field(raw, previous=None):
1047
  # column it is editing. Clearing it is deliberate work, not a side effect of renaming a header.
1048
  if prev.get('pinned') is True or raw.get('pinned') is True:
1049
  out['pinned'] = True
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1050
  # An automation column carries the binding the engine reads; it is preserved across a patch
1051
  # rather than re-declared, because the automation editor owns it and this route does not.
1052
  auto = raw.get('automation') if 'automation' in raw else prev.get('automation')
 
1047
  # column it is editing. Clearing it is deliberate work, not a side effect of renaming a header.
1048
  if prev.get('pinned') is True or raw.get('pinned') is True:
1049
  out['pinned'] = True
1050
+ # ⭐ 2026-08-09 (wave 28) β€” `agg` DECIDES WHETHER THE GRID'S TOTALS ROW SUMS THE COLUMN, and
1051
+ # this door silently dropped it while `clean_fields` (:402) kept it. Two validators, one
1052
+ # question, opposite answers β€” D-91's exact class, found from the other end: the register's row
1053
+ # described `clean_fields` as the lax door, and by the time it was read that half had been
1054
+ # fixed while THIS one still diverged.
1055
+ # β›” THE CONSEQUENCE IS QUIET, WHICH IS WHY IT SURVIVED. Every currency and money-rollup column
1056
+ # in the four Odoo databases declares `agg: "sum"`; the spawn writes definitions straight to
1057
+ # the store, so the totals row worked β€” until anyone PATCHED one of those fields (a rename, a
1058
+ # width, a description), at which point the column silently stopped totalling and nothing
1059
+ # anywhere went red. Caught by a gate leg written to assert the LINK bag survives this door,
1060
+ # which found `agg` instead.
1061
+ # ⚠ Sticky across a patch for the same reason as `default` and `pinned`: a client PATCH that
1062
+ # omits the key must not un-total a column as a side effect of renaming its header.
1063
+ if str(raw.get('agg') or prev.get('agg') or '').strip() == 'sum':
1064
+ out['agg'] = 'sum'
1065
  # An automation column carries the binding the engine reads; it is preserved across a patch
1066
  # rather than re-declared, because the automation editor owns it and this route does not.
1067
  auto = raw.get('automation') if 'automation' in raw else prev.get('automation')
web/src/automation/AutomationBuilder.tsx CHANGED
@@ -587,7 +587,15 @@ export default function AutomationBuilder({
587
  */
588
  if (kind === "enrich_instagram")
589
  return {
590
- tier: "anonymous", noFallback: false, postMetrics: false, commentMetrics: false,
 
 
 
 
 
 
 
 
591
  dryRun: false, maxPosts: 10,
592
  fromView: "", sortField: "first_found", sortDir: "desc",
593
  // ⚠ `skipRecent: true` MATCHES `_ensure_enrich_step`'s server seed, deliberately. One
@@ -2052,7 +2060,16 @@ function NewDatabase({
2052
  );
2053
  }
2054
 
2055
- function ActionProps({
 
 
 
 
 
 
 
 
 
2056
  action,
2057
  pinned,
2058
  catalog,
@@ -2433,35 +2450,65 @@ function ActionProps({
2433
  </div>
2434
  ) : null}
2435
 
2436
- <div className="auto-field">
2437
- <label htmlFor="autox-atier">Source</label>
2438
- <select
2439
- id="autox-atier"
2440
- className="auto-input"
2441
- value={String((cfg as { tier?: string }).tier || "anonymous")}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2442
  disabled={busy}
2443
- onChange={(e) => setCfg({ tier: e.target.value })}
2444
- >
2445
- <option value="anonymous">Anonymous β€” free, best effort</option>
2446
- <option value="brightdata">Paid provider β€” costs money per profile</option>
2447
- </select>
2448
- {/* ⚠ THE MONEY SENTENCE IS ON THE MONEY OPTION. R7's rule about the machine steps'
2449
- switches applies to their replacement: a control whose cost is invisible is a
2450
- control nobody can consent to. */}
2451
- <p className="auto-hint">
2452
- {String((cfg as { tier?: string }).tier || "anonymous") === "brightdata"
2453
- ? "Each profile is billed by the provider."
2454
- : "Free, and some profiles will come back thin."}
2455
- </p>
2456
- </div>
2457
  <label className="auto-check">
2458
  <input
 
2459
  type="checkbox"
2460
- checked={!!(cfg as { noFallback?: boolean }).noFallback}
2461
  disabled={busy}
2462
- onChange={(e) => setCfg({ noFallback: e.target.checked })}
2463
  />
2464
- Stop if that source fails, rather than trying the other one
2465
  </label>
2466
  {/*
2467
  ⭐ WAVE 26 Β· ITEM 7 / R2 + C5 β€” TWO CONTROLS, AND THEY ARE INDEPENDENT.
@@ -2497,41 +2544,19 @@ function ActionProps({
2497
  At most 12 β€” the provider returns a profile&rsquo;s top 12 posts and no more.
2498
  </p>
2499
  </div>
2500
- <label className="auto-check">
2501
- <input
2502
- type="checkbox"
2503
- checked={!!(cfg as { postMetrics?: boolean }).postMetrics}
2504
- disabled={busy}
2505
- onChange={(e) => setCfg({ postMetrics: e.target.checked })}
2506
- />
2507
- Also capture per-post engagement
2508
- </label>
2509
- {/* β›” THE ONE THAT MULTIPLIES THE BILL BY THE POST COUNT gets its own sentence, and only
2510
- while it is ON β€” a warning printed permanently is chrome the eye stops reading.
2511
- ⚠ IT IS THE SAME AFFORDANCE THE MONEY SWITCH ABOVE USES (C5: "the SAME affordance as
2512
- the existing Bright Data money switch") β€” a hint line under the control, naming what
2513
- is billed and per what. A distinct badge would read as a different KIND of warning
2514
- for the same kind of fact. */}
2515
- {(cfg as { postMetrics?: boolean }).postMetrics ? (
2516
- <p className="auto-hint">
2517
- Billed per post, not per profile β€” a separate scrape for each post kept above.
2518
- </p>
2519
- ) : null}
2520
- <label className="auto-check">
2521
- <input
2522
- type="checkbox"
2523
- checked={!!(cfg as { commentMetrics?: boolean }).commentMetrics}
2524
- disabled={busy}
2525
- onChange={(e) => setCfg({ commentMetrics: e.target.checked })}
2526
- />
2527
- Also capture per-comment engagement
2528
- </label>
2529
- {(cfg as { commentMetrics?: boolean }).commentMetrics ? (
2530
- <p className="auto-hint">
2531
- Billed by the separate Comments dataset. This is off by default; embedded comments
2532
- already returned with a paid post are retained without this additional request.
2533
- </p>
2534
- ) : null}
2535
  <label className="auto-check">
2536
  <input
2537
  type="checkbox"
 
587
  */
588
  if (kind === "enrich_instagram")
589
  return {
590
+ // ⭐ WAVE 28 Β· R5/R6 β€” `tier: "anonymous"` and `noFallback: false` are GONE from the seed.
591
+ // R5 retired the free ladder from enrichment, so a new action must not be born naming a
592
+ // source strategy the runner no longer consults. The server keeps ACCEPTING both keys on
593
+ // stored definitions and ignores them (C2, D-65's law) β€” but a SEED is what a new step
594
+ // starts with, and seeding a dead key is how a retired concept outlives its removal.
595
+ // ⚠ `postMetrics`/`commentMetrics` stay `false` here and that is R6's ruling verbatim
596
+ // ("always both off until toggled on"), matching `clean_action_config`'s reading of an
597
+ // absent key so the seed and the validator cannot disagree.
598
+ postMetrics: false, commentMetrics: false,
599
  dryRun: false, maxPosts: 10,
600
  fromView: "", sortField: "first_found", sortDir: "desc",
601
  // ⚠ `skipRecent: true` MATCHES `_ensure_enrich_step`'s server seed, deliberately. One
 
2060
  );
2061
  }
2062
 
2063
+ /**
2064
+ * ⭐ WAVE 28 β€” EXPORTED so the include-row render suite can mount the real panel.
2065
+ *
2066
+ * β›” THE EXPORT IS THE POINT, not a convenience. R6's three include rows are markup whose defect
2067
+ * modes are invisible to a source grep: a row that renders with no key attached, a Profile
2068
+ * checkbox that paints unchecked, two rows bound to the same key. `ReviewProps` below is already
2069
+ * exported for the same reason, so this is the file's existing shape rather than a new one.
2070
+ * `_test/` never ships (`deploy_web.py` excludes `_test/` and `_`-prefixed files both).
2071
+ */
2072
+ export function ActionProps({
2073
  action,
2074
  pinned,
2075
  catalog,
 
2450
  </div>
2451
  ) : null}
2452
 
2453
+ {/*
2454
+ ⭐⭐ WAVE 28 Β· R5 / R6 / R7 β€” THE SOURCE QUESTION IS GONE, AND THREE INCLUDE AXES
2455
+ REPLACE IT. What stood here was a `Source` select writing `config.tier`
2456
+ (Anonymous / Paid provider) plus a sibling `noFallback` checkbox β€” ONE control in two
2457
+ pieces, asking the user to choose a VENDOR STRATEGY.
2458
+
2459
+ β›” R5 RETIRED THE QUESTION, not just the control. Enrichment routes per capability to
2460
+ the paid providers and reports blocked on a refusal; there is no thin anonymous row to
2461
+ fall back to, so "which source" and "stop if it fails" no longer have answers a person
2462
+ could give. `tier` and `noFallback` are accepted-and-ignored in stored configs (C2,
2463
+ and D-65's law: never 400 a definition that was legal when it was written) β€” which is
2464
+ why this panel simply stops writing them rather than migrating anything.
2465
+
2466
+ β›” THE ROWS BELOW ARE A TRANSFORM, NOT AN ADDITION, and that distinction is the whole
2467
+ defect risk in this change. `postMetrics` and `commentMetrics` ALREADY had checkboxes
2468
+ 34 lines below this point ("Also capture per-post engagement" / "…per-comment…").
2469
+ Building three NEW rows and leaving those would have put TWO controls on each key β€”
2470
+ which compiles, renders, and satisfies any check asking whether a Post-data switch
2471
+ exists. The old pair is deleted; these carry their keys.
2472
+
2473
+ ⚠ KEYS UNCHANGED ON PURPOSE (C2). Every stored enrich action round-trips untouched;
2474
+ only the labels move. R7: no cost sentence anywhere in this panel β€” the run log keeps
2475
+ honest spend reporting, and a warning printed permanently is chrome the eye stops
2476
+ reading (DESIGN.md Β§4).
2477
+ */}
2478
+ {/* ⚠ A GROUP HEADING, NOT A `<label htmlFor>`. The first cut pointed it at the Post-data
2479
+ input, which is wrong twice over: it claims one row is "the" control for a group of
2480
+ three, and clicking the heading would toggle Posts. `auto-field-label` is this
2481
+ panel's own idiom for naming a group ("Records to enrich" above uses it). */}
2482
+ <p className="auto-field-label">Include</p>
2483
+ {/* β›” DISPLAY-ONLY, AND IT IS NOT DECORATION. The profile IS the unit of enrichment β€”
2484
+ there is no run that skips it β€” so a switch here would be a control that cannot be
2485
+ off ([[wrong-parent-not-broken-control]]). It carries NO config key: rendering it
2486
+ as state would invent a flag no cleaner reads.
2487
+ ⚠ `readOnly` beside `disabled`: a `checked` input with no `onChange` is a React
2488
+ warning, and `readOnly` says the honest thing about why. */}
2489
+ <label className="auto-check">
2490
+ <input id="autox-inc-profile" type="checkbox" checked readOnly disabled />
2491
+ Profile
2492
+ </label>
2493
+ <label className="auto-check">
2494
+ <input
2495
+ id="autox-inc-posts"
2496
+ type="checkbox"
2497
+ checked={!!(cfg as { postMetrics?: boolean }).postMetrics}
2498
  disabled={busy}
2499
+ onChange={(e) => setCfg({ postMetrics: e.target.checked })}
2500
+ />
2501
+ Post data
2502
+ </label>
 
 
 
 
 
 
 
 
 
 
2503
  <label className="auto-check">
2504
  <input
2505
+ id="autox-inc-comments"
2506
  type="checkbox"
2507
+ checked={!!(cfg as { commentMetrics?: boolean }).commentMetrics}
2508
  disabled={busy}
2509
+ onChange={(e) => setCfg({ commentMetrics: e.target.checked })}
2510
  />
2511
+ Comment data
2512
  </label>
2513
  {/*
2514
  ⭐ WAVE 26 Β· ITEM 7 / R2 + C5 β€” TWO CONTROLS, AND THEY ARE INDEPENDENT.
 
2544
  At most 12 β€” the provider returns a profile&rsquo;s top 12 posts and no more.
2545
  </p>
2546
  </div>
2547
+ {/*
2548
+ ⭐ WAVE 28 Β· R6/R7 β€” THE TWO CHECKBOXES THAT STOOD HERE MOVED UP INTO THE INCLUDE
2549
+ GROUP, KEYS AND ALL (`postMetrics`, `commentMetrics`). They are not deleted features:
2550
+ they are the SAME two switches, relabelled "Post data" and "Comment data" and grouped
2551
+ with the always-on Profile row so the three capture axes read as one decision.
2552
+ β›” THEIR CONDITIONAL COST HINTS WENT WITH THEM AND DID NOT COME BACK (R7): "Billed per
2553
+ post, not per profile" and "Billed by the separate Comments dataset". R7 puts spend
2554
+ reporting in the RUN LOG, where it is a measured fact about work already done, rather
2555
+ than in the panel, where it was a permanent caption on a switch.
2556
+ ⚠ Leaving them here as well as above is the duplicate-writer trap this wave's item 1
2557
+ was one literal reading away from shipping β€” two controls on one key, both correct,
2558
+ neither authoritative.
2559
+ */}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2560
  <label className="auto-check">
2561
  <input
2562
  type="checkbox"
web/src/automation/AutomationDetail.tsx CHANGED
@@ -1076,19 +1076,42 @@ export default function AutomationDetail({
1076
  datacenter egress, "204K not 204,312", which rung answers when. The
1077
  CHOICE a person makes is exact-or-estimated and what it costs; the rest
1078
  was the architecture explaining itself. DESIGN.md Β§4. */}
1079
- <h3>Exact counts</h3>
1080
- <p className="auto-hint">
1081
- Real follower numbers and the account&rsquo;s top posts, charged per profile.
1082
- {paidReady ? "" : " Not set up on this workspace yet."}
1083
- </p>
1084
- <h3>Post engagement</h3>
1085
- <p className="auto-hint">
1086
- Likes and comments per post. Charged per post, so it is its own switch.
1087
- </p>
1088
- <h3>Estimated counts</h3>
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1089
  <p className="auto-hint">
1090
- Free, but rounded (204K rather than 204,312) and marked as estimates. Switch
1091
- it off to get exact numbers or nothing.
1092
  </p>
1093
  </>
1094
  ) : null}
 
1076
  datacenter egress, "204K not 204,312", which rung answers when. The
1077
  CHOICE a person makes is exact-or-estimated and what it costs; the rest
1078
  was the architecture explaining itself. DESIGN.md Β§4. */}
1079
+ {/*
1080
+ ⭐⭐ WAVE 28 Β· R5 + R7 β€” THREE HEADINGS DESCRIBED A LADDER THAT NO LONGER
1081
+ EXISTS. "Exact counts" / "Post engagement" / "Estimated counts" named the
1082
+ capture RUNGS, and R5 retired the free anonymous rung from enrichment
1083
+ entirely: exact numbers or nothing, a vendor refusal reports blocked. A
1084
+ panel offering to "switch it off to get exact numbers or nothing" describes
1085
+ a choice the runner stopped having β€” a picture of the engine that disagrees
1086
+ with the engine, which is the defect this module refuses everywhere else.
1087
+ β›” THE MONEY SENTENCES WENT WITH THEM (R7): "charged per profile" and
1088
+ "Charged per post, so it is its own switch". Spend is reported in the RUN
1089
+ LOG against work already done, never as a permanent caption on a control.
1090
+
1091
+ ⚠ `paidReady` SURVIVES, and R5 makes it MORE load-bearing rather than less.
1092
+ It is the one live reader of the flag, and with no free ladder underneath, a
1093
+ workspace with no provider connected gets nothing at all rather than a thin
1094
+ row β€” so the panel says so. That is a system fact, not a price.
1095
+ */}
1096
+ {/*
1097
+ β›” THE SENTENCE MAKES NO CLAIM ABOUT WHAT THE SWITCHES DO, and the first cut
1098
+ did: it read "post and comment data follow the switches above", which is
1099
+ NOT what `postMetrics` gates. Measured against the engine β€”
1100
+ `automation_engine.py:5358` ("likes/comments per post are bought, or the
1101
+ engagement series does not grow") and `connectors_ig.py:710` ("a separate,
1102
+ opt-in, ~13x-cost rung rather than a free by-product") β€” the profile pull
1103
+ keeps up to `maxPosts` posts REGARDLESS of that switch; what the switch buys
1104
+ is per-post ENGAGEMENT. R6 rules the LABEL ("Post data"), and the label is
1105
+ the owner's to set; a caption asserting a mechanism is mine, and this one
1106
+ would have been a picture of the engine that disagrees with the engine β€”
1107
+ the defect this module refuses in five other places.
1108
+ ⚠ ASK ->B is open on whether R5's reshape changes that. Until it is answered
1109
+ the honest panel says the one thing that is true either way.
1110
+ */}
1111
+ <h3>Included</h3>
1112
  <p className="auto-hint">
1113
+ The profile is always read.
1114
+ {paidReady ? "" : " No provider is connected on this workspace yet."}
1115
  </p>
1116
  </>
1117
  ) : null}
web/src/automation/automationApi.ts CHANGED
@@ -31,12 +31,17 @@ export type AutomationKind =
31
  | "discover_instagram"
32
  | "plain";
33
  export type RunState = "idle" | "running" | "ok" | "error" | "partial";
34
- /**
35
- * ⚠ `hiker` is NOT in this union any more (wave 20, D-21) β€” the vendor moved.
36
- * Stored configs still SAY `hiker` and the server maps them forward
37
- * (`TIER_ALIASES`), so nothing in this client should ever write that word.
 
 
 
 
 
 
38
  */
39
- export type CaptureTier = "anonymous" | "brightdata";
40
 
41
  /**
42
  * A node's dot. Deliberately WIDER than `RunState` β€” a single step can be
@@ -1012,10 +1017,21 @@ export const COUNT_LABELS: [string, string][] = [
1012
  // (profile read, media not anonymously readable) but this table did not, so a run whose
1013
  // summary said "1/1 profiles read" showed a chip row reading "0 Read" β€” the chips
1014
  // contradicting the sentence directly above them. A count the summary names must have a chip.
 
 
 
1015
  ["partial", "Profile only"],
1016
  ["blocked", "Blocked"],
1017
  ["error", "Errors"],
1018
- ["paid", "Exact counts"],
 
 
 
 
 
 
 
 
1019
  ["posts", "Posts"],
1020
  ["new_posts", "New posts"],
1021
  // The engagement series only grows while the per-post rung is on, so its
 
31
  | "discover_instagram"
32
  | "plain";
33
  export type RunState = "idle" | "running" | "ok" | "error" | "partial";
34
+ /*
35
+ * ⭐ WAVE 28 Β· R5 β€” `CaptureTier` ("anonymous" | "brightdata") IS DELETED. It was the client's
36
+ * name for the capture-rung choice, and R5 retired that choice: enrichment routes per capability
37
+ * to the paid providers and reports blocked on a refusal, so there is no tier for a client to
38
+ * hold. MEASURED before deleting: the type had ZERO usages anywhere under `src/` β€” it was already
39
+ * a declaration nothing read, which is what a retired concept looks like on the way out.
40
+ * β›” THE SERVER'S OWN VOCABULARY IS UNTOUCHED, deliberately. `TIERS`, `TIER_ALIASES` and the
41
+ * `brightdata` vendor name stay in the engine (`verify_automation.py:6510` fences them), because
42
+ * a stored config still carries `tier` and is accepted-and-ignored rather than refused (C2,
43
+ * D-65's law). What died is the USER CONCEPT, not the wire's tolerance for it.
44
  */
 
45
 
46
  /**
47
  * A node's dot. Deliberately WIDER than `RunState` β€” a single step can be
 
1017
  // (profile read, media not anonymously readable) but this table did not, so a run whose
1018
  // summary said "1/1 profiles read" showed a chip row reading "0 Read" β€” the chips
1019
  // contradicting the sentence directly above them. A count the summary names must have a chip.
1020
+ // ⭐ WAVE 28 Β· R5 β€” KEPT, and the label is unchanged on purpose. "Profile only" describes WHAT
1021
+ // WAS CAPTURED (the profile answered, its media did not), which is still a state a single paid
1022
+ // pipeline reaches; it was never tier vocabulary the way "Exact counts" was.
1023
  ["partial", "Profile only"],
1024
  ["blocked", "Blocked"],
1025
  ["error", "Errors"],
1026
+ // ⭐ WAVE 28 Β· R5/R7 β€” WAS "Exact counts", WHICH NOW DISTINGUISHES NOTHING. The word only meant
1027
+ // anything against the free rung's "estimated" counts, and R5 retired that rung: every number a
1028
+ // run reports is exact, so a chip claiming exactness is a label with no complement.
1029
+ // β›” THE CHIP ITSELF STAYS, and R7 is why: cost prose leaves the PANEL and spend reporting lives
1030
+ // in the RUN LOG, against work already done. This is that reporting β€” the count of profiles the
1031
+ // run actually bought. Deleting it would take the honest half out with the prose.
1032
+ // ⚠ SAFE IN BOTH DIRECTIONS if B's R5 work stops emitting the count: the renderer filters on
1033
+ // `typeof r.counts?.[k] === "number"`, so an absent count paints no chip rather than a zero.
1034
+ ["paid", "Paid profiles"],
1035
  ["posts", "Posts"],
1036
  ["new_posts", "New posts"],
1037
  // The engagement series only grows while the per-post rung is on, so its
web/src/automation/steps.ts CHANGED
@@ -85,12 +85,15 @@ export interface Step {
85
  /**
86
  * Number the server's nodes for display.
87
  *
88
- * ⭐ THE NUMBER IS `col + 1`, NOT the array index, and that is the honest one. The
89
- * `field_instagram` graph forks: `capture_paid` and `capture_anon` both sit at
90
- * `col: 3` because they are the SAME position in the flow reached two ways (the
91
- * paid rung answers, or the anonymous ladder does). Numbering them 4 and 5 would
92
- * state a sequence that never happens. They share a number and the second is
93
- * marked `alt`, so the list says "either of these" rather than "then".
 
 
 
94
  *
95
  * "Step 1 is always the Trigger" (R9) therefore falls out of the payload β€” the
96
  * trigger is the node at `col: 0` β€” instead of being asserted by this client. If
@@ -150,10 +153,14 @@ export interface PanelGroup {
150
  *
151
  * β›” GROUPED, NEVER MAPPED ONE-TO-ONE, and the difference is a defect rather than a nicety.
152
  * `panel` is MANY-TO-ONE over nodes, which is easy to miss because the old surface hid it: you
153
- * clicked ONE card and got ONE panel. `field_instagram` gives `capture`, `capture_paid`,
154
- * `capture_anon` AND `capture_metrics` all `panel: "capture"` (`automation_engine.py:3334-3352`),
155
- * and `discover_instagram` gives both of its nodes `panel: "find"` (`:3381`/`:3387`). So a body
156
- * rendered per NODE would print the capture prose four times over, and β€” the one that actually
 
 
 
 
157
  * loses work β€” mount the 21-toggle discovery filter TWICE against a single `preds` array, two
158
  * editors writing one piece of state where whichever blurred last silently wins.
159
  *
 
85
  /**
86
  * Number the server's nodes for display.
87
  *
88
+ * ⭐ THE NUMBER IS `col + 1`, NOT the array index, and that is the honest one. It exists
89
+ * because a graph can FORK: two nodes at the same `col` are the SAME position in the flow
90
+ * reached two ways, and numbering them 4 and 5 would state a sequence that never happens.
91
+ * They share a number and the second is marked `alt`, so the list reads "either of these"
92
+ * rather than "then".
93
+ * ⚠ 2026-08-09 (wave 28, R5): the fork this was WRITTEN for is gone β€” `capture_paid` and
94
+ * `capture_anon` were the paid-rung/anonymous-ladder pair, and the ladder is retired, so
95
+ * today's graph is linear. The `col`-based rule STAYS because it is about forks in general,
96
+ * not about those two nodes; this note records that no shipped graph currently exercises it.
97
  *
98
  * "Step 1 is always the Trigger" (R9) therefore falls out of the payload β€” the
99
  * trigger is the node at `col: 0` β€” instead of being asserted by this client. If
 
153
  *
154
  * β›” GROUPED, NEVER MAPPED ONE-TO-ONE, and the difference is a defect rather than a nicety.
155
  * `panel` is MANY-TO-ONE over nodes, which is easy to miss because the old surface hid it: you
156
+ * clicked ONE card and got ONE panel. `discover_instagram` gives both of its nodes
157
+ * `panel: "find"`, and `field_instagram` gives every capture node `panel: "capture"`.
158
+ * ⚠ 2026-08-09 (wave 28, R5/C3): this used to name `capture`, `capture_paid`, `capture_anon`
159
+ * AND `capture_metrics` with a line citation β€” all four ids are DELETED and the citation was
160
+ * stale, which is worse than vague, because a stale line number reads as authoritative. The
161
+ * grouping rule is unchanged: the capture nodes are now `capture_posts` + `capture_comments`.
162
+ * So a body rendered per NODE would print the capture prose once per node, and β€” the one that
163
+ * actually
164
  * loses work β€” mount the 21-toggle discovery filter TWICE against a single `preds` array, two
165
  * editors writing one piece of state where whichever blurred last silently wins.
166
  *
web/src/customer-grid/ColumnMenu.tsx CHANGED
@@ -5,9 +5,10 @@ import { FieldTypeIcon, MenuLabel } from "./icons";
5
  import { FieldSelectButton } from "./FieldSelect";
6
  import { CODE_LANGUAGE_LABELS, CODE_LANGUAGES, CREATABLE_TYPES, choiceOptions, choiceRenames,
7
  codeLanguageOf, directionLabel, isMachineOwned,
8
- isProfileField, parseOptions, ratingMax, ROLLUP_FN_LABELS, ROLLUP_FNS } from "./types";
 
9
  import type { Field, FieldFormat, FieldScope, FieldType, Measure, RollupCondition, RollupFn,
10
- RollupSource, Viewer } from "./types";
11
  import type { LinkTarget, RollupSourceOffer } from "./apiBridge";
12
  import type { WindowSpec } from "./windows";
13
  import { normalizeWindow, windowLabel } from "./windows";
@@ -279,6 +280,41 @@ interface OptionDraft {
279
  }
280
 
281
  const DEFAULT_OPTION_LABELS = ["Not started", "In progress", "Blocked", "Done"];
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
282
  let optionDraftSequence = 0;
283
 
284
  function nextOptionDraftId(): string {
@@ -1234,6 +1270,37 @@ function ExtraTypeEditor({
1234
  </select>
1235
  </label>
1236
  ) : null}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1237
  <div className="cg-rollup-conditions">
1238
  <div className="cg-rollup-conditions__head">
1239
  <span>Conditions</span>
@@ -1276,34 +1343,117 @@ function ExtraTypeEditor({
1276
  <option key={targetField.key} value={targetField.key}>{targetField.label}</option>
1277
  ))}
1278
  </select>
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1279
  <select
1280
  className="cg-input"
1281
  aria-label={`Condition ${index + 1} operator`}
1282
- value={condition.op}
1283
- onChange={(event) => onRollupConditions?.(rollupConditions.map((item, i) =>
1284
- i === index
1285
- ? { ...item, op: event.target.value as RollupCondition["op"] }
1286
- : item
1287
- ))}
 
 
 
 
 
 
 
 
 
1288
  >
1289
- <option value="eq">is</option>
1290
- <option value="neq">is not</option>
1291
- <option value="contains">contains</option>
1292
- <option value="not_contains">does not contain</option>
1293
- <option value="is_empty">is empty</option>
1294
- <option value="is_not_empty">is not empty</option>
1295
- <option value="gt">is greater than</option>
1296
- <option value="gte">is at least</option>
1297
- <option value="lt">is less than</option>
1298
- <option value="lte">is at most</option>
 
 
 
 
 
 
 
1299
  </select>
1300
- {!(["is_empty", "is_not_empty"] as string[]).includes(condition.op) ? (
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1301
  <input
1302
  className="cg-input"
1303
  aria-label={`Condition ${index + 1} value`}
1304
  value={condition.value ?? ""}
1305
  onChange={(event) => onRollupConditions?.(rollupConditions.map((item, i) =>
1306
- i === index ? { ...item, value: event.target.value } : item
1307
  ))}
1308
  />
1309
  ) : null}
 
5
  import { FieldSelectButton } from "./FieldSelect";
6
  import { CODE_LANGUAGE_LABELS, CODE_LANGUAGES, CREATABLE_TYPES, choiceOptions, choiceRenames,
7
  codeLanguageOf, directionLabel, isMachineOwned,
8
+ isProfileField, parseOptions, ratingMax, ROLLUP_FN_LABELS, ROLLUP_FNS,
9
+ ROLLUP_REF_OPS } from "./types";
10
  import type { Field, FieldFormat, FieldScope, FieldType, Measure, RollupCondition, RollupFn,
11
+ RollupRefOp, RollupSource, Viewer } from "./types";
12
  import type { LinkTarget, RollupSourceOffer } from "./apiBridge";
13
  import type { WindowSpec } from "./windows";
14
  import { normalizeWindow, windowLabel } from "./windows";
 
280
  }
281
 
282
  const DEFAULT_OPTION_LABELS = ["Not started", "In progress", "Blocked", "Done"];
283
+
284
+ /**
285
+ * ⭐ WAVE 28 Β· R4 / C1 β€” how a SET-STATISTIC comparison reads in the operator list.
286
+ *
287
+ * β›” KEYED BY `RollupRefOp`, so the day `core.user_tables.ROLLUP_REF_OPS` gains or loses an
288
+ * operator this map fails to compile rather than rendering a bare identifier β€” the same
289
+ * discipline `ROLLUP_FN_LABELS` uses, and the reason the Οƒ picker maps `ROLLUP_REF_OPS` instead
290
+ * of hand-filtering the full operator list.
291
+ * ⚠ The wording names the STATISTIC, not the arithmetic: "2 Οƒ above the mean" is the sentence a
292
+ * person is trying to write, and "is greater than mean + kΒ·stdev" is the implementation of it.
293
+ */
294
+ /**
295
+ * ⭐ MIRRORS `core.user_tables.ROLLUP_MAX_SIGMAS`, and `verify_rollup_editor.py` HOLDS THE TWO IN
296
+ * STEP by importing the server's constant and comparing (a real enforcer this time β€” `types.ts`
297
+ * spent a wave claiming one that contained zero occurrences of the word it guarded,
298
+ * [[limit-with-no-enforcer]]).
299
+ *
300
+ * β›” WHY THE CONTROL MUST KNOW THE BOUND AT ALL. `_clean_rollup` REFUSES the whole field outside
301
+ * this range β€” not the leaf, the FIELD β€” so an unbounded box invites a value and then answers a
302
+ * 400 naming a cap the control never mentioned. That is `maxPosts` exactly: it offered 200 while
303
+ * the save door refused anything over 12 ([[default-must-pass-its-own-guard]]). A control must
304
+ * display what it will SEND.
305
+ */
306
+ const MAX_SIGMAS = 10;
307
+
308
+ /** The default when a Οƒ row is born or its box is left empty β€” C1's worked example uses 2. */
309
+ const DEFAULT_SIGMAS = 2;
310
+
311
+ const REF_OP_LABELS: Record<RollupRefOp, string> = {
312
+ gt: "is more than … Οƒ above the mean",
313
+ gte: "is at least … Οƒ above the mean",
314
+ lt: "is less than … Οƒ above the mean",
315
+ lte: "is at most … Οƒ above the mean",
316
+ };
317
+
318
  let optionDraftSequence = 0;
319
 
320
  function nextOptionDraftId(): string {
 
1270
  </select>
1271
  </label>
1272
  ) : null}
1273
+ {/*
1274
+ ⭐⭐ 2026-08-09 Β· D-99 β€” THE STATE THE OWNER'S REPORT DESCRIBES, SAID OUT LOUD.
1275
+ Owner: *"if we want to use Rollup field to apply to the related database, it doesn't
1276
+ work. Because the linked Field can't be selected yet."*
1277
+
1278
+ β›” WHAT IS ACTUALLY REACHABLE HERE, measured rather than guessed. The `Through` picker
1279
+ offers EVERY link column β€” `linkFields` filters on `f.type === "link"` and excludes
1280
+ nothing preset, derived or `automation:`-tagged, so the scouted hypothesis ("the picker
1281
+ omits preset links") is refuted at source. What CAN happen is one step later: the
1282
+ target is resolved out of `linkTargets`, which is `GET /tables` filtered by
1283
+ `user_tables.may_open` (creator, admin, or an explicit share). An automation-OWNED child
1284
+ dataset is stamped with a machine owner, so for a user who is neither its creator nor an
1285
+ admin it is simply ABSENT from that list β€” and then `targetFields` is `[]`, this Column
1286
+ picker holds nothing but its placeholder, and `Add condition` below is disabled.
1287
+
1288
+ ⚠ AND UNTIL NOW IT SAID NOTHING. An empty picker beside a dead button reads as "this
1289
+ feature is broken"; it is in fact a permission fact about ONE database, and the fix for
1290
+ it is a share, not a bug report. This is the absent-vs-empty third state the automation
1291
+ panel already distinguishes ("No view list was offered for that database") β€” the same
1292
+ discipline, applied to the surface that actually got reported.
1293
+ β›” IT NAMES THE TABLE RATHER THAN APOLOGISING. `chosenLink.link.table` is the key the
1294
+ bag points at, and printing it is what turns "it doesn't work" into a thing to go and
1295
+ grant.
1296
+ */}
1297
+ {chosenLink && !target ? (
1298
+ <div className="cg-field-hint">
1299
+ This link points at <b>{chosenLink.link?.table}</b>, which is not among the databases
1300
+ you can open β€” so its columns cannot be listed here and the conditions below stay
1301
+ unavailable. Ask an admin to share that database with you, then reopen this editor.
1302
+ </div>
1303
+ ) : null}
1304
  <div className="cg-rollup-conditions">
1305
  <div className="cg-rollup-conditions__head">
1306
  <span>Conditions</span>
 
1343
  <option key={targetField.key} value={targetField.key}>{targetField.label}</option>
1344
  ))}
1345
  </select>
1346
+ {/*
1347
+ ⭐⭐ WAVE 28 Β· R4 / C1 β€” THE THRESHOLD MAY NOW BE THE SET'S OWN STATISTICS.
1348
+ Owner's request (Nurilab): an average of the last 10 posts with outliers trimmed β€”
1349
+ "anything beyond 2 sigma". That is not a new aggregator, it is a CONDITION whose
1350
+ threshold is computed from the scoped rows themselves: mean + k*stdev of the same
1351
+ column, over the window `sortBy`+`limit` kept, BEFORE the conditions filter.
1352
+
1353
+ β›” ONE CONTROL, NOT TWO, AND THAT IS WHAT MAKES THE XOR SAFE. `value` and `ref` may
1354
+ never both be present β€” the server refuses the whole FIELD, not just the leaf β€” so
1355
+ a separate "compare against…" switch beside the operator would leave the editor
1356
+ free to write a `ref` while a stale `value` still rode along, producing a 400 on
1357
+ Save that names a key the user never typed. Folding the choice into the operator
1358
+ makes the illegal state unrepresentable: picking a plain op WRITES `value` and
1359
+ drops `ref`, picking a statistic op does the reverse, and there is no third path.
1360
+
1361
+ ⚠ THE OPS ARE `ROLLUP_REF_OPS`, IMPORTED, never a hand-filtered copy of the full
1362
+ list. `eq`/`neq` against a computed float is a coin flip on binary representation
1363
+ and `contains` against a number never matches β€” the server draws that line and a
1364
+ second copy here would be free to drift from it.
1365
+ ⚠ THE `value` IS COMPOUND (`ref:gt`) so the select always resolves to exactly one
1366
+ option. A `<select>` whose value matches nothing renders the FIRST option while
1367
+ the stored key says otherwise, and the next patch writes the lie back
1368
+ ([[cg-condition-builder-items]]).
1369
+ */}
1370
  <select
1371
  className="cg-input"
1372
  aria-label={`Condition ${index + 1} operator`}
1373
+ value={condition.ref ? `ref:${condition.op}` : condition.op}
1374
+ onChange={(event) => {
1375
+ const raw = event.target.value;
1376
+ const isRef = raw.startsWith("ref:");
1377
+ const op = (isRef ? raw.slice(4) : raw) as RollupCondition["op"];
1378
+ // β›” THE LEAF IS REBUILT, NEVER SPREAD-AND-PATCHED. `{ ...item, ref: undefined }`
1379
+ // leaves the key present-and-undefined; `JSON.stringify` drops it today, but the
1380
+ // contract is "the key is absent", and relying on a serialiser's treatment of
1381
+ // `undefined` to enforce a server-side XOR is a guarantee held by accident.
1382
+ const next: RollupCondition = isRef
1383
+ ? { field: condition.field, op,
1384
+ ref: { sigmas: condition.ref?.sigmas ?? DEFAULT_SIGMAS } }
1385
+ : { field: condition.field, op, value: condition.value ?? "" };
1386
+ onRollupConditions?.(rollupConditions.map((item, i) => (i === index ? next : item)));
1387
+ }}
1388
  >
1389
+ <optgroup label="Compared with a value">
1390
+ <option value="eq">is</option>
1391
+ <option value="neq">is not</option>
1392
+ <option value="contains">contains</option>
1393
+ <option value="not_contains">does not contain</option>
1394
+ <option value="is_empty">is empty</option>
1395
+ <option value="is_not_empty">is not empty</option>
1396
+ <option value="gt">is greater than</option>
1397
+ <option value="gte">is at least</option>
1398
+ <option value="lt">is less than</option>
1399
+ <option value="lte">is at most</option>
1400
+ </optgroup>
1401
+ <optgroup label="Compared with the set's own spread">
1402
+ {ROLLUP_REF_OPS.map((op) => (
1403
+ <option key={op} value={`ref:${op}`}>{REF_OP_LABELS[op]}</option>
1404
+ ))}
1405
+ </optgroup>
1406
  </select>
1407
+ {condition.ref ? (
1408
+ <label className="cg-cond-ref">
1409
+ {/* ⚠ A BARE NUMBER BOX WOULD BE A CONTROL THAT WILL NOT SAY WHAT IT SETS. The
1410
+ unit is sigmas and the sign is meaningful (negative selects the LOW tail), so
1411
+ both are printed beside the input rather than left to be inferred from a
1412
+ placeholder. */}
1413
+ <input
1414
+ className="cg-input"
1415
+ type="number"
1416
+ step="0.5"
1417
+ /* β›” THE SERVER'S OWN BOUND, BOTH WAYS. Outside it `_clean_rollup` refuses the
1418
+ whole FIELD, so an unbounded box would invite a 400 naming a cap it never
1419
+ showed β€” `maxPosts`'s scar, in a control built the same day it was quoted. */
1420
+ min={-MAX_SIGMAS}
1421
+ max={MAX_SIGMAS}
1422
+ aria-label={`Condition ${index + 1} standard deviations from the mean`}
1423
+ /* ⚠ UNCONTROLLED + COMMIT ON BLUR, the idiom `maxPosts` uses and for its
1424
+ reason: a controlled box patching per keystroke stores "-" as 0 and "1" on
1425
+ the way to "1.5" as a legal value, and `Number("")` is 0 β€” so simply
1426
+ CLEARING the box would silently store "0 sigma", a real but different
1427
+ query, and the field could never be retyped because it snaps back. */
1428
+ defaultValue={String(condition.ref.sigmas)}
1429
+ onBlur={(event) => {
1430
+ const raw = Number(event.target.value);
1431
+ // Empty/garbage falls back to the default rather than to 0 β€” 0 means "at
1432
+ // the mean" and is a value somebody must choose, never one they land on by
1433
+ // deleting a character.
1434
+ const parsed = event.target.value.trim() === "" || !Number.isFinite(raw)
1435
+ ? DEFAULT_SIGMAS
1436
+ : raw;
1437
+ // CLAMPED, not refused: the box is a dial, and a dial that rejects is a
1438
+ // dead control. The clamp is what makes "displays what it will send" true.
1439
+ const sigmas = Math.max(-MAX_SIGMAS, Math.min(MAX_SIGMAS, parsed));
1440
+ if (String(sigmas) !== event.target.value) event.target.value = String(sigmas);
1441
+ onRollupConditions?.(rollupConditions.map((item, i) => (
1442
+ i === index
1443
+ ? { field: item.field, op: item.op, ref: { sigmas } }
1444
+ : item
1445
+ )));
1446
+ }}
1447
+ />
1448
+ <span>Οƒ from the mean of this column (negative reads below it)</span>
1449
+ </label>
1450
+ ) : !(["is_empty", "is_not_empty"] as string[]).includes(condition.op) ? (
1451
  <input
1452
  className="cg-input"
1453
  aria-label={`Condition ${index + 1} value`}
1454
  value={condition.value ?? ""}
1455
  onChange={(event) => onRollupConditions?.(rollupConditions.map((item, i) =>
1456
+ i === index ? { field: item.field, op: item.op, value: event.target.value } : item
1457
  ))}
1458
  />
1459
  ) : null}
web/src/index.css CHANGED
@@ -1478,6 +1478,29 @@ body {
1478
  width: 30px;
1479
  height: 30px;
1480
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1481
  /* why a control is unavailable, in the place the control would have been. An empty popover
1482
  reads as broken; a sentence reads as a decision (CG-3). */
1483
  .cg-pop-note {
 
1478
  width: 30px;
1479
  height: 30px;
1480
  }
1481
+
1482
+ /* ── WAVE 28 Β· R4 / C1 (session C) β€” the set-statistic threshold's input ──────────────────────
1483
+ A condition row is a 3-column grid (field, operator, remove) and the ordinary value input
1484
+ already spans the first two on a second line. The sigma input needs the same span PLUS its
1485
+ unit beside it: the number is meaningless alone, and the sign is load-bearing (negative reads
1486
+ BELOW the mean), so a bare box would be a control that will not say what it sets
1487
+ ([[wrong-parent-not-broken-control]]).
1488
+ ⚠ `grid-column` on the LABEL, not on its input: the input is now a grandchild of the row, so
1489
+ the `.cg-rollup-condition > input` rule above no longer reaches it β€” which is the trap that
1490
+ turns an extra child into an extra column. */
1491
+ .cg-rollup-condition > .cg-cond-ref {
1492
+ grid-column: 1 / span 2;
1493
+ display: grid;
1494
+ grid-template-columns: 84px minmax(0, 1fr);
1495
+ align-items: center;
1496
+ gap: 8px;
1497
+ }
1498
+
1499
+ .cg-rollup-condition > .cg-cond-ref > span {
1500
+ color: var(--cg-muted);
1501
+ font-size: var(--lp-fs-2xs);
1502
+ line-height: 1.35;
1503
+ }
1504
  /* why a control is unavailable, in the place the control would have been. An empty popover
1505
  reads as broken; a sentence reads as a decision (CG-3). */
1506
  .cg-pop-note {