Deploy AIOS web (React glide grid + FastAPI slice)
Browse files- RELEASES.json +1 -1
- VERSION +1 -1
- api/automation_engine.py +335 -150
- api/connectors_ig.py +160 -364
- api/odoo_relational.py +334 -4
- api/routes_tables.py +24 -0
- platform/core/user_tables.py +15 -0
- web/src/automation/AutomationBuilder.tsx +85 -60
- web/src/automation/AutomationDetail.tsx +35 -12
- web/src/automation/automationApi.ts +22 -6
- web/src/automation/steps.ts +17 -10
- web/src/customer-grid/ColumnMenu.tsx +170 -20
- web/src/index.css +23 -0
RELEASES.json
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
{
|
| 2 |
-
"current": "
|
| 3 |
"releases": [
|
| 4 |
{
|
| 5 |
"version": "v24",
|
|
|
|
| 1 |
{
|
| 2 |
+
"current": "e43869a",
|
| 3 |
"releases": [
|
| 4 |
{
|
| 5 |
"version": "v24",
|
VERSION
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
e43869a
|
api/automation_engine.py
CHANGED
|
@@ -1233,9 +1233,9 @@ def clean_config(kind, raw, previous=None):
|
|
| 1233 |
|
| 1234 |
β THE NODE-SWITCH FLAGS FALL BACK TO `previous` WHEN THE KEY IS ABSENT, and that is a rail
|
| 1235 |
rather than a nicety. `patch` replaces the whole config, and the canvas's config panels do not
|
| 1236 |
-
edit `
|
| 1237 |
-
panel that never knew about them would silently
|
| 1238 |
-
|
| 1239 |
present β take it, including `false`.
|
| 1240 |
"""
|
| 1241 |
raw = raw if isinstance(raw, dict) else {}
|
|
@@ -1292,13 +1292,17 @@ def clean_config(kind, raw, previous=None):
|
|
| 1292 |
fkey = _s(raw.get("fieldKey"), 80).strip()
|
| 1293 |
if not fkey:
|
| 1294 |
return None, "pick the automation column this run writes into"
|
| 1295 |
-
#
|
| 1296 |
-
#
|
| 1297 |
-
#
|
| 1298 |
-
#
|
| 1299 |
-
|
| 1300 |
-
|
| 1301 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1302 |
# C5: absent β keep whatever is stored, like every other switch in this branch β a panel
|
| 1303 |
# that does not edit the post count must not reset it to the default on Save. And an
|
| 1304 |
# INHERITED value is clamped rather than refused; see `clean_max_posts`.
|
|
@@ -1309,8 +1313,6 @@ def clean_config(kind, raw, previous=None):
|
|
| 1309 |
return None, perr
|
| 1310 |
return {"targetTable": table, "fieldKey": fkey,
|
| 1311 |
"urlField": _s(raw.get("urlField"), 80).strip(),
|
| 1312 |
-
"tier": tier or "anonymous",
|
| 1313 |
-
"noFallback": flag("noFallback"),
|
| 1314 |
# β THE ONE FLAG THAT MULTIPLIES THE BILL BY THE POST COUNT. Off unless asked
|
| 1315 |
# for: the Profiles row carries post IDENTITY for free but NO engagement
|
| 1316 |
# (measured 2026-08-05), so likes/comments cost one extra vendor record PER POST.
|
|
@@ -1515,8 +1517,7 @@ def _ensure_enrich_step(flow_raw):
|
|
| 1515 |
# with only step 1 lands this at the end, which IS step 2.
|
| 1516 |
seeded.insert(1, {
|
| 1517 |
"id": "act_enrich", "kind": "enrich_instagram", "enabled": True, "when": None,
|
| 1518 |
-
"config": {"
|
| 1519 |
-
"commentMetrics": False,
|
| 1520 |
"dryRun": False, "maxPosts": DEFAULT_POSTS_PER_PULL,
|
| 1521 |
"fromView": "", "sortField": DEFAULT_ENRICH_SORT, "sortDir": "desc",
|
| 1522 |
"limit": DEFAULT_ENRICH_LIMIT, "skipRecent": True,
|
|
@@ -2593,7 +2594,7 @@ from connectors_ig import ( # noqa: E402
|
|
| 2593 |
_first, _ig_int,
|
| 2594 |
bd_call, bd_filter_rows, bd_filter_start, bd_filter_status, bd_ready,
|
| 2595 |
bd_snapshot_progress, depth_refusal,
|
| 2596 |
-
ig_handle, pull_profile,
|
| 2597 |
)
|
| 2598 |
|
| 2599 |
|
|
@@ -4620,22 +4621,24 @@ def preset_cells(res, pulled):
|
|
| 4620 |
|
| 4621 |
def run_field_instagram(rt, defn, username="automation", log=print, step=_no_step,
|
| 4622 |
rows=None):
|
| 4623 |
-
"""Automation #2 (R7
|
| 4624 |
-
|
| 4625 |
-
|
| 4626 |
-
|
|
|
|
|
|
|
|
|
|
| 4627 |
cfg = defn.get("config") or {}
|
| 4628 |
table_key, fkey = cfg.get("targetTable"), cfg.get("fieldKey")
|
| 4629 |
-
tier = clean_tier(cfg.get("tier")) or "anonymous"
|
| 4630 |
-
paid = tier == "brightdata"
|
| 4631 |
-
fallback = not cfg.get("noFallback")
|
| 4632 |
post_metrics = bool(cfg.get("postMetrics"))
|
| 4633 |
comment_metrics = bool(cfg.get("commentMetrics"))
|
| 4634 |
dry = bool(cfg.get("dryRun"))
|
| 4635 |
-
|
| 4636 |
-
|
| 4637 |
-
|
| 4638 |
-
|
|
|
|
|
|
|
| 4639 |
"write": "idle"}
|
| 4640 |
t = ut_get(rt, table_key)
|
| 4641 |
if not t:
|
|
@@ -4673,7 +4676,6 @@ def run_field_instagram(rt, defn, username="automation", log=print, step=_no_ste
|
|
| 4673 |
# `MAX_UT_IG_ROWS` makes the same loop hundreds of millions of dict copies, i.e. an automation
|
| 4674 |
# that no longer finishes. **Raising a cap and batching its writer are ONE change.** (W19-C.)
|
| 4675 |
in_snaps, in_posts, in_psnaps, in_comments = [], [], [], []
|
| 4676 |
-
paid_tried = paid_answered = anon_tried = anon_answered = False
|
| 4677 |
targets = [(rid, str(r.get(url_field, "") or "").strip())
|
| 4678 |
for rid, r in rows.items() if str(r.get(url_field, "") or "").strip()]
|
| 4679 |
for i, (rid, url) in enumerate(targets):
|
|
@@ -4685,24 +4687,17 @@ def run_field_instagram(rt, defn, username="automation", log=print, step=_no_ste
|
|
| 4685 |
step(f"Capturing profile {i + 1} of {len(targets)}")
|
| 4686 |
counts["profiles"] += 1
|
| 4687 |
res = pull_profile(url, max_posts=cfg.get("maxPosts") or DEFAULT_POSTS_PER_PULL, log=log,
|
| 4688 |
-
|
| 4689 |
-
comment_metrics=comment_metrics,
|
| 4690 |
pending_metrics=(pending_metrics if not dry else None))
|
| 4691 |
pulled = _iso()
|
| 4692 |
via = res.get("via") or ""
|
| 4693 |
read_ok = res["state"] in ("ok", "partial")
|
| 4694 |
-
#
|
| 4695 |
-
|
| 4696 |
-
|
| 4697 |
-
|
| 4698 |
-
|
| 4699 |
-
|
| 4700 |
-
elif fallback:
|
| 4701 |
-
anon_tried = True
|
| 4702 |
-
anon_answered = anon_answered or read_ok
|
| 4703 |
-
else:
|
| 4704 |
-
anon_tried = True
|
| 4705 |
-
anon_answered = anon_answered or read_ok
|
| 4706 |
if read_ok:
|
| 4707 |
counts["ok" if res["state"] == "ok" else "partial"] += 1
|
| 4708 |
prof = res["profile"]
|
|
@@ -4803,21 +4798,24 @@ def run_field_instagram(rt, defn, username="automation", log=print, step=_no_ste
|
|
| 4803 |
else:
|
| 4804 |
cap_state = "partial" if (counts["blocked"] or counts["error"]
|
| 4805 |
or counts["partial"]) else "ok"
|
| 4806 |
-
|
| 4807 |
-
|
| 4808 |
-
|
| 4809 |
-
|
| 4810 |
-
|
| 4811 |
# β MEASURED, LIKE EVERY OTHER DOT: it is `ok` only if an engagement row was actually
|
| 4812 |
# appended. "It was switched on" is not the same fact as "it answered", and painting the
|
| 4813 |
# second from the first is the fabrication `node_status` refuses to make.
|
| 4814 |
# β AND `blocked` IS A CLAIM ABOUT THE VENDOR, so it needs something to have been ASKED. A
|
| 4815 |
# run that found no posts to enrich did not have a rung refuse it β nothing was requested β
|
| 4816 |
# so that reads `skipped`, the same word an off switch earns.
|
| 4817 |
-
steps["
|
| 4818 |
-
|
| 4819 |
-
|
| 4820 |
-
|
|
|
|
|
|
|
|
|
|
| 4821 |
|
| 4822 |
summary = (f"{read}/{counts['profiles']} profiles read, {counts['posts']} posts "
|
| 4823 |
f"({counts['new_posts']} new)")
|
|
@@ -5351,17 +5349,23 @@ NODE_STATES = ("idle", "ok", "partial", "error", "blocked", "skipped")
|
|
| 5351 |
#: deliberate answer rather than an omission: a "Fetch the page" step that can be turned off is
|
| 5352 |
#: not an automation with a disabled step, it is a broken automation with a lie on it. The ones
|
| 5353 |
#: here are all REAL β each changes what the next run does:
|
| 5354 |
-
#: schedule
|
| 5355 |
-
#:
|
| 5356 |
-
#:
|
| 5357 |
-
#:
|
| 5358 |
-
#:
|
| 5359 |
-
#:
|
| 5360 |
-
#:
|
| 5361 |
-
#:
|
| 5362 |
-
|
| 5363 |
-
|
| 5364 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5365 |
|
| 5366 |
|
| 5367 |
def _cron_label(cron):
|
|
@@ -5445,56 +5449,46 @@ def graph(defn):
|
|
| 5445 |
edges = []
|
| 5446 |
|
| 5447 |
if defn.get("kind") == "field_instagram":
|
| 5448 |
-
|
| 5449 |
-
|
| 5450 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5451 |
metrics_on = bool(cfg.get("postMetrics"))
|
|
|
|
| 5452 |
max_posts = cfg.get("maxPosts") or DEFAULT_POSTS_PER_PULL
|
| 5453 |
nodes += [
|
| 5454 |
node("source", "source", "Profile set", cfg.get("targetTable") or "No database", 1,
|
| 5455 |
panel="source",
|
| 5456 |
detail=f"URL column: {cfg.get('urlField')}" if cfg.get("urlField")
|
| 5457 |
else "URL column: the one the field is bound to"),
|
| 5458 |
-
node("capture", "branch", "Capture",
|
| 5459 |
-
"Exact counts, then estimates" if paid else "Estimated counts only", 2, panel="capture",
|
| 5460 |
-
detail=f"Up to {max_posts} posts per profile"),
|
| 5461 |
-
node("capture_paid", "capture", "Exact counts",
|
| 5462 |
-
"Exact followers and top posts" if paid else "Off",
|
| 5463 |
-
3, row=-1, panel="capture", on=paid,
|
| 5464 |
-
detail="" if bd_ready() else "Not set up yet"),
|
| 5465 |
-
node("capture_anon", "capture", "Estimated counts",
|
| 5466 |
-
"Free, rounded numbers" if anon_on else "Off",
|
| 5467 |
-
3, row=1, panel="capture", on=anon_on,
|
| 5468 |
-
detail="Rounded to the nearest thousand"),
|
| 5469 |
# β ITS OWN NODE BECAUSE IT IS ITS OWN BILL. The profile row carries post IDENTITY
|
| 5470 |
# and no engagement (measured), so likes/comments are a SECOND vendor call per post.
|
| 5471 |
# A switch that multiplies a run's cost by the post count deserves to be visible on
|
| 5472 |
# the canvas rather than buried in a config panel.
|
| 5473 |
-
node("
|
| 5474 |
-
"Likes and comments per post" if metrics_on
|
| 5475 |
-
|
| 5476 |
-
|
| 5477 |
-
detail=(f"~{max_posts}x the records of a profile-only run"
|
| 5478 |
-
if metrics_on else
|
| 5479 |
"ut_ig_post_snapshots only grows while this is on")),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5480 |
node("write", "write", "Write",
|
| 5481 |
-
cfg.get("targetTable") or "No database",
|
| 5482 |
on=not cfg.get("dryRun"),
|
| 5483 |
detail="+ ut_ig_snapshots Β· ut_ig_posts Β· ut_ig_post_snapshots"),
|
| 5484 |
]
|
| 5485 |
edges = [{"from": "trigger", "to": "source", "label": ""},
|
| 5486 |
-
{"from": "source", "to": "
|
| 5487 |
-
{"from": "
|
| 5488 |
-
{"from": "
|
| 5489 |
-
{"from": "capture_paid", "to": "capture_metrics", "label": ""},
|
| 5490 |
-
{"from": "capture_anon", "to": "capture_metrics", "label": ""},
|
| 5491 |
-
{"from": "capture_metrics", "to": "write", "label": ""}]
|
| 5492 |
-
# When there is no paid rung the anonymous one is the ONLY rung, so it must not carry a
|
| 5493 |
-
# switch that would leave the automation with nothing to run.
|
| 5494 |
-
if not paid:
|
| 5495 |
-
for n in nodes:
|
| 5496 |
-
if n["id"] == "capture_anon":
|
| 5497 |
-
n["toggle"] = ""
|
| 5498 |
elif defn.get("kind") == "discover_instagram":
|
| 5499 |
limit = int(cfg.get("recordsLimit") or 0)
|
| 5500 |
est = discover_estimate(limit)
|
|
@@ -5583,13 +5577,10 @@ def toggle_node(rt, auto_id, node_id):
|
|
| 5583 |
return patch(rt, auto_id, {"trigger": trg})
|
| 5584 |
if which == "schedule":
|
| 5585 |
sched["enabled"] = not sched.get("enabled")
|
| 5586 |
-
elif which == "paid":
|
| 5587 |
-
cfg["tier"] = ("anonymous" if clean_tier(cfg.get("tier")) == "brightdata"
|
| 5588 |
-
else "brightdata")
|
| 5589 |
-
elif which == "fallback":
|
| 5590 |
-
cfg["noFallback"] = not cfg.get("noFallback")
|
| 5591 |
elif which == "postMetrics":
|
| 5592 |
cfg["postMetrics"] = not cfg.get("postMetrics")
|
|
|
|
|
|
|
| 5593 |
elif which == "write":
|
| 5594 |
cfg["dryRun"] = not cfg.get("dryRun")
|
| 5595 |
return patch(rt, auto_id, {"config": cfg, "schedule": sched})
|
|
@@ -6137,10 +6128,10 @@ def _clean_action_config(kind, cfg, depth, seen, count):
|
|
| 6137 |
"limit": min(e_limit, MAX_ENRICH_PER_RUN),
|
| 6138 |
"skipRecent": bool(cfg.get("skipRecent")),
|
| 6139 |
"skipRecentDays": max(1, e_days),
|
| 6140 |
-
#
|
| 6141 |
-
#
|
| 6142 |
-
|
| 6143 |
-
|
| 6144 |
# β THE FLAG THAT MULTIPLIES THE BILL BY THE POST COUNT. Off unless asked for.
|
| 6145 |
"postMetrics": bool(cfg.get("postMetrics")),
|
| 6146 |
# The separate Comments dataset is more granular than a post read and stays off
|
|
@@ -6292,13 +6283,21 @@ DEFAULT_ENRICH_SORT = "first_found"
|
|
| 6292 |
#: an underscore β `COUNT_LABELS` on the client is an allow-list and cannot pick it up by
|
| 6293 |
#: accident, and `_commit_run`'s numeric filter is the second net under it.
|
| 6294 |
RUN_NOTES_KEY = "_notes"
|
| 6295 |
-
#:
|
| 6296 |
-
#:
|
| 6297 |
-
#:
|
| 6298 |
-
#:
|
| 6299 |
-
#:
|
| 6300 |
-
#: the
|
| 6301 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6302 |
#: A queued profile snapshot the vendor never finishes is dropped after this, with a note. An
|
| 6303 |
#: unbounded pending list is the forever-loop this whole change exists to remove, wearing the
|
| 6304 |
#: opposite mask ([[gate-answers-the-wrong-question]]).
|
|
@@ -6375,6 +6374,48 @@ def _gone_key(row, handle):
|
|
| 6375 |
return f"{plat}:{str(handle or '').strip().lstrip('@').lower()}"
|
| 6376 |
|
| 6377 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6378 |
def enrich_selection(rt, table_key, cfg, profile_key, today=None, gone=None):
|
| 6379 |
"""β 2026-08-07 (owner ruling) β WHICH records this enrich step spends on, in order.
|
| 6380 |
|
|
@@ -6512,12 +6553,12 @@ def enrich_selection(rt, table_key, cfg, profile_key, today=None, gone=None):
|
|
| 6512 |
if not raw_handle:
|
| 6513 |
blank_handle += 1
|
| 6514 |
continue
|
| 6515 |
-
|
| 6516 |
-
|
| 6517 |
-
|
| 6518 |
-
|
| 6519 |
-
|
| 6520 |
-
|
| 6521 |
if cooling:
|
| 6522 |
since = _days_since(r.get("enriched_at"), today=today)
|
| 6523 |
if since is not None and since < days:
|
|
@@ -6532,9 +6573,12 @@ def enrich_selection(rt, table_key, cfg, profile_key, today=None, gone=None):
|
|
| 6532 |
notes.append(f"{cooled} skipped as enriched in the last {days} days")
|
| 6533 |
if skipped_gone:
|
| 6534 |
shown = ", ".join(f"@{h}" for h in skipped_gone[:5])
|
|
|
|
|
|
|
|
|
|
| 6535 |
notes.append(f"{len(skipped_gone)} skipped because Instagram has no such account "
|
| 6536 |
f"({shown}{', β¦' if len(skipped_gone) > 5 else ''}) β delete the row or "
|
| 6537 |
-
f"correct the handle;
|
| 6538 |
if blank_handle:
|
| 6539 |
notes.append(f"{blank_handle} skipped with no handle")
|
| 6540 |
if len(picked) < quota and (cooled or blank_handle or skipped_gone or ordered):
|
|
@@ -6770,8 +6814,6 @@ def apply_actions(rt, defn, table_key, row_ids, username="automation", log=print
|
|
| 6770 |
res = pull_profile(handle_raw,
|
| 6771 |
max_posts=cfg.get("maxPosts") or DEFAULT_POSTS_PER_PULL,
|
| 6772 |
log=log,
|
| 6773 |
-
tier=clean_tier(cfg.get("tier")) or "anonymous",
|
| 6774 |
-
fallback=not cfg.get("noFallback"),
|
| 6775 |
post_metrics=bool(cfg.get("postMetrics")),
|
| 6776 |
comment_metrics=bool(cfg.get("commentMetrics")),
|
| 6777 |
pending_metrics=(enrich["pending"]
|
|
@@ -6886,8 +6928,6 @@ def migrate_field_instagram(rt, defn):
|
|
| 6886 |
max_posts = 24
|
| 6887 |
act = {"id": "act_enrich", "kind": "enrich_instagram", "enabled": True, "when": None,
|
| 6888 |
"config": {"profileField": bound,
|
| 6889 |
-
"tier": clean_tier(cfg.get("tier")) or "anonymous",
|
| 6890 |
-
"noFallback": bool(cfg.get("noFallback")),
|
| 6891 |
"postMetrics": bool(cfg.get("postMetrics")),
|
| 6892 |
"commentMetrics": bool(cfg.get("commentMetrics")),
|
| 6893 |
"dryRun": bool(cfg.get("dryRun")),
|
|
@@ -7177,7 +7217,7 @@ def collect_pending_profile_snapshots(rt, defn, username="automation", log=print
|
|
| 7177 |
"""
|
| 7178 |
tasks = _pending_profile_tasks(defn)
|
| 7179 |
if not tasks:
|
| 7180 |
-
return ("ok", "No pending profile reads.", {}, [], {"
|
| 7181 |
step(f"Collecting {len(tasks)} deferred profile read{'' if len(tasks) == 1 else 's'}")
|
| 7182 |
remaining, patches, affected, notes = [], {}, [], []
|
| 7183 |
acc = {"snaps": [], "posts": [], "psnaps": [], "comments": [], "pending": [],
|
|
@@ -7196,7 +7236,8 @@ def collect_pending_profile_snapshots(rt, defn, username="automation", log=print
|
|
| 7196 |
acc["notes"].append(note)
|
| 7197 |
# β `failed` = the vendor finished, collected nothing, and blamed the TARGET. On a
|
| 7198 |
# profile request that means the account could not be reached at all, so the verdict
|
| 7199 |
-
# is remembered and the selection stops re-buying it (
|
|
|
|
| 7200 |
# β `done`-with-zero is NOT remembered: "we found no matches" is a statement about
|
| 7201 |
# the query, and turning it into "this account does not exist" would silently retire
|
| 7202 |
# live handles.
|
|
@@ -7268,7 +7309,7 @@ def collect_pending_profile_snapshots(rt, defn, username="automation", log=print
|
|
| 7268 |
f"; {waiting} still building" if waiting else ""])
|
| 7269 |
state = "ok" if ready and not closed else "partial"
|
| 7270 |
return (state, head + tail, counts, affected,
|
| 7271 |
-
{"
|
| 7272 |
|
| 7273 |
|
| 7274 |
def _write_collected_metric_rows(rt, defn, username, idents, snapshots, comments, log):
|
|
@@ -7318,10 +7359,10 @@ def collect_pending_metric_snapshots(rt, defn, username="automation", log=print,
|
|
| 7318 |
"""Collect deferred paid Post/Reel/Comment snapshots; never launches a new scrape request."""
|
| 7319 |
tasks = _pending_metric_tasks(defn)
|
| 7320 |
if not tasks:
|
| 7321 |
-
return ("ok", "No pending post-engagement snapshots.", {}, [], {"
|
| 7322 |
step(f"Collecting {len(tasks)} post-engagement batch{'' if len(tasks) == 1 else 'es'}")
|
| 7323 |
remaining, idents, snapshots, comments = [], [], [], []
|
| 7324 |
-
ready, waiting, closed,
|
| 7325 |
for task in tasks:
|
| 7326 |
# ββ 2026-08-09 β ASK THE STATUS DOCUMENT, NOT THE ROWS. `building` used to be
|
| 7327 |
# `not rows or β¦`, so a snapshot the vendor had FINISHED with zero records was re-queued
|
|
@@ -7333,7 +7374,7 @@ def collect_pending_metric_snapshots(rt, defn, username="automation", log=print,
|
|
| 7333 |
# β CLOSED, NOT RE-QUEUED. The vendor is finished and there is nothing to collect;
|
| 7334 |
# keeping the entry would be a pending task that can never resolve.
|
| 7335 |
closed += 1
|
| 7336 |
-
|
| 7337 |
continue
|
| 7338 |
payload, note = bd_call(f"{BD_PATH_SNAPSHOT}/{task['snapshotId']}", {"format": "json"})
|
| 7339 |
rows = _bd_rows(payload) if not note else []
|
|
@@ -7357,6 +7398,23 @@ def collect_pending_metric_snapshots(rt, defn, username="automation", log=print,
|
|
| 7357 |
post["influencer_key"] = task["influencer"]
|
| 7358 |
mapped_posts.append(post)
|
| 7359 |
if mapped_posts:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7360 |
_unused, post_rows, metric_rows, embedded = capture_rows(
|
| 7361 |
{"state": "ok", "profile": {"username": task["influencer"]},
|
| 7362 |
"posts": mapped_posts, "comments": [], "via": "brightdata:deferred"}, pulled)
|
|
@@ -7381,8 +7439,8 @@ def collect_pending_metric_snapshots(rt, defn, username="automation", log=print,
|
|
| 7381 |
# other call sites disagree about the shape of a run β and `_commit_run` already drops
|
| 7382 |
# non-numeric count values, so a pop that is ever missed degrades to today's behaviour rather
|
| 7383 |
# than to a crash.
|
| 7384 |
-
if
|
| 7385 |
-
counts[RUN_NOTES_KEY] =
|
| 7386 |
# β THE EMPTY ONES ARE NAMED, not silently dropped. A batch that finished with no records is
|
| 7387 |
# a real outcome the tenant paid for and it must read as an answer, not as a disappearance.
|
| 7388 |
tail = (f"; {closed} finished with nothing to collect" if closed else "")
|
|
@@ -7390,9 +7448,9 @@ def collect_pending_metric_snapshots(rt, defn, username="automation", log=print,
|
|
| 7390 |
return ("partial",
|
| 7391 |
f"{ready} post-engagement batch{'' if ready == 1 else 'es'} collected; "
|
| 7392 |
f"{waiting} still building and will be collected automatically{tail}",
|
| 7393 |
-
counts, [], {"
|
| 7394 |
return ("ok", f"{ready} post-engagement batch{'' if ready == 1 else 'es'} collected{tail}",
|
| 7395 |
-
counts, [], {"
|
| 7396 |
|
| 7397 |
|
| 7398 |
def ai_decide(rt, defn, act, row, row_id="", log=print):
|
|
@@ -7781,8 +7839,25 @@ def _ut():
|
|
| 7781 |
|
| 7782 |
|
| 7783 |
#: The fns by family β how a value is folded, and what a blank means in each.
|
| 7784 |
-
_ROLLUP_NUM_FNS = frozenset({"sum", "average", "min", "max"})
|
| 7785 |
_ROLLUP_BOOL_FNS = frozenset({"and", "or", "xor"})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7786 |
_ROLLUP_TEXT_FNS = frozenset({"concatenate", "arrayjoin", "arraycompact", "arrayunique"})
|
| 7787 |
#: What `arrayjoin` puts between values. Airtable uses ", "; `concatenate` uses nothing.
|
| 7788 |
_ROLLUP_JOIN = ", "
|
|
@@ -7826,6 +7901,46 @@ def _sort_key(value, ftype):
|
|
| 7826 |
return (raw.lower(),)
|
| 7827 |
|
| 7828 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7829 |
def _rollup_fold(fn, values):
|
| 7830 |
"""`values` (raw cells, in the order the window kept them) β the aggregate, as a STRING.
|
| 7831 |
|
|
@@ -7853,8 +7968,24 @@ def _rollup_fold(fn, values):
|
|
| 7853 |
nums = [n for n in (_lane_num(v) for v in values) if n is not None]
|
| 7854 |
if not nums:
|
| 7855 |
return ""
|
| 7856 |
-
|
| 7857 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7858 |
return f"{out:.0f}" if float(out).is_integer() else f"{out:.2f}"
|
| 7859 |
if fn in _ROLLUP_BOOL_FNS:
|
| 7860 |
# A checkbox cell is '1'/'' in this product, so truth is "non-blank and not a zero".
|
|
@@ -7864,7 +7995,11 @@ def _rollup_fold(fn, values):
|
|
| 7864 |
hit = (all(flags) if fn == "and" else any(flags) if fn == "or"
|
| 7865 |
else sum(1 for f in flags if f) % 2 == 1)
|
| 7866 |
return "1" if hit else ""
|
| 7867 |
-
# --- the text family
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7868 |
vals = [str(v).strip() for v in values if str(v or "").strip() != ""]
|
| 7869 |
if fn == "arrayunique":
|
| 7870 |
seen, uniq = set(), []
|
|
@@ -7912,8 +8047,17 @@ def _linked_rows_by_join(linked, on_key):
|
|
| 7912 |
return idx
|
| 7913 |
|
| 7914 |
|
| 7915 |
-
def _rollup_condition_matches(row, condition, field_types):
|
| 7916 |
-
"""Evaluate one Airtable-style linked-record condition against a candidate row.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7917 |
field = str((condition or {}).get("field") or "")
|
| 7918 |
op = str((condition or {}).get("op") or "")
|
| 7919 |
raw = (row or {}).get(field)
|
|
@@ -7922,6 +8066,18 @@ def _rollup_condition_matches(row, condition, field_types):
|
|
| 7922 |
return text == ""
|
| 7923 |
if op == "is_not_empty":
|
| 7924 |
return text != ""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7925 |
wanted = str((condition or {}).get("value") or "").strip()
|
| 7926 |
if op == "contains":
|
| 7927 |
return wanted.casefold() in text.casefold()
|
|
@@ -8021,16 +8177,21 @@ def compute_relation_cells(rt, table_key, tables=None):
|
|
| 8021 |
# β A rollup whose link field does not exist (renamed, deleted) resolves to NOTHING
|
| 8022 |
# and therefore to a blank cell β never to a stale number. A column that keeps
|
| 8023 |
# printing yesterday's answer after its input is gone is the worst of the options.
|
| 8024 |
-
|
| 8025 |
-
|
| 8026 |
-
|
| 8027 |
-
|
| 8028 |
-
|
| 8029 |
-
|
| 8030 |
-
|
| 8031 |
-
|
| 8032 |
-
|
| 8033 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8034 |
sort_by = str(bag.get("sortBy") or "")
|
| 8035 |
if sort_by:
|
| 8036 |
ftype = (linked_types.get(lk_key) or {}).get(sort_by, "text")
|
|
@@ -8060,6 +8221,30 @@ def compute_relation_cells(rt, table_key, tables=None):
|
|
| 8060 |
limit = int(bag.get("limit") or 0)
|
| 8061 |
if limit:
|
| 8062 |
hits = hits[:limit]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8063 |
src = str(bag.get("field") or "")
|
| 8064 |
want = _rollup_fold(str(bag.get("fn") or ""),
|
| 8065 |
[r.get(src) for _i, r in hits] if src else [1] * len(hits))
|
|
@@ -9068,9 +9253,9 @@ def compose_sentence(defn):
|
|
| 9068 |
host = urlparse(str(cfg.get("url") or "")).hostname or "the page"
|
| 9069 |
body = f"read {host} and upsert rows into {cfg.get('targetTable') or 'a new database'}"
|
| 9070 |
elif kind == "field_instagram":
|
| 9071 |
-
|
| 9072 |
-
|
| 9073 |
-
|
| 9074 |
elif kind == "discover_instagram":
|
| 9075 |
body = (f"search Instagram for up to {cfg.get('recordsLimit') or 0} profiles into "
|
| 9076 |
f"{cfg.get('targetTable') or DISCOVER_TABLE}")
|
|
|
|
| 1233 |
|
| 1234 |
β THE NODE-SWITCH FLAGS FALL BACK TO `previous` WHEN THE KEY IS ABSENT, and that is a rail
|
| 1235 |
rather than a nicety. `patch` replaces the whole config, and the canvas's config panels do not
|
| 1236 |
+
edit `postMetrics` / `commentMetrics` / `dryRun` β those are node SWITCHES. So a plain "Save"
|
| 1237 |
+
from a panel that never knew about them would silently turn off the dry run, or turn ON a
|
| 1238 |
+
per-post purchase: a save that quietly changes what the automation costs. Absent β keep;
|
| 1239 |
present β take it, including `false`.
|
| 1240 |
"""
|
| 1241 |
raw = raw if isinstance(raw, dict) else {}
|
|
|
|
| 1292 |
fkey = _s(raw.get("fieldKey"), 80).strip()
|
| 1293 |
if not fkey:
|
| 1294 |
return None, "pick the automation column this run writes into"
|
| 1295 |
+
# ββ `tier` AND `noFallback` ARE ACCEPTED AND IGNORED (wave 28 / R5, contract C2).
|
| 1296 |
+
# They are read from nothing and written to nothing: a stored definition carrying either
|
| 1297 |
+
# still SAVES β it simply loses them on the next write β and neither is ever a reason to
|
| 1298 |
+
# refuse. That asymmetry is D-65's law and it is not squeamishness: refusing an unknown
|
| 1299 |
+
# key would 400 every automation a tenant stored before this wave, forever, on a screen
|
| 1300 |
+
# that gives them no way to remove it. Dropping a retired key is a migration; refusing it
|
| 1301 |
+
# is an outage.
|
| 1302 |
+
# β `clean_tier`/`TIERS`/`bd_ready` KEEP THEIR NAMES. They are VENDOR vocabulary
|
| 1303 |
+
# (`brightdata` is a provider, and `verify_automation` fences the name), not the retired
|
| 1304 |
+
# USER concept β renaming them would be a second, unrelated change wearing this one's
|
| 1305 |
+
# justification.
|
| 1306 |
# C5: absent β keep whatever is stored, like every other switch in this branch β a panel
|
| 1307 |
# that does not edit the post count must not reset it to the default on Save. And an
|
| 1308 |
# INHERITED value is clamped rather than refused; see `clean_max_posts`.
|
|
|
|
| 1313 |
return None, perr
|
| 1314 |
return {"targetTable": table, "fieldKey": fkey,
|
| 1315 |
"urlField": _s(raw.get("urlField"), 80).strip(),
|
|
|
|
|
|
|
| 1316 |
# β THE ONE FLAG THAT MULTIPLIES THE BILL BY THE POST COUNT. Off unless asked
|
| 1317 |
# for: the Profiles row carries post IDENTITY for free but NO engagement
|
| 1318 |
# (measured 2026-08-05), so likes/comments cost one extra vendor record PER POST.
|
|
|
|
| 1517 |
# with only step 1 lands this at the end, which IS step 2.
|
| 1518 |
seeded.insert(1, {
|
| 1519 |
"id": "act_enrich", "kind": "enrich_instagram", "enabled": True, "when": None,
|
| 1520 |
+
"config": {"postMetrics": False, "commentMetrics": False,
|
|
|
|
| 1521 |
"dryRun": False, "maxPosts": DEFAULT_POSTS_PER_PULL,
|
| 1522 |
"fromView": "", "sortField": DEFAULT_ENRICH_SORT, "sortDir": "desc",
|
| 1523 |
"limit": DEFAULT_ENRICH_LIMIT, "skipRecent": True,
|
|
|
|
| 2594 |
_first, _ig_int,
|
| 2595 |
bd_call, bd_filter_rows, bd_filter_start, bd_filter_status, bd_ready,
|
| 2596 |
bd_snapshot_progress, depth_refusal,
|
| 2597 |
+
ig_handle, pull_profile, top_up_views,
|
| 2598 |
)
|
| 2599 |
|
| 2600 |
|
|
|
|
| 4621 |
|
| 4622 |
def run_field_instagram(rt, defn, username="automation", log=print, step=_no_step,
|
| 4623 |
rows=None):
|
| 4624 |
+
"""Automation #2 (R7): for every row of a database that carries a profile URL, pull the public
|
| 4625 |
+
profile through the paid capability chain, write a status string into the automation column,
|
| 4626 |
+
and append a timestamped row to each of the three IG tables.
|
| 4627 |
+
|
| 4628 |
+
β WAVE 28 / R5 β there is no tier and no rung choice here any more. `pull_profile` routes per
|
| 4629 |
+
CAPABILITY and reports `blocked` rather than downgrading to an approximate row, so the two
|
| 4630 |
+
"which rung answered" counters this function used to keep have nothing left to distinguish."""
|
| 4631 |
cfg = defn.get("config") or {}
|
| 4632 |
table_key, fkey = cfg.get("targetTable"), cfg.get("fieldKey")
|
|
|
|
|
|
|
|
|
|
| 4633 |
post_metrics = bool(cfg.get("postMetrics"))
|
| 4634 |
comment_metrics = bool(cfg.get("commentMetrics"))
|
| 4635 |
dry = bool(cfg.get("dryRun"))
|
| 4636 |
+
# β THE STEP KEYS ARE THE CANVAS NODE IDS (contract C3) and the two must move together β a
|
| 4637 |
+
# status written under a node id `graph()` no longer emits is a dot nothing renders, which is
|
| 4638 |
+
# indistinguishable from a step that never ran.
|
| 4639 |
+
steps = {"trigger": "ok", "source": "idle",
|
| 4640 |
+
"capture_posts": "idle" if post_metrics else "skipped",
|
| 4641 |
+
"capture_comments": "idle" if comment_metrics else "skipped",
|
| 4642 |
"write": "idle"}
|
| 4643 |
t = ut_get(rt, table_key)
|
| 4644 |
if not t:
|
|
|
|
| 4676 |
# `MAX_UT_IG_ROWS` makes the same loop hundreds of millions of dict copies, i.e. an automation
|
| 4677 |
# that no longer finishes. **Raising a cap and batching its writer are ONE change.** (W19-C.)
|
| 4678 |
in_snaps, in_posts, in_psnaps, in_comments = [], [], [], []
|
|
|
|
| 4679 |
targets = [(rid, str(r.get(url_field, "") or "").strip())
|
| 4680 |
for rid, r in rows.items() if str(r.get(url_field, "") or "").strip()]
|
| 4681 |
for i, (rid, url) in enumerate(targets):
|
|
|
|
| 4687 |
step(f"Capturing profile {i + 1} of {len(targets)}")
|
| 4688 |
counts["profiles"] += 1
|
| 4689 |
res = pull_profile(url, max_posts=cfg.get("maxPosts") or DEFAULT_POSTS_PER_PULL, log=log,
|
| 4690 |
+
post_metrics=post_metrics, comment_metrics=comment_metrics,
|
|
|
|
| 4691 |
pending_metrics=(pending_metrics if not dry else None))
|
| 4692 |
pulled = _iso()
|
| 4693 |
via = res.get("via") or ""
|
| 4694 |
read_ok = res["state"] in ("ok", "partial")
|
| 4695 |
+
# β `counts["paid"]` SURVIVES R5 AND IT IS NOT THE RETIRED TIER. It counts profiles a PAID
|
| 4696 |
+
# vendor actually answered, which is the run's spend report β the thing the owner reads to
|
| 4697 |
+
# reconcile a bill. What died is the free rung it used to be contrasted with, so the test
|
| 4698 |
+
# is now simply "did a vendor answer" rather than "did the vendor we were told to try".
|
| 4699 |
+
if read_ok and via in ("brightdata", "apify"):
|
| 4700 |
+
counts["paid"] += 1
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4701 |
if read_ok:
|
| 4702 |
counts["ok" if res["state"] == "ok" else "partial"] += 1
|
| 4703 |
prof = res["profile"]
|
|
|
|
| 4798 |
else:
|
| 4799 |
cap_state = "partial" if (counts["blocked"] or counts["error"]
|
| 4800 |
or counts["partial"]) else "ok"
|
| 4801 |
+
# β C3 β THE CAPTURE FORK IS GONE, SO ITS OUTCOME LANDS ON `source`. There is one way to
|
| 4802 |
+
# read a profile now, and the node that names the profile set is the honest owner of "did
|
| 4803 |
+
# reading them work". The old `capture`/`capture_paid`/`capture_anon` trio described a branch
|
| 4804 |
+
# that no longer exists in behaviour OR on screen.
|
| 4805 |
+
steps["source"] = cap_state if targets else steps["source"]
|
| 4806 |
# β MEASURED, LIKE EVERY OTHER DOT: it is `ok` only if an engagement row was actually
|
| 4807 |
# appended. "It was switched on" is not the same fact as "it answered", and painting the
|
| 4808 |
# second from the first is the fabrication `node_status` refuses to make.
|
| 4809 |
# β AND `blocked` IS A CLAIM ABOUT THE VENDOR, so it needs something to have been ASKED. A
|
| 4810 |
# run that found no posts to enrich did not have a rung refuse it β nothing was requested β
|
| 4811 |
# so that reads `skipped`, the same word an off switch earns.
|
| 4812 |
+
steps["capture_posts"] = ("ok" if counts["metrics"]
|
| 4813 |
+
else "partial" if counts.get("metric_batches_pending")
|
| 4814 |
+
else "blocked" if (post_metrics and counts["posts"])
|
| 4815 |
+
else "skipped")
|
| 4816 |
+
steps["capture_comments"] = ("ok" if counts.get("comments")
|
| 4817 |
+
else "blocked" if (comment_metrics and counts["posts"])
|
| 4818 |
+
else "skipped")
|
| 4819 |
|
| 4820 |
summary = (f"{read}/{counts['profiles']} profiles read, {counts['posts']} posts "
|
| 4821 |
f"({counts['new_posts']} new)")
|
|
|
|
| 5349 |
#: deliberate answer rather than an omission: a "Fetch the page" step that can be turned off is
|
| 5350 |
#: not an automation with a disabled step, it is a broken automation with a lie on it. The ones
|
| 5351 |
#: here are all REAL β each changes what the next run does:
|
| 5352 |
+
#: schedule the trigger fires on its cron, or only by hand
|
| 5353 |
+
#: postMetrics likes/comments per post are bought, or the engagement series does not grow β
|
| 5354 |
+
#: it costs a vendor record PER POST rather than per profile (measured: the
|
| 5355 |
+
#: profile row carries post identity and no engagement)
|
| 5356 |
+
#: commentMetrics the Comments dataset is bought, which can bill many rows per post
|
| 5357 |
+
#: write DRY RUN β read everything, compute the counts, write nothing anywhere
|
| 5358 |
+
#:
|
| 5359 |
+
#: ββ WAVE 28 / R5+R6 (contract C3) β `paid` AND `fallback` ARE GONE, and what replaced them is
|
| 5360 |
+
#: the point of the ruling: the money switches used to be "which rung do we try" (a question about
|
| 5361 |
+
#: our plumbing), and they are now "what do you want captured" (a question about the user's data).
|
| 5362 |
+
#: Profile is always captured and has no switch β an automation that fetches nothing is not an
|
| 5363 |
+
#: automation with a step turned off, it is a broken one with a lie on it.
|
| 5364 |
+
#: β `trigger: "schedule"` IS NOT PART OF THAT COLLAPSE. It is the trigger card's own on/off and
|
| 5365 |
+
#: has its own branch in `toggle_node` (event triggers flip themselves, not the cron); C3's "the
|
| 5366 |
+
#: three toggles" names the CAPTURE ladder it is reshaping.
|
| 5367 |
+
NODE_TOGGLES = {"trigger": "schedule", "capture_posts": "postMetrics",
|
| 5368 |
+
"capture_comments": "commentMetrics", "write": "write"}
|
| 5369 |
|
| 5370 |
|
| 5371 |
def _cron_label(cron):
|
|
|
|
| 5449 |
edges = []
|
| 5450 |
|
| 5451 |
if defn.get("kind") == "field_instagram":
|
| 5452 |
+
# ββ WAVE 28 / CONTRACT C3 β FOUR NODES, THREE OF THEM SWITCHES OVER WHAT IS CAPTURED.
|
| 5453 |
+
# This branch used to draw a FORK: `Capture` splitting into `Exact counts` (paid) and
|
| 5454 |
+
# `Estimated counts` (the free anonymous ladder), rejoining at `Post engagement`. R5
|
| 5455 |
+
# deleted the ladder, so the fork had one arm; keeping it would have drawn a decision the
|
| 5456 |
+
# engine no longer makes, with a switch (`fallback`) flipping a config key the cleaners
|
| 5457 |
+
# now discard. A canvas that offers a choice the runtime ignores is worse than no canvas.
|
| 5458 |
+
# β `Profile set` STILL CARRIES NO SWITCH, and now that is the whole ruling rather than an
|
| 5459 |
+
# implementation detail: the profile is always captured (R6), Posts and Comments are the
|
| 5460 |
+
# opt-ins, and both are OFF until asked for.
|
| 5461 |
metrics_on = bool(cfg.get("postMetrics"))
|
| 5462 |
+
comments_on = bool(cfg.get("commentMetrics"))
|
| 5463 |
max_posts = cfg.get("maxPosts") or DEFAULT_POSTS_PER_PULL
|
| 5464 |
nodes += [
|
| 5465 |
node("source", "source", "Profile set", cfg.get("targetTable") or "No database", 1,
|
| 5466 |
panel="source",
|
| 5467 |
detail=f"URL column: {cfg.get('urlField')}" if cfg.get("urlField")
|
| 5468 |
else "URL column: the one the field is bound to"),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5469 |
# β ITS OWN NODE BECAUSE IT IS ITS OWN BILL. The profile row carries post IDENTITY
|
| 5470 |
# and no engagement (measured), so likes/comments are a SECOND vendor call per post.
|
| 5471 |
# A switch that multiplies a run's cost by the post count deserves to be visible on
|
| 5472 |
# the canvas rather than buried in a config panel.
|
| 5473 |
+
node("capture_posts", "capture", "Post data",
|
| 5474 |
+
"Likes and comments per post" if metrics_on else "Off",
|
| 5475 |
+
2, panel="capture", on=metrics_on,
|
| 5476 |
+
detail=(f"Up to {max_posts} posts per profile" if metrics_on else
|
|
|
|
|
|
|
| 5477 |
"ut_ig_post_snapshots only grows while this is on")),
|
| 5478 |
+
node("capture_comments", "capture", "Comment data",
|
| 5479 |
+
"Comments on those posts" if comments_on else "Off",
|
| 5480 |
+
3, panel="capture", on=comments_on,
|
| 5481 |
+
detail=("The full comments dataset β many rows per post" if comments_on else
|
| 5482 |
+
"Comments already embedded in a paid post row are kept either way")),
|
| 5483 |
node("write", "write", "Write",
|
| 5484 |
+
cfg.get("targetTable") or "No database", 4, panel="write",
|
| 5485 |
on=not cfg.get("dryRun"),
|
| 5486 |
detail="+ ut_ig_snapshots Β· ut_ig_posts Β· ut_ig_post_snapshots"),
|
| 5487 |
]
|
| 5488 |
edges = [{"from": "trigger", "to": "source", "label": ""},
|
| 5489 |
+
{"from": "source", "to": "capture_posts", "label": ""},
|
| 5490 |
+
{"from": "capture_posts", "to": "capture_comments", "label": ""},
|
| 5491 |
+
{"from": "capture_comments", "to": "write", "label": ""}]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5492 |
elif defn.get("kind") == "discover_instagram":
|
| 5493 |
limit = int(cfg.get("recordsLimit") or 0)
|
| 5494 |
est = discover_estimate(limit)
|
|
|
|
| 5577 |
return patch(rt, auto_id, {"trigger": trg})
|
| 5578 |
if which == "schedule":
|
| 5579 |
sched["enabled"] = not sched.get("enabled")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5580 |
elif which == "postMetrics":
|
| 5581 |
cfg["postMetrics"] = not cfg.get("postMetrics")
|
| 5582 |
+
elif which == "commentMetrics":
|
| 5583 |
+
cfg["commentMetrics"] = not cfg.get("commentMetrics")
|
| 5584 |
elif which == "write":
|
| 5585 |
cfg["dryRun"] = not cfg.get("dryRun")
|
| 5586 |
return patch(rt, auto_id, {"config": cfg, "schedule": sched})
|
|
|
|
| 6128 |
"limit": min(e_limit, MAX_ENRICH_PER_RUN),
|
| 6129 |
"skipRecent": bool(cfg.get("skipRecent")),
|
| 6130 |
"skipRecentDays": max(1, e_days),
|
| 6131 |
+
# β `tier`/`noFallback` are ACCEPTED AND IGNORED here for the same reason as in
|
| 6132 |
+
# `clean_config` (R5 / C2): a stored action carrying them still saves and simply
|
| 6133 |
+
# loses them, because refusing a retired key 400s every definition written before
|
| 6134 |
+
# this wave (D-65).
|
| 6135 |
# β THE FLAG THAT MULTIPLIES THE BILL BY THE POST COUNT. Off unless asked for.
|
| 6136 |
"postMetrics": bool(cfg.get("postMetrics")),
|
| 6137 |
# The separate Comments dataset is more granular than a post read and stays off
|
|
|
|
| 6283 |
#: an underscore β `COUNT_LABELS` on the client is an allow-list and cannot pick it up by
|
| 6284 |
#: accident, and `_commit_run`'s numeric filter is the second net under it.
|
| 6285 |
RUN_NOTES_KEY = "_notes"
|
| 6286 |
+
#: ββ WAVE 28 / OWNER RULING R9 β `NOT_FOUND_RETRY_DAYS` IS RETIRED. A handle a VENDOR SAID DOES
|
| 6287 |
+
#: NOT EXIST is now a TOMBSTONE: never auto-retried, at any interval.
|
| 6288 |
+
#: β THE 30-DAY BACKOFF WAS THE DEFENSIBLE ANSWER AND IT WAS STILL WRONG, which is the sentence
|
| 6289 |
+
#: worth keeping. Its argument was that a handle can be renamed back or a suspension lifted, so
|
| 6290 |
+
#: the verdict should expire. But nothing about US changes on day 31 β the only new information
|
| 6291 |
+
#: is a guess that the world moved β so the timer buys one paid vendor call per dead handle per
|
| 6292 |
+
#: month, forever, on a row nobody is looking at, and reports it as "1 blocked". A tenant with a
|
| 6293 |
+
#: hundred stale handles pays a hundred times a month to be told the same thing.
|
| 6294 |
+
#: β WHAT RE-ARMS IT IS A HUMAN, and there are two doors:
|
| 6295 |
+
#: 1. the KEY is `(platform, handle)` β so correcting a typo re-arms IMMEDIATELY, with no
|
| 6296 |
+
#: wiring at all, because the corrected handle simply is not the one we recorded; and
|
| 6297 |
+
#: 2. `clear_gone()` β called when the profile CELL is written, so re-typing the SAME handle
|
| 6298 |
+
#: (a person saying "try it again") also re-arms.
|
| 6299 |
+
#: β The run keeps NAMING the skipped handles every time, not only on the run that discovered
|
| 6300 |
+
#: them: a tombstone the owner cannot see is just a disappearance.
|
| 6301 |
#: A queued profile snapshot the vendor never finishes is dropped after this, with a note. An
|
| 6302 |
#: unbounded pending list is the forever-loop this whole change exists to remove, wearing the
|
| 6303 |
#: opposite mask ([[gate-answers-the-wrong-question]]).
|
|
|
|
| 6374 |
return f"{plat}:{str(handle or '').strip().lstrip('@').lower()}"
|
| 6375 |
|
| 6376 |
|
| 6377 |
+
def clear_gone(rt, table_key, handle, row=None):
|
| 6378 |
+
"""R9's re-arm door: forget the "this account does not exist" verdict for ONE handle.
|
| 6379 |
+
|
| 6380 |
+
Returns the number of automations whose memory was changed β 0 is the ordinary answer and is
|
| 6381 |
+
not an error, because most cell edits are not on a dead handle.
|
| 6382 |
+
|
| 6383 |
+
ββ WHY THIS IS A PUBLIC FUNCTION IN THE ENGINE RATHER THAN A LOOKUP IN THE ROW WRITER.
|
| 6384 |
+
The verdict lives on the AUTOMATION (`state.enrichNotFound`), not on the row β it has to,
|
| 6385 |
+
because it is a fact about what a run learned and paid for. But the thing that re-arms it is a
|
| 6386 |
+
ROW event, and the row writer must not need to know the shape of an automation's state to
|
| 6387 |
+
trigger it. So the seam is one call with the three things the writer already has, and every
|
| 6388 |
+
walk of `all_definitions` stays on this side of the fence.
|
| 6389 |
+
β IT MATCHES THE SAME WAY THE SKIP DOES. `_gone_key` is the one implementation of "which
|
| 6390 |
+
handle is this", so a verdict can never be recorded under a key this cannot find
|
| 6391 |
+
([[one-evaluator-per-question]]).
|
| 6392 |
+
|
| 6393 |
+
β WITHOUT ITS CALLER THIS IS INERT, AND THAT IS RECORDED RATHER THAN ASSUMED. R9's first
|
| 6394 |
+
door β correcting a typo β works with no wiring at all, because the key IS the handle and a
|
| 6395 |
+
different handle is simply not the one we recorded. This second door only opens when the row
|
| 6396 |
+
write path calls it, which lives in `routes_tables.patch_row` (another session's fence).
|
| 6397 |
+
Until that line lands, re-typing the SAME dead handle stays skipped
|
| 6398 |
+
([[flag-shipped-without-its-writer]] β named on purpose, so it is not discovered later).
|
| 6399 |
+
"""
|
| 6400 |
+
key = _gone_key(row or {}, handle)
|
| 6401 |
+
if not str(handle or "").strip():
|
| 6402 |
+
return 0
|
| 6403 |
+
changed = 0
|
| 6404 |
+
for auto_id, defn in (all_definitions(rt) or {}).items():
|
| 6405 |
+
if str((defn.get("config") or {}).get("targetTable") or "") != str(table_key):
|
| 6406 |
+
continue
|
| 6407 |
+
known = dict(((defn.get("state") or {}).get("enrichNotFound")) or {})
|
| 6408 |
+
if key not in known:
|
| 6409 |
+
continue
|
| 6410 |
+
known.pop(key, None)
|
| 6411 |
+
# β `or None` β an empty dict must clear the key rather than store `{}`, which is what
|
| 6412 |
+
# every other state writer here does and what keeps a definition from growing a graveyard
|
| 6413 |
+
# of empty maps.
|
| 6414 |
+
set_state(rt, str(auto_id), {"enrichNotFound": known or None})
|
| 6415 |
+
changed += 1
|
| 6416 |
+
return changed
|
| 6417 |
+
|
| 6418 |
+
|
| 6419 |
def enrich_selection(rt, table_key, cfg, profile_key, today=None, gone=None):
|
| 6420 |
"""β 2026-08-07 (owner ruling) β WHICH records this enrich step spends on, in order.
|
| 6421 |
|
|
|
|
| 6553 |
if not raw_handle:
|
| 6554 |
blank_handle += 1
|
| 6555 |
continue
|
| 6556 |
+
# β R9 β NO EXPIRY. There is deliberately no date arithmetic here any more: a verdict is
|
| 6557 |
+
# a verdict until a human edits the cell. An `at` stamp is still STORED (it is what the
|
| 6558 |
+
# owner reads to know when we last paid to be told this), it is simply not a clock.
|
| 6559 |
+
if isinstance((gone or {}).get(_gone_key(r, raw_handle)), dict):
|
| 6560 |
+
skipped_gone.append(raw_handle)
|
| 6561 |
+
continue
|
| 6562 |
if cooling:
|
| 6563 |
since = _days_since(r.get("enriched_at"), today=today)
|
| 6564 |
if since is not None and since < days:
|
|
|
|
| 6573 |
notes.append(f"{cooled} skipped as enriched in the last {days} days")
|
| 6574 |
if skipped_gone:
|
| 6575 |
shown = ", ".join(f"@{h}" for h in skipped_gone[:5])
|
| 6576 |
+
# β THE SENTENCE IS THE FEATURE. It must name the handles AND the two things a person can
|
| 6577 |
+
# do, because nothing else will ever retry them β under R9 this note is the only path
|
| 6578 |
+
# back from a tombstone, so a vaguer version would strand the row permanently.
|
| 6579 |
notes.append(f"{len(skipped_gone)} skipped because Instagram has no such account "
|
| 6580 |
f"({shown}{', β¦' if len(skipped_gone) > 5 else ''}) β delete the row or "
|
| 6581 |
+
f"correct the handle; they are not retried automatically")
|
| 6582 |
if blank_handle:
|
| 6583 |
notes.append(f"{blank_handle} skipped with no handle")
|
| 6584 |
if len(picked) < quota and (cooled or blank_handle or skipped_gone or ordered):
|
|
|
|
| 6814 |
res = pull_profile(handle_raw,
|
| 6815 |
max_posts=cfg.get("maxPosts") or DEFAULT_POSTS_PER_PULL,
|
| 6816 |
log=log,
|
|
|
|
|
|
|
| 6817 |
post_metrics=bool(cfg.get("postMetrics")),
|
| 6818 |
comment_metrics=bool(cfg.get("commentMetrics")),
|
| 6819 |
pending_metrics=(enrich["pending"]
|
|
|
|
| 6928 |
max_posts = 24
|
| 6929 |
act = {"id": "act_enrich", "kind": "enrich_instagram", "enabled": True, "when": None,
|
| 6930 |
"config": {"profileField": bound,
|
|
|
|
|
|
|
| 6931 |
"postMetrics": bool(cfg.get("postMetrics")),
|
| 6932 |
"commentMetrics": bool(cfg.get("commentMetrics")),
|
| 6933 |
"dryRun": bool(cfg.get("dryRun")),
|
|
|
|
| 7217 |
"""
|
| 7218 |
tasks = _pending_profile_tasks(defn)
|
| 7219 |
if not tasks:
|
| 7220 |
+
return ("ok", "No pending profile reads.", {}, [], {"source": "idle"})
|
| 7221 |
step(f"Collecting {len(tasks)} deferred profile read{'' if len(tasks) == 1 else 's'}")
|
| 7222 |
remaining, patches, affected, notes = [], {}, [], []
|
| 7223 |
acc = {"snaps": [], "posts": [], "psnaps": [], "comments": [], "pending": [],
|
|
|
|
| 7236 |
acc["notes"].append(note)
|
| 7237 |
# β `failed` = the vendor finished, collected nothing, and blamed the TARGET. On a
|
| 7238 |
# profile request that means the account could not be reached at all, so the verdict
|
| 7239 |
+
# is remembered and the selection stops re-buying it (R9: permanently, until a
|
| 7240 |
+
# human edits the handle cell β see `clear_gone`).
|
| 7241 |
# β `done`-with-zero is NOT remembered: "we found no matches" is a statement about
|
| 7242 |
# the query, and turning it into "this account does not exist" would silently retire
|
| 7243 |
# live handles.
|
|
|
|
| 7309 |
f"; {waiting} still building" if waiting else ""])
|
| 7310 |
state = "ok" if ready and not closed else "partial"
|
| 7311 |
return (state, head + tail, counts, affected,
|
| 7312 |
+
{"source": "ok" if ready else "partial", "write": "ok" if ready else "idle"})
|
| 7313 |
|
| 7314 |
|
| 7315 |
def _write_collected_metric_rows(rt, defn, username, idents, snapshots, comments, log):
|
|
|
|
| 7359 |
"""Collect deferred paid Post/Reel/Comment snapshots; never launches a new scrape request."""
|
| 7360 |
tasks = _pending_metric_tasks(defn)
|
| 7361 |
if not tasks:
|
| 7362 |
+
return ("ok", "No pending post-engagement snapshots.", {}, [], {"capture_posts": "idle"})
|
| 7363 |
step(f"Collecting {len(tasks)} post-engagement batch{'' if len(tasks) == 1 else 'es'}")
|
| 7364 |
remaining, idents, snapshots, comments = [], [], [], []
|
| 7365 |
+
ready, waiting, closed, run_notes = 0, 0, 0, []
|
| 7366 |
for task in tasks:
|
| 7367 |
# ββ 2026-08-09 β ASK THE STATUS DOCUMENT, NOT THE ROWS. `building` used to be
|
| 7368 |
# `not rows or β¦`, so a snapshot the vendor had FINISHED with zero records was re-queued
|
|
|
|
| 7374 |
# β CLOSED, NOT RE-QUEUED. The vendor is finished and there is nothing to collect;
|
| 7375 |
# keeping the entry would be a pending task that can never resolve.
|
| 7376 |
closed += 1
|
| 7377 |
+
run_notes.append(f"{task['influencer']}: {_s(empty_note, 160)}")
|
| 7378 |
continue
|
| 7379 |
payload, note = bd_call(f"{BD_PATH_SNAPSHOT}/{task['snapshotId']}", {"format": "json"})
|
| 7380 |
rows = _bd_rows(payload) if not note else []
|
|
|
|
| 7398 |
post["influencer_key"] = task["influencer"]
|
| 7399 |
mapped_posts.append(post)
|
| 7400 |
if mapped_posts:
|
| 7401 |
+
# ββ 2026-08-09 β THE VIEWS TOP-UP RUNS HERE TOO, and its absence is why the
|
| 7402 |
+
# owner's `theresalearns` run filled every column except Views.
|
| 7403 |
+
#
|
| 7404 |
+
# β Bright Data is DECLARED INCAPABLE of `ig_post_views` (`providers.py`), so a
|
| 7405 |
+
# Posts row physically cannot carry a view count β MEASURED on the stored
|
| 7406 |
+
# payloads: 12 rows, `content_type: "Reel"`, and no view/play key in any of them.
|
| 7407 |
+
# Views only ever comes from the Apify capability. That top-up lived INSIDE
|
| 7408 |
+
# `pull_profile_bd`, so it ran only when the Posts scrape answered within the wait
|
| 7409 |
+
# budget; when the batch deferred β which is routine, and what happened here β the
|
| 7410 |
+
# rows came back through THIS function and Apify was never asked.
|
| 7411 |
+
# β `top_up_views` is now one function with two callers rather than a copy, so
|
| 7412 |
+
# the inline and deferred paths cannot answer this differently again.
|
| 7413 |
+
# β It mutates `mapped_posts` in place and must run BEFORE `capture_rows`, which
|
| 7414 |
+
# is what freezes the values into the post + snapshot rows.
|
| 7415 |
+
v_note = top_up_views(mapped_posts, log=log)
|
| 7416 |
+
if v_note:
|
| 7417 |
+
run_notes.append(f"@{task['influencer']}: {v_note}")
|
| 7418 |
_unused, post_rows, metric_rows, embedded = capture_rows(
|
| 7419 |
{"state": "ok", "profile": {"username": task["influencer"]},
|
| 7420 |
"posts": mapped_posts, "comments": [], "via": "brightdata:deferred"}, pulled)
|
|
|
|
| 7439 |
# other call sites disagree about the shape of a run β and `_commit_run` already drops
|
| 7440 |
# non-numeric count values, so a pop that is ever missed degrades to today's behaviour rather
|
| 7441 |
# than to a crash.
|
| 7442 |
+
if run_notes:
|
| 7443 |
+
counts[RUN_NOTES_KEY] = run_notes
|
| 7444 |
# β THE EMPTY ONES ARE NAMED, not silently dropped. A batch that finished with no records is
|
| 7445 |
# a real outcome the tenant paid for and it must read as an answer, not as a disappearance.
|
| 7446 |
tail = (f"; {closed} finished with nothing to collect" if closed else "")
|
|
|
|
| 7448 |
return ("partial",
|
| 7449 |
f"{ready} post-engagement batch{'' if ready == 1 else 'es'} collected; "
|
| 7450 |
f"{waiting} still building and will be collected automatically{tail}",
|
| 7451 |
+
counts, [], {"capture_posts": "partial", "write": "ok"})
|
| 7452 |
return ("ok", f"{ready} post-engagement batch{'' if ready == 1 else 'es'} collected{tail}",
|
| 7453 |
+
counts, [], {"capture_posts": "ok", "write": "ok"})
|
| 7454 |
|
| 7455 |
|
| 7456 |
def ai_decide(rt, defn, act, row, row_id="", log=print):
|
|
|
|
| 7839 |
|
| 7840 |
|
| 7841 |
#: The fns by family β how a value is folded, and what a blank means in each.
|
| 7842 |
+
_ROLLUP_NUM_FNS = frozenset({"sum", "average", "stdev", "min", "max"})
|
| 7843 |
_ROLLUP_BOOL_FNS = frozenset({"and", "or", "xor"})
|
| 7844 |
+
#: The count/order family β folded by their own arms above the lanes.
|
| 7845 |
+
_ROLLUP_SEQ_FNS = frozenset({"countall", "counta", "count", "latest"})
|
| 7846 |
+
#: The text family. β IT IS A NAMED SET NOW AND IT USED TO BE THE FALL-THROUGH, which is how
|
| 7847 |
+
#: wave 28 nearly shipped a wrong number that looked like data: `stdev` validated and STORED
|
| 7848 |
+
#: (`core.user_tables.ROLLUP_FNS`) a commit before this function learned it, and an fn no arm
|
| 7849 |
+
#: claims fell past both lanes into the join below β so a "Std deviation" column rendered
|
| 7850 |
+
#: `"137684, 19561, 8123"`. Filled, plausible, and not a statistic. An unrecognised fn now
|
| 7851 |
+
#: returns `""` ([[gate-answers-the-wrong-question]]: blank is the honest answer to a question
|
| 7852 |
+
#: nothing can answer; a comma-joined list is a different question's answer wearing this label).
|
| 7853 |
+
_ROLLUP_TEXT_FNS = frozenset({"concatenate", "arrayjoin", "arraycompact", "arrayunique"})
|
| 7854 |
+
#: ββ WHAT THIS FOLD ACTUALLY IMPLEMENTS, DERIVED FROM THE ARMS RATHER THAN RESTATED.
|
| 7855 |
+
#: `verify_automation` asserts this is IDENTICAL to `core.user_tables.ROLLUP_FNS` name for name
|
| 7856 |
+
#: (contract C1's parity leg), so the validator can never again accept a function the fold cannot
|
| 7857 |
+
#: compute β in EITHER direction. A hand-listed copy in the gate would have gone green on the
|
| 7858 |
+
#: defect above, because the defect was that the two lists already disagreed.
|
| 7859 |
+
ROLLUP_FOLD_FNS = frozenset(_ROLLUP_NUM_FNS | _ROLLUP_BOOL_FNS | _ROLLUP_SEQ_FNS
|
| 7860 |
+
| _ROLLUP_TEXT_FNS)
|
| 7861 |
_ROLLUP_TEXT_FNS = frozenset({"concatenate", "arrayjoin", "arraycompact", "arrayunique"})
|
| 7862 |
#: What `arrayjoin` puts between values. Airtable uses ", "; `concatenate` uses nothing.
|
| 7863 |
_ROLLUP_JOIN = ", "
|
|
|
|
| 7901 |
return (raw.lower(),)
|
| 7902 |
|
| 7903 |
|
| 7904 |
+
def _sample_stdev(nums):
|
| 7905 |
+
"""SAMPLE standard deviation (n-1) of `nums`, or None under two values.
|
| 7906 |
+
|
| 7907 |
+
β ONE IMPLEMENTATION, TWO READERS, and that is the point of lifting four lines into a
|
| 7908 |
+
function: `_rollup_fold` RENDERS this number into a "Std deviation" column and
|
| 7909 |
+
`_rollup_ref_threshold` COMPARES rows against it inside a `sigmas` condition. A second copy
|
| 7910 |
+
would let a column and the filter beside it disagree about the same word on the same set β
|
| 7911 |
+
the exact drift `top_up_views` was extracted to prevent one module over
|
| 7912 |
+
([[one-evaluator-per-question]]).
|
| 7913 |
+
β None means UNANSWERABLE and no caller may read it as 0.
|
| 7914 |
+
"""
|
| 7915 |
+
if len(nums) < 2:
|
| 7916 |
+
return None
|
| 7917 |
+
mean = sum(nums) / len(nums)
|
| 7918 |
+
return (sum((n - mean) ** 2 for n in nums) / (len(nums) - 1)) ** 0.5
|
| 7919 |
+
|
| 7920 |
+
|
| 7921 |
+
def _rollup_ref_threshold(rows, field, sigmas):
|
| 7922 |
+
"""`mean + sigmas*stdev` of `field` over `rows` β a statistic OF THE SCOPED SET (contract C1).
|
| 7923 |
+
|
| 7924 |
+
β `rows` MUST already be the scoped window: ranked by `sortBy`, deduped, and cut by `limit`,
|
| 7925 |
+
and NOT yet filtered by the conditions this threshold feeds. That order is the contract
|
| 7926 |
+
(`core.user_tables.ROLLUP_REF_OPS`'s note says so in as many words) and both other orders
|
| 7927 |
+
produce a plausible number: computing it before `limit` answers "2 sigma of everything this
|
| 7928 |
+
account ever posted" under a column that says "of the last 10", and computing it after the
|
| 7929 |
+
filter makes the threshold depend on the rows it is choosing β a definition that chases
|
| 7930 |
+
itself.
|
| 7931 |
+
|
| 7932 |
+
Returns None when the window cannot answer β fewer than two numeric values in `field`.
|
| 7933 |
+
β THE CALLER MUST DROP THE ROW, NOT KEEP IT. "Beyond 2 sigma of one post" is not a question
|
| 7934 |
+
with a permissive answer; letting an unanswerable leaf pass everything would silently turn
|
| 7935 |
+
"the outliers" into "all of them", which is this module's worst failure mode wearing a filter.
|
| 7936 |
+
"""
|
| 7937 |
+
nums = [n for n in (_lane_num((r or {}).get(field)) for r in rows) if n is not None]
|
| 7938 |
+
sd = _sample_stdev(nums)
|
| 7939 |
+
if sd is None:
|
| 7940 |
+
return None
|
| 7941 |
+
return (sum(nums) / len(nums)) + float(sigmas) * sd
|
| 7942 |
+
|
| 7943 |
+
|
| 7944 |
def _rollup_fold(fn, values):
|
| 7945 |
"""`values` (raw cells, in the order the window kept them) β the aggregate, as a STRING.
|
| 7946 |
|
|
|
|
| 7968 |
nums = [n for n in (_lane_num(v) for v in values) if n is not None]
|
| 7969 |
if not nums:
|
| 7970 |
return ""
|
| 7971 |
+
if fn == "stdev":
|
| 7972 |
+
# ββ SAMPLE standard deviation (n-1), and the divisor is a ruling, not a preference
|
| 7973 |
+
# (C1 / `core.user_tables.ROLLUP_FNS`'s note): a rollup folds the rows that happen to
|
| 7974 |
+
# be LINKED, which is a sample of an account's posting history and not its entirety.
|
| 7975 |
+
# β Fixture to check a refactor against: [2,4,4,4,5,5,7,9] -> 2.14. The POPULATION
|
| 7976 |
+
# form gives 2.00 on the same input, so a test that ever reads 2.00 has silently
|
| 7977 |
+
# switched divisors.
|
| 7978 |
+
# β FEWER THAN TWO VALUES IS "" AND NEVER "0". n-1 = 0 would divide by zero, but the
|
| 7979 |
+
# honest reason is upstream of the arithmetic: one measurement has no spread to
|
| 7980 |
+
# report, and a 0 in a "Std deviation" column reads as PERFECT CONSISTENCY β the
|
| 7981 |
+
# single most confident thing this column can say, asserted from a single row. Same
|
| 7982 |
+
# law as the blank `sum`, and it bites harder here.
|
| 7983 |
+
out = _sample_stdev(nums)
|
| 7984 |
+
if out is None:
|
| 7985 |
+
return ""
|
| 7986 |
+
else:
|
| 7987 |
+
out = (sum(nums) if fn == "sum" else min(nums) if fn == "min"
|
| 7988 |
+
else max(nums) if fn == "max" else sum(nums) / len(nums))
|
| 7989 |
return f"{out:.0f}" if float(out).is_integer() else f"{out:.2f}"
|
| 7990 |
if fn in _ROLLUP_BOOL_FNS:
|
| 7991 |
# A checkbox cell is '1'/'' in this product, so truth is "non-blank and not a zero".
|
|
|
|
| 7995 |
hit = (all(flags) if fn == "and" else any(flags) if fn == "or"
|
| 7996 |
else sum(1 for f in flags if f) % 2 == 1)
|
| 7997 |
return "1" if hit else ""
|
| 7998 |
+
# --- the text family. β CLAIMED BY NAME, NEVER BY FALL-THROUGH β see `_ROLLUP_TEXT_FNS`.
|
| 7999 |
+
# An fn no arm above recognises returns "" rather than a comma-joined dump of every value,
|
| 8000 |
+
# which is the shape a not-yet-implemented aggregate wore for one commit of wave 28.
|
| 8001 |
+
if fn not in _ROLLUP_TEXT_FNS:
|
| 8002 |
+
return ""
|
| 8003 |
vals = [str(v).strip() for v in values if str(v or "").strip() != ""]
|
| 8004 |
if fn == "arrayunique":
|
| 8005 |
seen, uniq = set(), []
|
|
|
|
| 8047 |
return idx
|
| 8048 |
|
| 8049 |
|
| 8050 |
+
def _rollup_condition_matches(row, condition, field_types, ref_value=None):
|
| 8051 |
+
"""Evaluate one Airtable-style linked-record condition against a candidate row.
|
| 8052 |
+
|
| 8053 |
+
`ref_value` is the threshold a `ref: {sigmas}` leaf compares against, already computed by the
|
| 8054 |
+
caller over the SCOPED set (`_rollup_ref_threshold`). β It is passed IN rather than computed
|
| 8055 |
+
here because this function sees one row and the statistic is a property of the whole window β
|
| 8056 |
+
a version that reached for the set from inside would be recomputing the same mean once per
|
| 8057 |
+
row, and would have to be handed the window anyway.
|
| 8058 |
+
β `None` means the window could not answer, and the leaf then matches NOTHING. See the
|
| 8059 |
+
threshold helper for why the permissive reading is the dangerous one.
|
| 8060 |
+
"""
|
| 8061 |
field = str((condition or {}).get("field") or "")
|
| 8062 |
op = str((condition or {}).get("op") or "")
|
| 8063 |
raw = (row or {}).get(field)
|
|
|
|
| 8066 |
return text == ""
|
| 8067 |
if op == "is_not_empty":
|
| 8068 |
return text != ""
|
| 8069 |
+
if (condition or {}).get("ref") is not None:
|
| 8070 |
+
# β NUMERIC LANE ONLY, BOTH SIDES. The validator already restricts `ref` to the ordering
|
| 8071 |
+
# ops, and a row whose cell is blank or unparseable has no position relative to a computed
|
| 8072 |
+
# threshold β it is not "below" it. Dropping it is the same partition law `_sort_key`
|
| 8073 |
+
# follows: unrankable is not a rank ([[sentinel-in-a-sort-key]]).
|
| 8074 |
+
left_num = _lane_num(text)
|
| 8075 |
+
if ref_value is None or left_num is None:
|
| 8076 |
+
return False
|
| 8077 |
+
return ((op == "gt" and left_num > ref_value)
|
| 8078 |
+
or (op == "gte" and left_num >= ref_value)
|
| 8079 |
+
or (op == "lt" and left_num < ref_value)
|
| 8080 |
+
or (op == "lte" and left_num <= ref_value))
|
| 8081 |
wanted = str((condition or {}).get("value") or "").strip()
|
| 8082 |
if op == "contains":
|
| 8083 |
return wanted.casefold() in text.casefold()
|
|
|
|
| 8177 |
# β A rollup whose link field does not exist (renamed, deleted) resolves to NOTHING
|
| 8178 |
# and therefore to a blank cell β never to a stale number. A column that keeps
|
| 8179 |
# printing yesterday's answer after its input is gone is the worst of the options.
|
| 8180 |
+
# ββ WAVE 28 / CONTRACT C1 β SCOPE FIRST, FILTER SECOND, AND THE TWO USED TO BE THE
|
| 8181 |
+
# OTHER WAY ROUND. The conditions block stood HERE, above the ranking, so
|
| 8182 |
+
# "last 10 posts where views > X" meant *the 10 most recent of the posts over X*
|
| 8183 |
+
# rather than *the ones over X among the last 10* β two different windows wearing one
|
| 8184 |
+
# sentence. Harmless while a threshold was a literal; incoherent the moment a
|
| 8185 |
+
# threshold is a statistic OF the window, because the set being described and the set
|
| 8186 |
+
# doing the describing would be different sets.
|
| 8187 |
+
# β THIS ORDER IS THE CONTRACT, not an implementation choice: `core.user_tables`'s
|
| 8188 |
+
# `ROLLUP_REF_OPS` note states it ("the scope picks the window, THEN the threshold is
|
| 8189 |
+
# computed over that window, THEN the conditions filter it") and the validator half
|
| 8190 |
+
# was written against it.
|
| 8191 |
+
# β IT IS A BEHAVIOUR CHANGE FOR EXACTLY ONE SHAPE: a stored rollup carrying BOTH
|
| 8192 |
+
# `conditions` AND `limit`. No shipped preset does (measured across `odoo_relational`
|
| 8193 |
+
# and the IG presets β the one preset with conditions, `_OPEN_ONLY`, is a `countall`
|
| 8194 |
+
# with no limit), so the blast radius is user-built rollups only.
|
| 8195 |
sort_by = str(bag.get("sortBy") or "")
|
| 8196 |
if sort_by:
|
| 8197 |
ftype = (linked_types.get(lk_key) or {}).get(sort_by, "text")
|
|
|
|
| 8221 |
limit = int(bag.get("limit") or 0)
|
| 8222 |
if limit:
|
| 8223 |
hits = hits[:limit]
|
| 8224 |
+
# --- the window is now FIXED, so a set-statistic threshold has a set to be about.
|
| 8225 |
+
conditions = list(bag.get("conditions") or [])
|
| 8226 |
+
if conditions:
|
| 8227 |
+
# β RESOLVED ONCE PER LEAF, NOT ONCE PER ROW. The threshold is a property of the
|
| 8228 |
+
# window; computing it inside `matches` would recompute the same mean for every
|
| 8229 |
+
# candidate and β worse β would invite computing it over a set that the filter is
|
| 8230 |
+
# already shrinking underneath it.
|
| 8231 |
+
# β `.get("sigmas", 0.0)`, never `... or 0.0` β a legitimate `sigmas: 0` ("beyond
|
| 8232 |
+
# the mean") is falsy, and the `or` spelling would silently rewrite it to the same
|
| 8233 |
+
# number by accident. It reads identically and is right for the wrong reason,
|
| 8234 |
+
# which is how it survives a review.
|
| 8235 |
+
refs = [_rollup_ref_threshold(
|
| 8236 |
+
[r for _i, r in hits], str(c.get("field") or ""),
|
| 8237 |
+
(c.get("ref") or {}).get("sigmas", 0.0))
|
| 8238 |
+
if isinstance(c, dict) and c.get("ref") is not None else None
|
| 8239 |
+
for c in conditions]
|
| 8240 |
+
matches = lambda pair: [
|
| 8241 |
+
_rollup_condition_matches(pair[1], condition,
|
| 8242 |
+
linked_types.get(lk_key) or {}, ref_value=ref)
|
| 8243 |
+
for condition, ref in zip(conditions, refs)]
|
| 8244 |
+
if str(bag.get("conditionConj") or "and") == "or":
|
| 8245 |
+
hits = [pair for pair in hits if any(matches(pair))]
|
| 8246 |
+
else:
|
| 8247 |
+
hits = [pair for pair in hits if all(matches(pair))]
|
| 8248 |
src = str(bag.get("field") or "")
|
| 8249 |
want = _rollup_fold(str(bag.get("fn") or ""),
|
| 8250 |
[r.get(src) for _i, r in hits] if src else [1] * len(hits))
|
|
|
|
| 9253 |
host = urlparse(str(cfg.get("url") or "")).hostname or "the page"
|
| 9254 |
body = f"read {host} and upsert rows into {cfg.get('targetTable') or 'a new database'}"
|
| 9255 |
elif kind == "field_instagram":
|
| 9256 |
+
# β NO RUNG CLAUSE (R5). There is one way to capture a profile now, so ", exact counts
|
| 9257 |
+
# first" / ", anonymous only" described a choice that no longer exists.
|
| 9258 |
+
body = f"capture Instagram profiles for {cfg.get('targetTable') or 'the database'}"
|
| 9259 |
elif kind == "discover_instagram":
|
| 9260 |
body = (f"search Instagram for up to {cfg.get('recordsLimit') or 0} profiles into "
|
| 9261 |
f"{cfg.get('targetTable') or DISCOVER_TABLE}")
|
api/connectors_ig.py
CHANGED
|
@@ -34,8 +34,8 @@ a caller reached first. The server imports the engine first and would have worke
|
|
| 34 |
probe doing `import connectors_ig` would have got a half-built engine and an AttributeError from
|
| 35 |
inside a vendor call. **An intermittent-by-import-order failure is the worst shape available**, so
|
| 36 |
every reach-back is a `from automation_engine import β¦` INSIDE the function that needs it, where
|
| 37 |
-
the module is always fully built. There are exactly
|
| 38 |
-
`apify_posts`, `
|
| 39 |
marked `# lazy β see the module header`, and they reach for four kinds of thing:
|
| 40 |
|
| 41 |
fetch / fetch_json / Refused the SSRF-guarded HTTP rail (the scrape runner shares it)
|
|
@@ -54,7 +54,6 @@ allowed to know.
|
|
| 54 |
"""
|
| 55 |
from __future__ import annotations
|
| 56 |
|
| 57 |
-
import datetime as _dt
|
| 58 |
import hashlib
|
| 59 |
import json
|
| 60 |
import os
|
|
@@ -68,11 +67,16 @@ import providers
|
|
| 68 |
|
| 69 |
|
| 70 |
# ---------------------------------------------------------------------------------------------
|
| 71 |
-
#
|
| 72 |
# ---------------------------------------------------------------------------------------------
|
| 73 |
-
|
| 74 |
-
|
| 75 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 76 |
|
| 77 |
def ig_handle(url):
|
| 78 |
"""The handle out of a profile URL (or a bare handle). '' when it is not one."""
|
|
@@ -90,25 +94,6 @@ def ig_handle(url):
|
|
| 90 |
return re.sub(r"[^A-Za-z0-9._]", "", first)[:40]
|
| 91 |
|
| 92 |
|
| 93 |
-
def _ig_get(url, headers=None, timeout=20.0):
|
| 94 |
-
"""One anonymous GET with 429 backoff. Returns `(status, body_bytes, note)`."""
|
| 95 |
-
from automation_engine import PACE_SECONDS, Refused, fetch # lazy β see the module header
|
| 96 |
-
delay = PACE_SECONDS
|
| 97 |
-
for attempt in range(IG_MAX_RETRIES + 1):
|
| 98 |
-
try:
|
| 99 |
-
status, _final, body = fetch(url, timeout=timeout, headers=headers, max_kb=4096)
|
| 100 |
-
except Refused:
|
| 101 |
-
raise
|
| 102 |
-
except Exception as e: # noqa: BLE001
|
| 103 |
-
return 0, b"", f"{type(e).__name__}: {str(e)[:120]}"
|
| 104 |
-
if status == 429 and attempt < IG_MAX_RETRIES:
|
| 105 |
-
time.sleep(delay)
|
| 106 |
-
delay *= 2 # exponential, never a tight retry
|
| 107 |
-
continue
|
| 108 |
-
return status, body, ""
|
| 109 |
-
return 429, b"", "rate limited after retries"
|
| 110 |
-
|
| 111 |
-
|
| 112 |
def _ig_int(v):
|
| 113 |
try:
|
| 114 |
return int(v)
|
|
@@ -131,104 +116,6 @@ def _ig_zero_is_blank(v):
|
|
| 131 |
return None if n == 0 else n
|
| 132 |
|
| 133 |
|
| 134 |
-
def _profile_from_web_api(node, handle):
|
| 135 |
-
return {
|
| 136 |
-
"username": node.get("username") or handle,
|
| 137 |
-
"full_name": node.get("full_name") or "",
|
| 138 |
-
"bio": node.get("biography") or "",
|
| 139 |
-
"followers": _ig_int((node.get("edge_followed_by") or {}).get("count")),
|
| 140 |
-
"following": _ig_int((node.get("edge_follow") or {}).get("count")),
|
| 141 |
-
"posts_count": _ig_int((node.get("edge_owner_to_timeline_media") or {}).get("count")),
|
| 142 |
-
"verified": "1" if node.get("is_verified") else "",
|
| 143 |
-
}
|
| 144 |
-
|
| 145 |
-
|
| 146 |
-
def _posts_from_edges(edges):
|
| 147 |
-
out = []
|
| 148 |
-
for e in edges or []:
|
| 149 |
-
n = (e or {}).get("node") or {}
|
| 150 |
-
code = n.get("shortcode")
|
| 151 |
-
if not code:
|
| 152 |
-
continue
|
| 153 |
-
caption = ""
|
| 154 |
-
for ce in ((n.get("edge_media_to_caption") or {}).get("edges") or []):
|
| 155 |
-
caption = ((ce or {}).get("node") or {}).get("text") or ""
|
| 156 |
-
break
|
| 157 |
-
ts = n.get("taken_at_timestamp")
|
| 158 |
-
out.append({
|
| 159 |
-
"shortcode": code,
|
| 160 |
-
"posted_at": (_dt.datetime.fromtimestamp(ts).strftime("%Y-%m-%d %H:%M")
|
| 161 |
-
if isinstance(ts, (int, float)) else ""),
|
| 162 |
-
"type": ("video" if n.get("is_video") else
|
| 163 |
-
"carousel" if n.get("__typename") == "GraphSidecar" else "image"),
|
| 164 |
-
"caption": caption,
|
| 165 |
-
"url": f"https://www.instagram.com/p/{code}/",
|
| 166 |
-
"likes": _ig_int((n.get("edge_liked_by") or n.get("edge_media_preview_like")
|
| 167 |
-
or {}).get("count")),
|
| 168 |
-
"comments": _ig_int((n.get("edge_media_to_comment") or {}).get("count")),
|
| 169 |
-
})
|
| 170 |
-
return out
|
| 171 |
-
|
| 172 |
-
|
| 173 |
-
_OG_COUNTS = re.compile(
|
| 174 |
-
r"([\d.,KMkm]+)\s+Followers?,\s*([\d.,KMkm]+)\s+Following,\s*([\d.,KMkm]+)\s+Posts?")
|
| 175 |
-
|
| 176 |
-
|
| 177 |
-
def _loose_count(txt):
|
| 178 |
-
"""'1.2M' / '12,345' -> int. IG's og:description abbreviates, so the snapshot is honest
|
| 179 |
-
about being approximate rather than pretending to a precision it does not have."""
|
| 180 |
-
t = str(txt or "").strip().replace(",", "")
|
| 181 |
-
mult = 1
|
| 182 |
-
if t[-1:].lower() == "k":
|
| 183 |
-
mult, t = 1_000, t[:-1]
|
| 184 |
-
elif t[-1:].lower() == "m":
|
| 185 |
-
mult, t = 1_000_000, t[:-1]
|
| 186 |
-
try:
|
| 187 |
-
return int(float(t) * mult)
|
| 188 |
-
except Exception:
|
| 189 |
-
return None
|
| 190 |
-
|
| 191 |
-
# ---------------------------------------------------------------------------------------------
|
| 192 |
-
# BRIGHT DATA β THE PAID RUNG (owner ruling R1 / wave-20 D-21, contract C4)
|
| 193 |
-
# ---------------------------------------------------------------------------------------------
|
| 194 |
-
# WHY A VENDOR AT ALL, stated once so nobody re-opens it: the anonymous ladder above is honest but
|
| 195 |
-
# it is ALSO geographically fenced β Instagram blocks datacenter egress, so from the HF Space or a
|
| 196 |
-
# Lambda the ladder degrades to `blocked` and the whole automation becomes a thing that only works
|
| 197 |
-
# on the owner's desk. The vendor takes the egress risk, returns EXACT counts (not
|
| 198 |
-
# og:description's "204K") and reaches post-level data the anonymous surface has closed.
|
| 199 |
-
#
|
| 200 |
-
# β WHY THIS VENDOR AND NOT THE PREVIOUS ONE. HikerAPI was crypto-only with no legal entity β
|
| 201 |
-
# no invoice, no counterparty, nothing that can go on a subprocessor list at the EU/enterprise
|
| 202 |
-
# gates. Bright Data is nameable, litigated Meta and won on logged-off public scraping, and runs
|
| 203 |
-
# KYC. That is a POSTURE purchase, not a feature one; the price is ~2.5Γ and it is the right
|
| 204 |
-
# trade. Full comparison: `.claude/wiki/research/instagram-capture.md` Β§2a.
|
| 205 |
-
#
|
| 206 |
-
# β THE FOUR RAILS TRANSFER UNCHANGED (contract C4 β they were written vendor-agnostic):
|
| 207 |
-
# 1. **FAIL-CLOSED on the key.** No `AIOS_BRIGHTDATA_KEY` β this rung refuses with an honest
|
| 208 |
-
# blocked ATTEMPT and the anonymous ladder still runs. Never a crash, never a silent skip.
|
| 209 |
-
# 2. **The key is never logged, never in a URL, never in an exception.** It travels in the
|
| 210 |
-
# `Authorization: Bearer` header only, and every error below quotes a STATUS, never the
|
| 211 |
-
# request. A secret that reaches a log line has leaked to everyone who can read logs.
|
| 212 |
-
# 3. **The base URL is guarded like any other.** `AIOS_BRIGHTDATA_BASE` is env-provided, so it
|
| 213 |
-
# goes through the same SSRF rail β an env var pointed at the metadata endpoint would
|
| 214 |
-
# otherwise be a credential-bearing request to it.
|
| 215 |
-
# 4. **A rung failure is a RUNG failure.** `bd_call` swallows its own `Refused`/transport
|
| 216 |
-
# errors, because `run_now` turns a `Refused` into a whole-run error β and a mistyped base
|
| 217 |
-
# URL must not kill a run whose anonymous rung would have answered.
|
| 218 |
-
# 5. **Exact β NO `approx` TAG.** That tag is the anonymous rung's rounding disclosure.
|
| 219 |
-
#
|
| 220 |
-
# β THE KEY IS READ LAZILY, INSIDE THE CALL. A module-level snapshot would be taken at import,
|
| 221 |
-
# which makes the "key absent β blocked, not crash" gate untestable (nothing could flip it between
|
| 222 |
-
# checks) and would also mean a deploy that sets the secret after import silently has no key.
|
| 223 |
-
#
|
| 224 |
-
# β TWO SNAPSHOT NAMESPACES THAT 404 EACH OTHER (measured; the trap that costs an hour):
|
| 225 |
-
# the SCRAPER `/datasets/v3/trigger` + `/v3/scrape` -> `sd_β¦` read at `/datasets/v3/β¦`
|
| 226 |
-
# the CORPUS `/datasets/filter` (β NO `/v3/`) -> `snap_β¦` read at `/datasets/β¦`
|
| 227 |
-
# Cross them and you get a flat "404 Snapshot does not exist" about a snapshot that is alive.
|
| 228 |
-
|
| 229 |
-
#: Pinned from live probes against `api.brightdata.com` (2026-08-04 and 2026-08-05) β see
|
| 230 |
-
#: `.claude/wiki/research/instagram-capture.md` Β§2c/Β§2d for the recipe and its provenance tags.
|
| 231 |
-
#: They live in ONE named block so a vendor rename is a one-line fix rather than a hunt.
|
| 232 |
BD_BASE_DEFAULT = "https://api.brightdata.com"
|
| 233 |
BD_DS_PROFILES = "gd_l1vikfch901nx3by4" # Instagram β Profiles. 36 fields, 620M records
|
| 234 |
BD_DS_POSTS = "gd_lk5ns7kz21pck8jpis" # Instagram β Posts. 43 fields
|
|
@@ -1213,6 +1100,83 @@ def _tag_metric_deferrals(items, start, kind, influencer_key):
|
|
| 1213 |
item["requestedAt"] = _iso()
|
| 1214 |
|
| 1215 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1216 |
def pull_profile_bd(url, max_posts=None, post_metrics=False,
|
| 1217 |
comment_metrics=False, log=print, pending_metrics=None,
|
| 1218 |
pending_profile=None):
|
|
@@ -1380,61 +1344,7 @@ def pull_profile_bd(url, max_posts=None, post_metrics=False,
|
|
| 1380 |
elif metric_notes:
|
| 1381 |
note = f"some post metrics were unavailable: {'; '.join(metric_notes)}"
|
| 1382 |
|
| 1383 |
-
|
| 1384 |
-
# 2026-08-08, `providers.py`). Bright Data has already answered profile + likes + comments
|
| 1385 |
-
# above and is declared INCAPABLE of `ig_post_views`, so the chain for that ONE capability
|
| 1386 |
-
# resolves to Apify and only the video permalinks are re-bought. Re-running the whole
|
| 1387 |
-
# profile on the second vendor would pay twice for the 90% that already worked, which at
|
| 1388 |
-
# thousands of tenants x twelve posts is the entire bill.
|
| 1389 |
-
# β VIDEO ONLY. A carousel or image has no view count, so sending it would buy a record
|
| 1390 |
-
# that can only come back blank.
|
| 1391 |
-
video_urls = [p["url"] for p in posts
|
| 1392 |
-
if p.get("type") == "video" and p.get("url")]
|
| 1393 |
-
if video_urls:
|
| 1394 |
-
def _views_work(prov, _urls=tuple(video_urls)):
|
| 1395 |
-
if prov.key == "apify":
|
| 1396 |
-
return apify_posts(list(_urls))
|
| 1397 |
-
# A provider in the chain with no runner here is a configuration error, not a
|
| 1398 |
-
# vendor outage β say so rather than returning an empty list that reads as "the
|
| 1399 |
-
# vendor had nothing".
|
| 1400 |
-
return None, f"no {prov.key} runner is wired for ig_post_views"
|
| 1401 |
-
|
| 1402 |
-
got_views, attempts = providers.run(
|
| 1403 |
-
"ig_post_views", _views_work,
|
| 1404 |
-
# β THE SATISFIED PREDICATE IS THE FALLBACK TRIGGER. Rows that come back with
|
| 1405 |
-
# every `views` blank are a FAILURE for this capability even at HTTP 200 β which
|
| 1406 |
-
# is exactly how Bright Data behaves, and why a chain that only caught exceptions
|
| 1407 |
-
# would never have reached a second provider.
|
| 1408 |
-
satisfied=lambda rows: bool(rows) and any(r.get("views") for r in rows),
|
| 1409 |
-
log=log)
|
| 1410 |
-
for row in (got_views or []):
|
| 1411 |
-
target = next((p for p in posts
|
| 1412 |
-
if p.get("shortcode") == row.get("shortcode")), None)
|
| 1413 |
-
# β TAKE ONLY THE CAPABILITY THAT WAS ASKED FOR. This provider also returns
|
| 1414 |
-
# likes/comments/caption, and letting them land would silently switch the source
|
| 1415 |
-
# of columns Bright Data already answered β the schema would be stable but the
|
| 1416 |
-
# PROVENANCE would flip halfway through a row.
|
| 1417 |
-
if target and row.get("views") not in (None, ""):
|
| 1418 |
-
target["views"] = row["views"]
|
| 1419 |
-
# β LOCATION RIDES ALONG FOR FREE, AND ONLY INTO A GAP (2026-08-08).
|
| 1420 |
-
# MEASURED: Bright Data's `location_details` is RICHER when present β it carries
|
| 1421 |
-
# real coordinates (`lat -6.9246, lng 106.9292, name "Sukabumi"`) which Apify does
|
| 1422 |
-
# not return at all β but it is populated on only 44 of 227 posts, and on
|
| 1423 |
-
# `DblAkEbv0ry` it returned nothing while Apify returned "Jakarta, Indonesia".
|
| 1424 |
-
# So Bright Data stays the source and this fills only what it left BLANK.
|
| 1425 |
-
# β THE COST ARGUMENT IS WHY IT IS HERE AND NOT ITS OWN CAPABILITY: this Apify
|
| 1426 |
-
# record has ALREADY been bought for the view count and carries `locationName` in
|
| 1427 |
-
# the same payload. Routing location as a separate capability would buy a second
|
| 1428 |
-
# record for a field that is already sitting in this response.
|
| 1429 |
-
if target and not str(target.get("tagged_location") or "").strip() and str(row.get("tagged_location") or "").strip():
|
| 1430 |
-
target["tagged_location"] = row["tagged_location"]
|
| 1431 |
-
used = next((a.provider for a in attempts if a.ok), "")
|
| 1432 |
-
if used:
|
| 1433 |
-
note = "; ".join(x for x in (note, f"view counts via {used}") if x)
|
| 1434 |
-
elif attempts:
|
| 1435 |
-
note = "; ".join(x for x in (
|
| 1436 |
-
note, "no provider could supply view counts: "
|
| 1437 |
-
+ "; ".join(f"{a.provider} {a.note}" for a in attempts)) if x)
|
| 1438 |
|
| 1439 |
# ββ DEBT D-82 (owner ruling R12: *"price, then wire the Posts-dataset call so posts_count
|
| 1440 |
# fills with the true value"*). The Profiles dataset's `posts_count: 0` is discarded as the
|
|
@@ -1744,32 +1654,36 @@ def bd_filter_rows(snapshot_id):
|
|
| 1744 |
return rows, ""
|
| 1745 |
|
| 1746 |
|
| 1747 |
-
def pull_profile(url, max_posts=None, log=print,
|
| 1748 |
-
fallback=True,
|
| 1749 |
post_metrics=False, comment_metrics=False, pending_metrics=None,
|
| 1750 |
pending_profile=None):
|
| 1751 |
-
"""Everything readable about one public profile
|
| 1752 |
|
| 1753 |
Returns `{state, profile, posts, via, note}` with state β ok | partial | blocked | error.
|
| 1754 |
-
`partial` means the identity was read but the media was not
|
| 1755 |
-
|
| 1756 |
-
|
| 1757 |
-
|
| 1758 |
-
|
| 1759 |
-
|
| 1760 |
-
|
| 1761 |
-
|
| 1762 |
-
|
| 1763 |
-
|
| 1764 |
-
|
| 1765 |
-
|
| 1766 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1767 |
|
| 1768 |
β `max_posts=None` means the default β see `pull_profile_bd` for why the constant cannot be
|
| 1769 |
named in the signature any more.
|
| 1770 |
"""
|
| 1771 |
from automation_engine import (DEFAULT_POSTS_PER_PULL, PACE_SECONDS, # lazy β module header
|
| 1772 |
-
_s
|
| 1773 |
max_posts = DEFAULT_POSTS_PER_PULL if max_posts is None else max_posts
|
| 1774 |
handle = ig_handle(url)
|
| 1775 |
if not handle:
|
|
@@ -1781,173 +1695,55 @@ def pull_profile(url, max_posts=None, log=print, tier="anonymous",
|
|
| 1781 |
# inferred from a mere failure, only from a vendor stating it.
|
| 1782 |
gone = ""
|
| 1783 |
|
| 1784 |
-
|
| 1785 |
-
|
| 1786 |
-
|
| 1787 |
-
|
| 1788 |
-
|
| 1789 |
-
|
| 1790 |
-
|
| 1791 |
-
|
| 1792 |
-
|
| 1793 |
-
|
| 1794 |
-
|
| 1795 |
-
|
| 1796 |
-
|
| 1797 |
-
|
| 1798 |
-
|
| 1799 |
-
|
| 1800 |
-
|
| 1801 |
-
|
| 1802 |
-
|
| 1803 |
-
|
| 1804 |
-
|
| 1805 |
-
|
| 1806 |
-
|
| 1807 |
-
|
| 1808 |
-
|
| 1809 |
-
|
| 1810 |
-
|
| 1811 |
-
|
| 1812 |
-
|
| 1813 |
-
|
| 1814 |
-
|
| 1815 |
-
|
| 1816 |
-
|
| 1817 |
-
|
| 1818 |
-
|
| 1819 |
-
|
| 1820 |
-
|
| 1821 |
-
attempts.append(f"apify:{_s(a_note or 'answered without follower counts', 200)}")
|
| 1822 |
-
time.sleep(PACE_SECONDS)
|
| 1823 |
-
if not fallback:
|
| 1824 |
-
return {"state": "blocked", "profile": {}, "posts": [], "comments": [],
|
| 1825 |
-
"via": "brightdata", "gone": bool(gone),
|
| 1826 |
-
"note": (gone + " (the anonymous fallback is turned off for this automation)"
|
| 1827 |
-
if gone else
|
| 1828 |
-
"the paid rungs did not answer and the anonymous fallback is turned "
|
| 1829 |
-
f"off for this automation ({'; '.join(attempts) or 'blocked'})")}
|
| 1830 |
time.sleep(PACE_SECONDS)
|
| 1831 |
|
| 1832 |
-
#
|
| 1833 |
-
#
|
| 1834 |
-
|
| 1835 |
-
|
| 1836 |
-
|
| 1837 |
-
|
| 1838 |
-
|
| 1839 |
-
|
| 1840 |
-
|
| 1841 |
-
|
| 1842 |
-
|
| 1843 |
-
|
| 1844 |
-
|
| 1845 |
-
if node:
|
| 1846 |
-
profile = _profile_from_web_api(node, handle)
|
| 1847 |
-
media = node.get("edge_owner_to_timeline_media") or {}
|
| 1848 |
-
posts = _posts_from_edges(media.get("edges"))[:max_posts]
|
| 1849 |
-
page = media.get("page_info") or {}
|
| 1850 |
-
if page.get("has_next_page") and page.get("end_cursor") and len(posts) < max_posts:
|
| 1851 |
-
more, pnote = _paginate(node.get("id"), page.get("end_cursor"),
|
| 1852 |
-
max_posts - len(posts), log)
|
| 1853 |
-
posts.extend(more)
|
| 1854 |
-
if pnote:
|
| 1855 |
-
attempts.append(pnote)
|
| 1856 |
-
state = "ok" if posts else "partial"
|
| 1857 |
-
return {"state": state, "profile": profile, "posts": posts,
|
| 1858 |
-
"via": "web_profile_info",
|
| 1859 |
-
"note": "" if posts else "profile read; media edges empty ("
|
| 1860 |
-
+ ", ".join(attempts) + ")"}
|
| 1861 |
-
|
| 1862 |
-
time.sleep(PACE_SECONDS)
|
| 1863 |
-
|
| 1864 |
-
# RUNG 2 β the profile HTML. Its og:description carries the three counts even when every
|
| 1865 |
-
# JSON surface refuses, so it is the honest floor: identity without media.
|
| 1866 |
-
status, body, note = _ig_get(f"https://www.instagram.com/{handle}/")
|
| 1867 |
-
attempts.append(f"html:{status or note}")
|
| 1868 |
-
if 200 <= status < 300 and body:
|
| 1869 |
-
text = body.decode("utf-8", "replace")
|
| 1870 |
-
if "/accounts/login" in text[:4000] and "og:description" not in text:
|
| 1871 |
-
return {"state": "blocked", "profile": {}, "posts": [], "via": "html",
|
| 1872 |
-
"note": "Instagram served the login wall (" + ", ".join(attempts) + ")"}
|
| 1873 |
-
desc = ""
|
| 1874 |
-
m = re.search(r'<meta[^>]+property="og:description"[^>]+content="([^"]*)"', text)
|
| 1875 |
-
if m:
|
| 1876 |
-
desc = m.group(1)
|
| 1877 |
-
counts = _OG_COUNTS.search(desc or "")
|
| 1878 |
-
title = ""
|
| 1879 |
-
mt = re.search(r'<meta[^>]+property="og:title"[^>]+content="([^"]*)"', text)
|
| 1880 |
-
if mt:
|
| 1881 |
-
title = mt.group(1)
|
| 1882 |
-
if counts:
|
| 1883 |
-
profile = {
|
| 1884 |
-
"username": handle, "full_name": title.split("(")[0].strip(),
|
| 1885 |
-
# β NO BIO. `og:description` on a profile page is
|
| 1886 |
-
# "204K Followers, 5,245 Following, 750 Posts - See Instagram photos and videos
|
| 1887 |
-
# from Inayma (@inayma)" β the tail is Instagram's own boilerplate, not the
|
| 1888 |
-
# person's biography. Storing it would fill a Bio column with the same sentence
|
| 1889 |
-
# for every influencer, which reads as data and is not. Measured 2026-08-04.
|
| 1890 |
-
"bio": "",
|
| 1891 |
-
"followers": _loose_count(counts.group(1)),
|
| 1892 |
-
"following": _loose_count(counts.group(2)),
|
| 1893 |
-
"posts_count": _loose_count(counts.group(3)),
|
| 1894 |
-
"verified": "",
|
| 1895 |
-
# The page abbreviates ("204K"), so the number is the page's rounding, not a
|
| 1896 |
-
# count we read. Recorded on the row rather than left for someone to discover
|
| 1897 |
-
# when 204,000 fails to reconcile with anything.
|
| 1898 |
-
"approx": "1",
|
| 1899 |
-
}
|
| 1900 |
-
return {"state": "partial", "profile": profile, "posts": [], "via": "og:description",
|
| 1901 |
-
"note": "counts are the page's own abbreviations; media is not anonymously "
|
| 1902 |
-
"readable (" + ", ".join(attempts) + ")"}
|
| 1903 |
-
# ββ THE DEFINITIVE ANSWER WINS OVER THE LAST RUNG'S SHRUG. If a vendor said the account does
|
| 1904 |
-
# not exist and the free rungs then also found nothing, the honest report is *"this account
|
| 1905 |
-
# does not exist"* β not *"nothing anonymously readable"*, which is a statement about US.
|
| 1906 |
-
# β It is checked HERE, after the free rungs have run, and not as an early return: an
|
| 1907 |
-
# anonymous read that DOES succeed above proves the vendor wrong and returns `ok`/`partial`
|
| 1908 |
-
# on its own. A vendor's not-found is strong evidence, never a reason to stop looking.
|
| 1909 |
-
if gone:
|
| 1910 |
-
return {"state": "blocked", "profile": {}, "posts": [], "via": "", "gone": True,
|
| 1911 |
-
"note": f"{gone}. Nothing else could read it either ({', '.join(attempts)})"}
|
| 1912 |
-
if status in (401, 403, 429) or status == 0:
|
| 1913 |
-
return {"state": "blocked", "profile": {}, "posts": [], "via": "",
|
| 1914 |
-
"note": f"Instagram refused the anonymous read ({', '.join(attempts)})"}
|
| 1915 |
-
return {"state": "error", "profile": {}, "posts": [], "via": "",
|
| 1916 |
-
"note": f"nothing anonymously readable ({', '.join(attempts)})"}
|
| 1917 |
-
|
| 1918 |
-
|
| 1919 |
-
#: The historical anonymous media query. Kept because trying it and RECORDING the refusal is
|
| 1920 |
-
#: what makes "as deep as anonymously accessible" (R7) a measurement instead of an assumption.
|
| 1921 |
-
IG_MEDIA_QUERY_HASH = "e769aa130647d2354c40ea6a439bfc08"
|
| 1922 |
-
|
| 1923 |
-
|
| 1924 |
-
def _paginate(user_id, cursor, want, log=print):
|
| 1925 |
-
"""Walk further back through the media edges while the anonymous surface allows it."""
|
| 1926 |
-
from automation_engine import PACE_SECONDS # lazy β see the module header
|
| 1927 |
-
out, note = [], ""
|
| 1928 |
-
while user_id and cursor and len(out) < want:
|
| 1929 |
-
time.sleep(PACE_SECONDS)
|
| 1930 |
-
variables = json.dumps({"id": str(user_id), "first": min(50, want - len(out)),
|
| 1931 |
-
"after": cursor}, separators=(",", ":"))
|
| 1932 |
-
url = (f"https://www.instagram.com/graphql/query/?query_hash={IG_MEDIA_QUERY_HASH}"
|
| 1933 |
-
f"&variables={requests.utils.quote(variables)}")
|
| 1934 |
-
status, body, err = _ig_get(url, headers={"X-IG-App-ID": IG_APP_ID})
|
| 1935 |
-
if not (200 <= status < 300) or not body:
|
| 1936 |
-
note = f"graphql:{status or err}"
|
| 1937 |
-
break
|
| 1938 |
-
try:
|
| 1939 |
-
media = (((json.loads(body.decode("utf-8", "replace")) or {}).get("data") or {})
|
| 1940 |
-
.get("user") or {}).get("edge_owner_to_timeline_media") or {}
|
| 1941 |
-
except Exception:
|
| 1942 |
-
note = "graphql:unparseable"
|
| 1943 |
-
break
|
| 1944 |
-
edges = media.get("edges") or []
|
| 1945 |
-
if not edges:
|
| 1946 |
-
note = "graphql:empty"
|
| 1947 |
-
break
|
| 1948 |
-
out.extend(_posts_from_edges(edges))
|
| 1949 |
-
page = media.get("page_info") or {}
|
| 1950 |
-
if not page.get("has_next_page"):
|
| 1951 |
-
break
|
| 1952 |
-
cursor = page.get("end_cursor")
|
| 1953 |
-
return out[:want], note
|
|
|
|
| 34 |
probe doing `import connectors_ig` would have got a half-built engine and an AttributeError from
|
| 35 |
inside a vendor call. **An intermittent-by-import-order failure is the worst shape available**, so
|
| 36 |
every reach-back is a `from automation_engine import β¦` INSIDE the function that needs it, where
|
| 37 |
+
the module is always fully built. There are exactly seven β `_bd_why`, `bd_call`,
|
| 38 |
+
`apify_posts`, `apify_profile`, `_tag_metric_deferrals`, `pull_profile_bd`, `pull_profile` β each
|
| 39 |
marked `# lazy β see the module header`, and they reach for four kinds of thing:
|
| 40 |
|
| 41 |
fetch / fetch_json / Refused the SSRF-guarded HTTP rail (the scrape runner shares it)
|
|
|
|
| 54 |
"""
|
| 55 |
from __future__ import annotations
|
| 56 |
|
|
|
|
| 57 |
import hashlib
|
| 58 |
import json
|
| 59 |
import os
|
|
|
|
| 67 |
|
| 68 |
|
| 69 |
# ---------------------------------------------------------------------------------------------
|
| 70 |
+
# HANDLE + COUNT READERS (no vendor, no key β pure parsing)
|
| 71 |
# ---------------------------------------------------------------------------------------------
|
| 72 |
+
# β THE ANONYMOUS LADDER THAT USED TO LIVE HERE IS DELETED (wave 28, owner ruling R5), and this
|
| 73 |
+
# note is the reason it must not come back on a "we could read this for free" impulse. Its two
|
| 74 |
+
# rungs β Instagram's `web_profile_info` endpoint and the profile HTML's `og:description` β
|
| 75 |
+
# returned counts the PAGE had already rounded ("204K"), which landed in the same `followers`
|
| 76 |
+
# column as a measured 204,318 under an `approx: "1"` flag nobody sees until after they have
|
| 77 |
+
# averaged it. Free was never the problem; a column you cannot do arithmetic on was.
|
| 78 |
+
# Gone with it: `_ig_get`, `IG_APP_ID`, `IG_MEDIA_QUERY_HASH`, `_paginate`, `_posts_from_edges`,
|
| 79 |
+
# `_profile_from_web_api`, `_OG_COUNTS`, `_loose_count`.
|
| 80 |
|
| 81 |
def ig_handle(url):
|
| 82 |
"""The handle out of a profile URL (or a bare handle). '' when it is not one."""
|
|
|
|
| 94 |
return re.sub(r"[^A-Za-z0-9._]", "", first)[:40]
|
| 95 |
|
| 96 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 97 |
def _ig_int(v):
|
| 98 |
try:
|
| 99 |
return int(v)
|
|
|
|
| 116 |
return None if n == 0 else n
|
| 117 |
|
| 118 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 119 |
BD_BASE_DEFAULT = "https://api.brightdata.com"
|
| 120 |
BD_DS_PROFILES = "gd_l1vikfch901nx3by4" # Instagram β Profiles. 36 fields, 620M records
|
| 121 |
BD_DS_POSTS = "gd_lk5ns7kz21pck8jpis" # Instagram β Posts. 43 fields
|
|
|
|
| 1100 |
item["requestedAt"] = _iso()
|
| 1101 |
|
| 1102 |
|
| 1103 |
+
def top_up_views(posts, log=print):
|
| 1104 |
+
"""Fill `views` on video posts from the `ig_post_views` capability. MUTATES `posts`; returns
|
| 1105 |
+
a note for the run (`""` when there was nothing to do).
|
| 1106 |
+
|
| 1107 |
+
ββ CAPABILITY ROUTING, NOT PROVIDER FAILOVER (owner ruling 2026-08-08, `providers.py`).
|
| 1108 |
+
Bright Data answers profile + likes + comments and is declared INCAPABLE of `ig_post_views`,
|
| 1109 |
+
so the chain for that ONE capability resolves to Apify and only the video permalinks are
|
| 1110 |
+
re-bought. Re-running the whole profile on the second vendor would pay twice for the 90% that
|
| 1111 |
+
already worked, which at thousands of tenants x twelve posts is the entire bill.
|
| 1112 |
+
|
| 1113 |
+
ββ WHY THIS IS A FUNCTION AND NOT INLINE, which is the whole point of the 2026-08-09 fix.
|
| 1114 |
+
It used to live inside `pull_profile_bd`'s post-metrics block, so it ran ONLY when the Posts
|
| 1115 |
+
scrape answered within the wait budget. When that batch DEFERRED β routine, and the normal
|
| 1116 |
+
outcome on a busy account β the posts came back through
|
| 1117 |
+
`automation_engine.collect_pending_metric_snapshots` instead, which never called it. MEASURED
|
| 1118 |
+
on nurilab's `theresalearns`: profile, 12 posts and 154 comments all landed, and **every
|
| 1119 |
+
Views cell was blank**, because the deferred path never asked Apify at all. One capability,
|
| 1120 |
+
two collection paths, one of them wired: the same shape as the `ig_profile` rung that was
|
| 1121 |
+
declared and never called ([[flag-shipped-without-its-writer]]).
|
| 1122 |
+
β ONE implementation, TWO callers. A copy in the collector would have been a second thing to
|
| 1123 |
+
keep in step, and the two would drift on the next ruling.
|
| 1124 |
+
|
| 1125 |
+
β VIDEO ONLY. A carousel or image has no view count, so sending it would buy a record that
|
| 1126 |
+
can only come back blank.
|
| 1127 |
+
"""
|
| 1128 |
+
video_urls = [p["url"] for p in (posts or [])
|
| 1129 |
+
if p.get("type") == "video" and p.get("url")]
|
| 1130 |
+
if not video_urls:
|
| 1131 |
+
return ""
|
| 1132 |
+
|
| 1133 |
+
def _views_work(prov, _urls=tuple(video_urls)):
|
| 1134 |
+
if prov.key == "apify":
|
| 1135 |
+
return apify_posts(list(_urls))
|
| 1136 |
+
# A provider in the chain with no runner here is a configuration error, not a vendor
|
| 1137 |
+
# outage β say so rather than returning an empty list that reads as "the vendor had
|
| 1138 |
+
# nothing".
|
| 1139 |
+
return None, f"no {prov.key} runner is wired for ig_post_views"
|
| 1140 |
+
|
| 1141 |
+
got_views, attempts = providers.run(
|
| 1142 |
+
"ig_post_views", _views_work,
|
| 1143 |
+
# β THE SATISFIED PREDICATE IS THE FALLBACK TRIGGER. Rows that come back with every
|
| 1144 |
+
# `views` blank are a FAILURE for this capability even at HTTP 200 β which is exactly how
|
| 1145 |
+
# Bright Data behaves, and why a chain that only caught exceptions would never have
|
| 1146 |
+
# reached a second provider.
|
| 1147 |
+
satisfied=lambda rows: bool(rows) and any(r.get("views") for r in rows),
|
| 1148 |
+
log=log)
|
| 1149 |
+
for row in (got_views or []):
|
| 1150 |
+
target = next((p for p in posts
|
| 1151 |
+
if p.get("shortcode") == row.get("shortcode")), None)
|
| 1152 |
+
# β TAKE ONLY THE CAPABILITY THAT WAS ASKED FOR. This provider also returns
|
| 1153 |
+
# likes/comments/caption, and letting them land would silently switch the source of
|
| 1154 |
+
# columns Bright Data already answered β the schema would be stable but the PROVENANCE
|
| 1155 |
+
# would flip halfway through a row.
|
| 1156 |
+
if target and row.get("views") not in (None, ""):
|
| 1157 |
+
target["views"] = row["views"]
|
| 1158 |
+
# β LOCATION RIDES ALONG FOR FREE, AND ONLY INTO A GAP (2026-08-08).
|
| 1159 |
+
# MEASURED: Bright Data's `location_details` is RICHER when present β it carries real
|
| 1160 |
+
# coordinates (`lat -6.9246, lng 106.9292, name "Sukabumi"`) which Apify does not return
|
| 1161 |
+
# at all β but it is populated on only 44 of 227 posts, and on `DblAkEbv0ry` it returned
|
| 1162 |
+
# nothing while Apify returned "Jakarta, Indonesia". So Bright Data stays the source and
|
| 1163 |
+
# this fills only what it left BLANK.
|
| 1164 |
+
# β THE COST ARGUMENT IS WHY IT IS HERE AND NOT ITS OWN CAPABILITY: this Apify record has
|
| 1165 |
+
# ALREADY been bought for the view count and carries `locationName` in the same payload.
|
| 1166 |
+
# Routing location as a separate capability would buy a second record for a field that is
|
| 1167 |
+
# already sitting in this response.
|
| 1168 |
+
if target and not str(target.get("tagged_location") or "").strip() \
|
| 1169 |
+
and str(row.get("tagged_location") or "").strip():
|
| 1170 |
+
target["tagged_location"] = row["tagged_location"]
|
| 1171 |
+
used = next((a.provider for a in attempts if a.ok), "")
|
| 1172 |
+
if used:
|
| 1173 |
+
return f"view counts via {used}"
|
| 1174 |
+
if attempts:
|
| 1175 |
+
return ("no provider could supply view counts: "
|
| 1176 |
+
+ "; ".join(f"{a.provider} {a.note}" for a in attempts))
|
| 1177 |
+
return ""
|
| 1178 |
+
|
| 1179 |
+
|
| 1180 |
def pull_profile_bd(url, max_posts=None, post_metrics=False,
|
| 1181 |
comment_metrics=False, log=print, pending_metrics=None,
|
| 1182 |
pending_profile=None):
|
|
|
|
| 1344 |
elif metric_notes:
|
| 1345 |
note = f"some post metrics were unavailable: {'; '.join(metric_notes)}"
|
| 1346 |
|
| 1347 |
+
note = "; ".join(x for x in (note, top_up_views(posts, log=log)) if x)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1348 |
|
| 1349 |
# ββ DEBT D-82 (owner ruling R12: *"price, then wire the Posts-dataset call so posts_count
|
| 1350 |
# fills with the true value"*). The Profiles dataset's `posts_count: 0` is discarded as the
|
|
|
|
| 1654 |
return rows, ""
|
| 1655 |
|
| 1656 |
|
| 1657 |
+
def pull_profile(url, max_posts=None, log=print,
|
|
|
|
| 1658 |
post_metrics=False, comment_metrics=False, pending_metrics=None,
|
| 1659 |
pending_profile=None):
|
| 1660 |
+
"""Everything readable about one public profile β EXACT NUMBERS OR NOTHING.
|
| 1661 |
|
| 1662 |
Returns `{state, profile, posts, via, note}` with state β ok | partial | blocked | error.
|
| 1663 |
+
`partial` means the identity was read but the media was not.
|
| 1664 |
+
|
| 1665 |
+
ββ WAVE 28 / OWNER RULING R5 β THE FREE ANONYMOUS LADDER IS RETIRED, AND THE REASON IS WHAT
|
| 1666 |
+
IT PUT IN THE COLUMN, not what it cost. Two rungs used to sit under the paid one: Instagram's
|
| 1667 |
+
own `web_profile_info` endpoint, and the profile HTML's `og:description`, which carries
|
| 1668 |
+
"204K Followers, 5,245 Following, 750 Posts". That last one is the page's OWN ROUNDING, and it
|
| 1669 |
+
landed in the same `followers` column as a measured 204,318, stamped `approx: "1"` β a flag
|
| 1670 |
+
that tells you the number is an abbreviation only AFTER you have already averaged it, sorted
|
| 1671 |
+
on it, or filtered a shortlist by it. A table where some rows are measurements and some are
|
| 1672 |
+
the page's shorthand is a table you cannot do arithmetic on.
|
| 1673 |
+
β A vendor refusal now returns `blocked` and the row KEEPS WHAT IT LAST KNEW, instead of being
|
| 1674 |
+
overwritten with a rounder number. The user-facing tier/fallback choice is gone with the
|
| 1675 |
+
rungs; there is one behaviour and it is this one.
|
| 1676 |
+
|
| 1677 |
+
β THIS IS STILL CAPABILITY ROUTING, NOT A SINGLE VENDOR. `ig_profile` is declared on Bright
|
| 1678 |
+
Data AND Apify (`providers.py`), so a profile Bright Data cannot read is asked of Apify before
|
| 1679 |
+
anything reports blocked. What was retired is the FREE approximate rung, not the second
|
| 1680 |
+
opinion β the fallback that costs money and answers exactly is the one worth having.
|
| 1681 |
|
| 1682 |
β `max_posts=None` means the default β see `pull_profile_bd` for why the constant cannot be
|
| 1683 |
named in the signature any more.
|
| 1684 |
"""
|
| 1685 |
from automation_engine import (DEFAULT_POSTS_PER_PULL, PACE_SECONDS, # lazy β module header
|
| 1686 |
+
_s)
|
| 1687 |
max_posts = DEFAULT_POSTS_PER_PULL if max_posts is None else max_posts
|
| 1688 |
handle = ig_handle(url)
|
| 1689 |
if not handle:
|
|
|
|
| 1695 |
# inferred from a mere failure, only from a vendor stating it.
|
| 1696 |
gone = ""
|
| 1697 |
|
| 1698 |
+
paid = pull_profile_bd(url, max_posts=max_posts, post_metrics=post_metrics,
|
| 1699 |
+
comment_metrics=comment_metrics, log=log,
|
| 1700 |
+
pending_metrics=pending_metrics,
|
| 1701 |
+
pending_profile=pending_profile)
|
| 1702 |
+
if paid["state"] in ("ok", "partial"):
|
| 1703 |
+
return paid
|
| 1704 |
+
# β 200, NOT 90. This string is the ONLY account of what the vendor said, and at 90 the
|
| 1705 |
+
# measured sentence truncated to *"β¦was not ready within"* β mid-clause, with the budget and
|
| 1706 |
+
# the snapshot id cut off. A reason nobody can read is the reason being discarded with extra
|
| 1707 |
+
# steps (D-103).
|
| 1708 |
+
attempts.append(f"brightdata:{_s(paid.get('note'), 200)}")
|
| 1709 |
+
# β THE APIFY RUNG (owner report 2026-08-09: "both Bright Data and Apify working in tandem").
|
| 1710 |
+
# `providers.py` has declared `ig_profile -> ("brightdata", "apify")` since 2026-08-08 and
|
| 1711 |
+
# nothing ever walked it: Bright Data failing went straight to the anonymous rungs, past a
|
| 1712 |
+
# configured provider declared capable of exactly this.
|
| 1713 |
+
#
|
| 1714 |
+
# β IT RUNS ONLY AFTER BRIGHT DATA HAS ACTUALLY FAILED, never as a top-up. Bright Data answers
|
| 1715 |
+
# profile+likes+comments correctly and far more cheaply; re-buying a profile from a second
|
| 1716 |
+
# vendor on every read is the expensive mistake the capability split exists to prevent. This
|
| 1717 |
+
# is a fallback, and the `attempts` trail records that it was needed.
|
| 1718 |
+
#
|
| 1719 |
+
# β AND IT IS GATED ON `configured()`, not merely on being in the chain β an unconfigured
|
| 1720 |
+
# provider must read as a rung that was never tried, not as one that refused.
|
| 1721 |
+
if providers.PROVIDERS["apify"].can("ig_profile"):
|
| 1722 |
+
a_prof, a_note = apify_profile(handle)
|
| 1723 |
+
if a_prof and a_prof.get("followers") is not None:
|
| 1724 |
+
log(f" apify: profile ok ({a_prof.get('followers')} followers)")
|
| 1725 |
+
return {"state": "ok", "profile": a_prof, "posts": [], "comments": [],
|
| 1726 |
+
"via": "apify",
|
| 1727 |
+
"note": ("Bright Data could not read this profile "
|
| 1728 |
+
f"({_s(paid.get('note'), 90)}), so it came from Apify")}
|
| 1729 |
+
# β A VENDOR SAYING *not found* IS THE ANSWER, NOT A FAILED ATTEMPT. Recorded here and
|
| 1730 |
+
# carried to the return below, so the run can say the one thing that ends the loop
|
| 1731 |
+
# instead of the fourth variation on "we could not read it".
|
| 1732 |
+
if a_note == ACCOUNT_GONE_NOTE:
|
| 1733 |
+
gone = a_note
|
| 1734 |
+
attempts.append(f"apify:{_s(a_note or 'answered without follower counts', 200)}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1735 |
time.sleep(PACE_SECONDS)
|
| 1736 |
|
| 1737 |
+
# ββ THE END OF THE WALK (R5). There were two more rungs here β `web_profile_info` and the
|
| 1738 |
+
# profile HTML's `og:description` β and they are DELETED, not disabled. What they returned was
|
| 1739 |
+
# a `partial` carrying `approx: "1"` and counts rounded by Instagram's own page furniture; the
|
| 1740 |
+
# docstring says why that is worse than nothing. `blocked` keeps the row's last real values.
|
| 1741 |
+
# β `via` STAYS "brightdata" on this path rather than "" β it names the chain that was walked,
|
| 1742 |
+
# which is what a person reading a blocked row needs in order to know who to ask.
|
| 1743 |
+
return {"state": "blocked", "profile": {}, "posts": [], "comments": [],
|
| 1744 |
+
"via": "brightdata", "gone": bool(gone),
|
| 1745 |
+
"note": (gone if gone else
|
| 1746 |
+
"no provider could read this profile "
|
| 1747 |
+
f"({'; '.join(attempts) or 'blocked'})")}
|
| 1748 |
+
|
| 1749 |
+
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
api/odoo_relational.py
CHANGED
|
@@ -54,11 +54,29 @@ INVOICES_KEY = "ut_odoo_invoices"
|
|
| 54 |
CUSTOMERS_KEY = "ut_odoo_customers"
|
| 55 |
ORDERS_KEY = "ut_odoo_orders"
|
| 56 |
PRODUCTS_KEY = "ut_odoo_products"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 57 |
|
| 58 |
#: The join column the partner-grain tables carry. Derived links resolve through it (`on`/`from`).
|
| 59 |
JOIN_KEY = "partner_id"
|
| 60 |
#: The product-grain equivalent.
|
| 61 |
PRODUCT_JOIN_KEY = "product_id"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 62 |
|
| 63 |
#: The oracle's own predicate β `modules/ar._open_docs`, copied rather than re-derived so the two
|
| 64 |
#: cannot drift. It now selects a SUBSET of the invoices table rather than defining it.
|
|
@@ -119,6 +137,134 @@ def _refreshed_field():
|
|
| 119 |
"default": False, "description": "When this row was last reconciled against Odoo."}
|
| 120 |
|
| 121 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 122 |
def invoice_fields():
|
| 123 |
"""One row per POSTED customer invoice or refund β the full history, not just what is open."""
|
| 124 |
return [_preset(f) for f in (
|
|
@@ -155,6 +301,19 @@ def invoice_fields():
|
|
| 155 |
{"key": "customer_link", "label": "Customer record", "type": "link", "source": "overlay",
|
| 156 |
"default": False,
|
| 157 |
"link": {"table": CUSTOMERS_KEY, "on": JOIN_KEY, "from": JOIN_KEY}},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 158 |
_refreshed_field(),
|
| 159 |
)]
|
| 160 |
|
|
@@ -184,6 +343,11 @@ def order_fields():
|
|
| 184 |
{"key": "customer_link", "label": "Customer record", "type": "link", "source": "overlay",
|
| 185 |
"default": False,
|
| 186 |
"link": {"table": CUSTOMERS_KEY, "on": JOIN_KEY, "from": JOIN_KEY}},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 187 |
_refreshed_field(),
|
| 188 |
)]
|
| 189 |
|
|
@@ -268,6 +432,12 @@ def customer_fields():
|
|
| 268 |
"default": True,
|
| 269 |
"description": "The customer's assigned agent (res.partner.agent_ids[0] β the "
|
| 270 |
"Customers-module convention)."},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 271 |
_scope_field(),
|
| 272 |
|
| 273 |
# --- the relations -------------------------------------------------------------------
|
|
@@ -277,6 +447,11 @@ def customer_fields():
|
|
| 277 |
{"key": "orders", "label": "Orders", "type": "link", "source": "overlay",
|
| 278 |
"default": True,
|
| 279 |
"link": {"table": ORDERS_KEY, "on": JOIN_KEY, "from": JOIN_KEY}},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 280 |
|
| 281 |
# --- link rollups over the invoice history --------------------------------------------
|
| 282 |
# β NO CONDITION, and that is measured rather than assumed: a settled document's residual
|
|
@@ -437,12 +612,19 @@ def read_invoices(cur, excluded=None, open_only=False):
|
|
| 437 |
"""
|
| 438 |
excluded = excluded if excluded is not None else excluded_ids(cur)
|
| 439 |
where = _AR_WHERE if open_only else _POSTED_DOCS
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 440 |
sql = ("SELECT id, name, partner_id, partner_name, invoice_date, invoice_date_due, "
|
| 441 |
-
" amount_untaxed_signed, amount_residual_signed, payment_state, move_type "
|
|
|
|
| 442 |
f"FROM account_move WHERE {where} AND partner_id IS NOT NULL")
|
| 443 |
out = []
|
| 444 |
for r in cur.execute(sql).fetchall():
|
| 445 |
-
(mid, name, pid, pname, inv_date, due, untaxed, residual, pay_state, mtype) = r
|
| 446 |
scope = _in_scope(pid, excluded)
|
| 447 |
if open_only and not scope:
|
| 448 |
continue
|
|
@@ -458,6 +640,7 @@ def read_invoices(cur, excluded=None, open_only=False):
|
|
| 458 |
"amount_untaxed": float(untaxed or 0.0),
|
| 459 |
"payment_state": str(pay_state or ""),
|
| 460 |
"move_type": str(mtype or ""),
|
|
|
|
| 461 |
"wholesale_scope": scope,
|
| 462 |
})
|
| 463 |
return out
|
|
@@ -547,9 +730,10 @@ def read_customers(cur, excluded=None):
|
|
| 547 |
# β THE AGENT JOIN IS DROPPED WHOLE when `agent_id` is absent, not merely NULL-ed: the join
|
| 548 |
# itself names the column, so `_col` on the SELECT list alone would still fail to bind.
|
| 549 |
agent = ("ag.name" if "agent_id" in have else "NULL")
|
|
|
|
| 550 |
join = ("LEFT JOIN res_partner ag ON ag.id = p.agent_id " if "agent_id" in have else "")
|
| 551 |
sql = (f"SELECT p.id, p.name, {_col(have, 'p.city')}, {_col(have, 'p.state_name')}, "
|
| 552 |
-
f" {_col(have, 'p.country_name')}, {agent} "
|
| 553 |
"FROM res_partner p "
|
| 554 |
f"{join}"
|
| 555 |
"WHERE p.id IN ("
|
|
@@ -559,7 +743,7 @@ def read_customers(cur, excluded=None):
|
|
| 559 |
f" WHERE {_POSTED_DOCS} AND partner_id IS NOT NULL)")
|
| 560 |
out = []
|
| 561 |
for r in cur.execute(sql).fetchall():
|
| 562 |
-
(pid, name, city, state, country, agent) = r
|
| 563 |
out.append({
|
| 564 |
"_id": str(pid),
|
| 565 |
"customer": str(name or ""),
|
|
@@ -568,11 +752,146 @@ def read_customers(cur, excluded=None):
|
|
| 568 |
"state": str(state or ""),
|
| 569 |
"country": str(country or ""),
|
| 570 |
"agent": str(agent or ""),
|
|
|
|
| 571 |
"wholesale_scope": _in_scope(pid, excluded),
|
| 572 |
})
|
| 573 |
return out
|
| 574 |
|
| 575 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 576 |
def customers_from(invoice_rows):
|
| 577 |
"""The partners carrying the given invoice rows β the pre-2026-08-09 population builder.
|
| 578 |
|
|
@@ -608,6 +927,13 @@ TABLES = (
|
|
| 608 |
("products", PRODUCTS_KEY, "Odoo products", product_fields),
|
| 609 |
("invoices", INVOICES_KEY, "Odoo invoices", invoice_fields),
|
| 610 |
("orders", ORDERS_KEY, "Odoo orders", order_fields),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 611 |
)
|
| 612 |
|
| 613 |
|
|
@@ -631,6 +957,10 @@ def plan(cur, rt=None):
|
|
| 631 |
"products": read_products(cur),
|
| 632 |
"invoices": read_invoices(cur, excluded=excluded),
|
| 633 |
"orders": read_orders(cur, excluded=excluded),
|
|
|
|
|
|
|
|
|
|
|
|
|
| 634 |
}
|
| 635 |
problems = []
|
| 636 |
|
|
|
|
| 54 |
CUSTOMERS_KEY = "ut_odoo_customers"
|
| 55 |
ORDERS_KEY = "ut_odoo_orders"
|
| 56 |
PRODUCTS_KEY = "ut_odoo_products"
|
| 57 |
+
#: β WAVE 28 (owner R1): *"ALL of Unique ID in Odoo is a database e.g. Customers/Products/Agents,
|
| 58 |
+
#: etc. Including expenses and GL codes."* Four more DOCUMENT/REGISTRY grains, each measured to
|
| 59 |
+
#: fit far inside `MAX_ROWS` (19 / 192 / 6,538 / 393 against 60,000).
|
| 60 |
+
#: β THE LINE GRAINS ARE DELIBERATELY NOT HERE and this is R2, not an omission: 256,810 order
|
| 61 |
+
#: lines, 154,917 expense GL lines and 311,140 commission lines cannot live in a `ut_*` document
|
| 62 |
+
#: at any cap (63.9 MB / 2.57 s per copy for the order lines alone). "Expenses" as a BROWSABLE
|
| 63 |
+
#: LEDGER is a read-through mirror grid; "expenses" as a NUMBER is a rollup on the GL account row.
|
| 64 |
+
AGENTS_KEY = "ut_odoo_agents"
|
| 65 |
+
ACCOUNTS_KEY = "ut_odoo_accounts"
|
| 66 |
+
BILLS_KEY = "ut_odoo_bills"
|
| 67 |
+
VENDORS_KEY = "ut_odoo_vendors"
|
| 68 |
|
| 69 |
#: The join column the partner-grain tables carry. Derived links resolve through it (`on`/`from`).
|
| 70 |
JOIN_KEY = "partner_id"
|
| 71 |
#: The product-grain equivalent.
|
| 72 |
PRODUCT_JOIN_KEY = "product_id"
|
| 73 |
+
#: β AN AGENT IS A `res.partner`, so its id shares the partner namespace with a customer's β but
|
| 74 |
+
#: it is a DIFFERENT COLUMN on the customer row (`agent_id`, the customer's assigned agent) and the
|
| 75 |
+
#: two must never be joined through `JOIN_KEY`, which would link every customer to itself.
|
| 76 |
+
AGENT_JOIN_KEY = "agent_id"
|
| 77 |
+
#: A vendor is also a `res.partner`; same reasoning, its own column.
|
| 78 |
+
VENDOR_JOIN_KEY = "vendor_id"
|
| 79 |
+
ACCOUNT_JOIN_KEY = "account_code"
|
| 80 |
|
| 81 |
#: The oracle's own predicate β `modules/ar._open_docs`, copied rather than re-derived so the two
|
| 82 |
#: cannot drift. It now selects a SUBSET of the invoices table rather than defining it.
|
|
|
|
| 137 |
"default": False, "description": "When this row was last reconciled against Odoo."}
|
| 138 |
|
| 139 |
|
| 140 |
+
def agent_fields():
|
| 141 |
+
"""One row per SALES AGENT, keyed on the `res.partner` id.
|
| 142 |
+
|
| 143 |
+
β THE POPULATION IS A UNION OF TWO DISAGREEING SOURCES, and the disagreement is the reason it
|
| 144 |
+
is a union rather than a pick. MEASURED 2026-08-09: 16 partners carry commission lines, 17
|
| 145 |
+
carry `res_partner.agent = TRUE`, and the union is 19 β so **2 agents earn commission without
|
| 146 |
+
the flag and 3 are flagged with no commission yet**. Either source alone silently drops real
|
| 147 |
+
agents. Same shape as `read_customers`' two document universes, for the same reason.
|
| 148 |
+
"""
|
| 149 |
+
return [_preset(f) for f in (
|
| 150 |
+
{"key": "agent", "label": "Agent", "type": "text", "source": "overlay",
|
| 151 |
+
"default": True, "pinned": True},
|
| 152 |
+
{"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay",
|
| 153 |
+
"default": False, "description": "The `res.partner` id. Also this row's id."},
|
| 154 |
+
{"key": AGENT_JOIN_KEY, "label": "Odoo agent id", "type": "int", "source": "overlay",
|
| 155 |
+
"default": False},
|
| 156 |
+
{"key": "flagged", "label": "Flagged in Odoo", "type": "checkbox", "source": "overlay",
|
| 157 |
+
"default": True,
|
| 158 |
+
"description": "Ticked = `res.partner.agent` is set. Unticked agents were found by "
|
| 159 |
+
"their commission lines instead - both are real, which is why this "
|
| 160 |
+
"table is the union of the two."},
|
| 161 |
+
{"key": "commissioned", "label": "Has commission lines", "type": "checkbox",
|
| 162 |
+
"source": "overlay", "default": True},
|
| 163 |
+
# β THE INVERSE HALF: the customers whose `agent_id` names this agent. MEASURED: 2,093
|
| 164 |
+
# customers carry one and ALL 2,093 resolve to a row in this table (zero dangling).
|
| 165 |
+
{"key": "customers", "label": "Customers", "type": "link", "source": "overlay",
|
| 166 |
+
"default": True,
|
| 167 |
+
"link": {"table": CUSTOMERS_KEY, "on": AGENT_JOIN_KEY, "from": AGENT_JOIN_KEY}},
|
| 168 |
+
_refreshed_field(),
|
| 169 |
+
)]
|
| 170 |
+
|
| 171 |
+
|
| 172 |
+
def account_fields():
|
| 173 |
+
"""One row per `account.account` β the GL chart, the owner's "GL codes".
|
| 174 |
+
|
| 175 |
+
β NO LINK COLUMN, and that is a finding rather than an omission. A GL account meets the rest
|
| 176 |
+
of this schema only at LINE grain (963,783 `account_move_line` rows, 154,917 of them on
|
| 177 |
+
expense-type accounts), and a `ut_*` link folds rows that live in the store. The honest
|
| 178 |
+
binding is a read-through rollup naming a governed topic, or the mirror grid (R2) β never a
|
| 179 |
+
link into a table that does not exist. Declaring one here would render a permanently blank
|
| 180 |
+
column, which is the exact trap D-87 warns about from the value site.
|
| 181 |
+
"""
|
| 182 |
+
return [_preset(f) for f in (
|
| 183 |
+
{"key": "account_code", "label": "Code", "type": "text", "source": "overlay",
|
| 184 |
+
"default": True, "pinned": True},
|
| 185 |
+
{"key": "account_name", "label": "Account", "type": "text", "source": "overlay",
|
| 186 |
+
"default": True},
|
| 187 |
+
{"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay",
|
| 188 |
+
"default": False, "description": "The `account.account` id. Also this row's id."},
|
| 189 |
+
# β 15 DISTINCT VALUES MEASURED IN THE MIRROR, all declared. A `select` storing a value its
|
| 190 |
+
# options omit is wave-26 item 24: the filter panel answers with a list that cannot match
|
| 191 |
+
# what is stored.
|
| 192 |
+
{"key": "account_type", "label": "Type", "type": "select", "source": "overlay",
|
| 193 |
+
"default": True,
|
| 194 |
+
"options": ["expense", "expense_direct_cost", "expense_depreciation", "income",
|
| 195 |
+
"income_other", "asset_cash", "asset_current", "asset_receivable",
|
| 196 |
+
"asset_fixed", "asset_non_current", "asset_prepayments",
|
| 197 |
+
"liability_current", "liability_payable", "liability_credit_card",
|
| 198 |
+
"liability_non_current", "equity", "equity_unaffected", "off_balance"]},
|
| 199 |
+
{"key": "is_expense", "label": "Expense account", "type": "checkbox", "source": "overlay",
|
| 200 |
+
"default": True,
|
| 201 |
+
"description": "Ticked for the expense family - the same predicate the semantic layer's "
|
| 202 |
+
"gl_lines topic uses, so this column and that topic cannot disagree."},
|
| 203 |
+
_refreshed_field(),
|
| 204 |
+
)]
|
| 205 |
+
|
| 206 |
+
|
| 207 |
+
def vendor_fields():
|
| 208 |
+
"""One row per partner we have POSTED a vendor bill to, keyed on the `res.partner` id.
|
| 209 |
+
|
| 210 |
+
β A VENDOR IS NOT A CUSTOMER TABLE ROW, even though both are `res.partner`. MEASURED: 393
|
| 211 |
+
vendors, of which only 9 also appear in the customer population. Pointing bills at
|
| 212 |
+
`ut_odoo_customers` would have dangled 384 of 393 links β the failure would have been a mostly
|
| 213 |
+
empty column, not an error.
|
| 214 |
+
"""
|
| 215 |
+
return [_preset(f) for f in (
|
| 216 |
+
{"key": "vendor", "label": "Vendor", "type": "text", "source": "overlay",
|
| 217 |
+
"default": True, "pinned": True},
|
| 218 |
+
{"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay",
|
| 219 |
+
"default": False, "description": "The `res.partner` id. Also this row's id."},
|
| 220 |
+
{"key": VENDOR_JOIN_KEY, "label": "Odoo vendor id", "type": "int", "source": "overlay",
|
| 221 |
+
"default": False},
|
| 222 |
+
{"key": "country", "label": "Country", "type": "text", "source": "overlay",
|
| 223 |
+
"default": True},
|
| 224 |
+
{"key": "bills", "label": "Bills", "type": "link", "source": "overlay", "default": True,
|
| 225 |
+
"link": {"table": BILLS_KEY, "on": VENDOR_JOIN_KEY, "from": VENDOR_JOIN_KEY}},
|
| 226 |
+
_refreshed_field(),
|
| 227 |
+
)]
|
| 228 |
+
|
| 229 |
+
|
| 230 |
+
def bill_fields():
|
| 231 |
+
"""One row per POSTED vendor bill or refund β the owner's "expenses", at DOCUMENT grain.
|
| 232 |
+
|
| 233 |
+
β DOCUMENT GRAIN IS A CHOICE AND IT IS THE ONLY ONE THAT FITS: 6,538 bills against 154,917
|
| 234 |
+
expense GL lines. What a person calls "expenses" is both, and they are different tables - the
|
| 235 |
+
bill is what you pay, the line is what it was coded to. This is the payable; the line ledger
|
| 236 |
+
is the read-through mirror grid (R2).
|
| 237 |
+
"""
|
| 238 |
+
return [_preset(f) for f in (
|
| 239 |
+
{"key": "bill_no", "label": "Bill", "type": "text", "source": "overlay",
|
| 240 |
+
"default": True, "pinned": True},
|
| 241 |
+
{"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay",
|
| 242 |
+
"default": False, "description": "The `account.move` id. Also this row's id."},
|
| 243 |
+
{"key": "vendor", "label": "Vendor", "type": "text", "source": "overlay", "default": True},
|
| 244 |
+
{"key": VENDOR_JOIN_KEY, "label": "Odoo vendor id", "type": "int", "source": "overlay",
|
| 245 |
+
"default": False},
|
| 246 |
+
{"key": "invoice_date", "label": "Bill date", "type": "date", "source": "overlay",
|
| 247 |
+
"default": True},
|
| 248 |
+
{"key": "due_date", "label": "Due date", "type": "date", "source": "overlay",
|
| 249 |
+
"default": True},
|
| 250 |
+
# β SIGNED, like the customer side: Odoo's `_signed` fields already carry the refund's
|
| 251 |
+
# direction, so a refund reduces a total without anybody re-deriving a sign here.
|
| 252 |
+
{"key": "amount_untaxed", "label": "Billed $", "type": "currency", "source": "overlay",
|
| 253 |
+
"default": True, "agg": "sum"},
|
| 254 |
+
{"key": "residual", "label": "Outstanding $", "type": "currency", "source": "overlay",
|
| 255 |
+
"default": True, "agg": "sum"},
|
| 256 |
+
{"key": "payment_state", "label": "Payment state", "type": "select", "source": "overlay",
|
| 257 |
+
"default": True,
|
| 258 |
+
"options": ["not_paid", "partial", "in_payment", "paid", "reversed"]},
|
| 259 |
+
{"key": "move_type", "label": "Document", "type": "select", "source": "overlay",
|
| 260 |
+
"default": False, "options": ["in_invoice", "in_refund"]},
|
| 261 |
+
{"key": "vendor_link", "label": "Vendor record", "type": "link", "source": "overlay",
|
| 262 |
+
"default": False,
|
| 263 |
+
"link": {"table": VENDORS_KEY, "on": VENDOR_JOIN_KEY, "from": VENDOR_JOIN_KEY}},
|
| 264 |
+
_refreshed_field(),
|
| 265 |
+
)]
|
| 266 |
+
|
| 267 |
+
|
| 268 |
def invoice_fields():
|
| 269 |
"""One row per POSTED customer invoice or refund β the full history, not just what is open."""
|
| 270 |
return [_preset(f) for f in (
|
|
|
|
| 301 |
{"key": "customer_link", "label": "Customer record", "type": "link", "source": "overlay",
|
| 302 |
"default": False,
|
| 303 |
"link": {"table": CUSTOMERS_KEY, "on": JOIN_KEY, "from": JOIN_KEY}},
|
| 304 |
+
# β DEBT D-88, closed 2026-08-09. `invoice_origin` carries the ORDER NAME an invoice was
|
| 305 |
+
# raised from, and the mirror did not sync it until this wave β so order->invoice was a
|
| 306 |
+
# two-hop join through 963,783 `account_move_line` rows and wave 27 shipped no link at all.
|
| 307 |
+
# β IT IS A NAME, NOT AN ID, and Odoo writes free text there (a manual invoice can hold
|
| 308 |
+
# anything; a merged one can hold several origins space-separated). The link resolves
|
| 309 |
+
# against `order_no` and finds nothing when the text is not an order name β the honest
|
| 310 |
+
# outcome, and the reason this is a join HINT rather than a foreign key.
|
| 311 |
+
{"key": "origin_order", "label": "Source order", "type": "text", "source": "overlay",
|
| 312 |
+
"default": False,
|
| 313 |
+
"description": "Odoo's `invoice_origin` - usually the order name, sometimes blank."},
|
| 314 |
+
{"key": "order_link", "label": "Order record", "type": "link", "source": "overlay",
|
| 315 |
+
"default": False,
|
| 316 |
+
"link": {"table": ORDERS_KEY, "on": "order_no", "from": "origin_order"}},
|
| 317 |
_refreshed_field(),
|
| 318 |
)]
|
| 319 |
|
|
|
|
| 343 |
{"key": "customer_link", "label": "Customer record", "type": "link", "source": "overlay",
|
| 344 |
"default": False,
|
| 345 |
"link": {"table": CUSTOMERS_KEY, "on": JOIN_KEY, "from": JOIN_KEY}},
|
| 346 |
+
# The reciprocal of the invoice's `order_link` (D-88): the invoices raised from THIS
|
| 347 |
+
# order, matched on the order's own name.
|
| 348 |
+
{"key": "invoices", "label": "Invoices", "type": "link", "source": "overlay",
|
| 349 |
+
"default": True,
|
| 350 |
+
"link": {"table": INVOICES_KEY, "on": "origin_order", "from": "order_no"}},
|
| 351 |
_refreshed_field(),
|
| 352 |
)]
|
| 353 |
|
|
|
|
| 432 |
"default": True,
|
| 433 |
"description": "The customer's assigned agent (res.partner.agent_ids[0] β the "
|
| 434 |
"Customers-module convention)."},
|
| 435 |
+
# β WAVE 28 β the agent's ID beside its NAME, because a link joins on an id and this
|
| 436 |
+
# table carried only the display string. β It is `agent_id`, NEVER `partner_id`: both are
|
| 437 |
+
# `res.partner` ids, and joining agents through `JOIN_KEY` would link every customer to
|
| 438 |
+
# itself and look plausible doing it.
|
| 439 |
+
{"key": AGENT_JOIN_KEY, "label": "Odoo agent id", "type": "int", "source": "overlay",
|
| 440 |
+
"default": False},
|
| 441 |
_scope_field(),
|
| 442 |
|
| 443 |
# --- the relations -------------------------------------------------------------------
|
|
|
|
| 447 |
{"key": "orders", "label": "Orders", "type": "link", "source": "overlay",
|
| 448 |
"default": True,
|
| 449 |
"link": {"table": ORDERS_KEY, "on": JOIN_KEY, "from": JOIN_KEY}},
|
| 450 |
+
# MEASURED: 2,093 customers carry an `agent_id` and all 2,093 resolve to a row in the
|
| 451 |
+
# agents table β zero dangling, which is why this ships as a link rather than a lookup.
|
| 452 |
+
{"key": "agent_link", "label": "Agent record", "type": "link", "source": "overlay",
|
| 453 |
+
"default": False,
|
| 454 |
+
"link": {"table": AGENTS_KEY, "on": AGENT_JOIN_KEY, "from": AGENT_JOIN_KEY}},
|
| 455 |
|
| 456 |
# --- link rollups over the invoice history --------------------------------------------
|
| 457 |
# β NO CONDITION, and that is measured rather than assumed: a settled document's residual
|
|
|
|
| 612 |
"""
|
| 613 |
excluded = excluded if excluded is not None else excluded_ids(cur)
|
| 614 |
where = _AR_WHERE if open_only else _POSTED_DOCS
|
| 615 |
+
# β `invoice_origin` (D-88) is read through `_col` DELIBERATELY. It was added to `ENTITIES` in
|
| 616 |
+
# this same wave, so a Space whose mirror is still hydrating from a pre-wave seed snapshot does
|
| 617 |
+
# not have the column yet β and DuckDB answers a missing identifier with a Binder error that
|
| 618 |
+
# reaches the operator as a bare 500. This is the exact class `columns()` was written for: the
|
| 619 |
+
# link degrades to blank for one sync cycle instead of refusing the whole spawn.
|
| 620 |
+
have = columns(cur, "account_move")
|
| 621 |
sql = ("SELECT id, name, partner_id, partner_name, invoice_date, invoice_date_due, "
|
| 622 |
+
" amount_untaxed_signed, amount_residual_signed, payment_state, move_type, "
|
| 623 |
+
f" {_col(have, 'invoice_origin', chr(39) + chr(39))} "
|
| 624 |
f"FROM account_move WHERE {where} AND partner_id IS NOT NULL")
|
| 625 |
out = []
|
| 626 |
for r in cur.execute(sql).fetchall():
|
| 627 |
+
(mid, name, pid, pname, inv_date, due, untaxed, residual, pay_state, mtype, origin) = r
|
| 628 |
scope = _in_scope(pid, excluded)
|
| 629 |
if open_only and not scope:
|
| 630 |
continue
|
|
|
|
| 640 |
"amount_untaxed": float(untaxed or 0.0),
|
| 641 |
"payment_state": str(pay_state or ""),
|
| 642 |
"move_type": str(mtype or ""),
|
| 643 |
+
"origin_order": str(origin or "").strip(),
|
| 644 |
"wholesale_scope": scope,
|
| 645 |
})
|
| 646 |
return out
|
|
|
|
| 730 |
# β THE AGENT JOIN IS DROPPED WHOLE when `agent_id` is absent, not merely NULL-ed: the join
|
| 731 |
# itself names the column, so `_col` on the SELECT list alone would still fail to bind.
|
| 732 |
agent = ("ag.name" if "agent_id" in have else "NULL")
|
| 733 |
+
agent_id_col = ("p.agent_id" if "agent_id" in have else "NULL")
|
| 734 |
join = ("LEFT JOIN res_partner ag ON ag.id = p.agent_id " if "agent_id" in have else "")
|
| 735 |
sql = (f"SELECT p.id, p.name, {_col(have, 'p.city')}, {_col(have, 'p.state_name')}, "
|
| 736 |
+
f" {_col(have, 'p.country_name')}, {agent}, {agent_id_col} "
|
| 737 |
"FROM res_partner p "
|
| 738 |
f"{join}"
|
| 739 |
"WHERE p.id IN ("
|
|
|
|
| 743 |
f" WHERE {_POSTED_DOCS} AND partner_id IS NOT NULL)")
|
| 744 |
out = []
|
| 745 |
for r in cur.execute(sql).fetchall():
|
| 746 |
+
(pid, name, city, state, country, agent, agent_id) = r
|
| 747 |
out.append({
|
| 748 |
"_id": str(pid),
|
| 749 |
"customer": str(name or ""),
|
|
|
|
| 752 |
"state": str(state or ""),
|
| 753 |
"country": str(country or ""),
|
| 754 |
"agent": str(agent or ""),
|
| 755 |
+
AGENT_JOIN_KEY: int(agent_id) if agent_id else "",
|
| 756 |
"wholesale_scope": _in_scope(pid, excluded),
|
| 757 |
})
|
| 758 |
return out
|
| 759 |
|
| 760 |
|
| 761 |
+
def read_agents(cur):
|
| 762 |
+
"""[(row dict)] β the UNION of both agent sources, keyed on the `res.partner` id.
|
| 763 |
+
|
| 764 |
+
β `res_partner.agent` is a BOOLEAN and `datastore.BOOL_FIELDS` lists it for a measured reason:
|
| 765 |
+
Odoo returns False both for "empty" and for "boolean false", so a bool missing from that list
|
| 766 |
+
silently becomes NULL and every row would read "not an agent" indistinguishably from
|
| 767 |
+
"unknown". Read it as a truth value, never as a presence test.
|
| 768 |
+
"""
|
| 769 |
+
have = columns(cur, "res_partner")
|
| 770 |
+
if "id" not in have:
|
| 771 |
+
return []
|
| 772 |
+
flagged = "p.agent" if "agent" in have else "FALSE"
|
| 773 |
+
# ββ THE COMMISSION TABLE IS GUARDED AS A **TABLE**, not just as a column, and that
|
| 774 |
+
# distinction is the whole point of this block. `columns()` was written for a missing COLUMN
|
| 775 |
+
# (a backfill that has not run yet); `account_invoice_line_agent` is an OCA module entity that
|
| 776 |
+
# a mirror hydrated from an older seed snapshot may not have AT ALL. A SELECT naming an absent
|
| 777 |
+
# table is a DuckDB Binder error, and this reader runs inside `plan()` β so one missing table
|
| 778 |
+
# would fail the WHOLE eight-table spawn and reach the operator as a bare 500. That is
|
| 779 |
+
# precisely D-107's shape, and it would have arrived on the first deploy of this feature.
|
| 780 |
+
# β DEGRADE, NEVER REFUSE, which is the posture `columns()`'s own docstring sets: without the
|
| 781 |
+
# commission table the population falls back to the FLAGGED partners alone and `commissioned`
|
| 782 |
+
# reads blank for every row β fewer agents and an honestly empty column, rather than no spawn.
|
| 783 |
+
has_comm = bool(columns(cur, "account_invoice_line_agent"))
|
| 784 |
+
commissioned = ("(p.id IN (SELECT agent_id FROM account_invoice_line_agent "
|
| 785 |
+
" WHERE agent_id IS NOT NULL))" if has_comm else "FALSE")
|
| 786 |
+
union_leg = (" SELECT agent_id FROM account_invoice_line_agent WHERE agent_id IS NOT NULL "
|
| 787 |
+
" UNION " if has_comm else "")
|
| 788 |
+
sql = (f"SELECT p.id, p.name, {flagged}, {commissioned} AS commissioned "
|
| 789 |
+
"FROM res_partner p WHERE p.id IN ("
|
| 790 |
+
f"{union_leg}SELECT id FROM res_partner WHERE {flagged})")
|
| 791 |
+
out = []
|
| 792 |
+
for (aid, name, flag, comm) in cur.execute(sql).fetchall():
|
| 793 |
+
out.append({
|
| 794 |
+
"_id": str(aid),
|
| 795 |
+
"agent": str(name or ""),
|
| 796 |
+
"odoo_id": int(aid),
|
| 797 |
+
AGENT_JOIN_KEY: int(aid),
|
| 798 |
+
"flagged": "1" if flag else "",
|
| 799 |
+
"commissioned": "1" if comm else "",
|
| 800 |
+
})
|
| 801 |
+
return out
|
| 802 |
+
|
| 803 |
+
|
| 804 |
+
def read_accounts(cur):
|
| 805 |
+
"""[(row dict)] β the whole GL chart, keyed on the `account.account` id.
|
| 806 |
+
|
| 807 |
+
β THE EXPENSE PREDICATE IS THE SEMANTIC LAYER'S, copied rather than invented:
|
| 808 |
+
`harness/semantic.py`'s `gl_lines` topic scopes expenses as
|
| 809 |
+
`account_type in ('expense','expense_depreciation')`. A second definition here is how a
|
| 810 |
+
column and a topic start disagreeing about the same word.
|
| 811 |
+
β `account.account` has NO `active` column in this Odoo version (a domain naming it 500s), so
|
| 812 |
+
there is nothing to filter and every account is a row.
|
| 813 |
+
"""
|
| 814 |
+
have = columns(cur, "account_account")
|
| 815 |
+
if not have:
|
| 816 |
+
return []
|
| 817 |
+
sql = (f"SELECT id, {_col(have, 'code', chr(39) + chr(39))}, "
|
| 818 |
+
f" {_col(have, 'name', chr(39) + chr(39))}, "
|
| 819 |
+
f" {_col(have, 'account_type', chr(39) + chr(39))} FROM account_account")
|
| 820 |
+
out = []
|
| 821 |
+
for (aid, code, name, atype) in cur.execute(sql).fetchall():
|
| 822 |
+
t = str(atype or "")
|
| 823 |
+
out.append({
|
| 824 |
+
"_id": str(aid),
|
| 825 |
+
ACCOUNT_JOIN_KEY: str(code or ""),
|
| 826 |
+
"account_name": str(name or ""),
|
| 827 |
+
"odoo_id": int(aid),
|
| 828 |
+
"account_type": t,
|
| 829 |
+
"is_expense": "1" if t in ("expense", "expense_depreciation") else "",
|
| 830 |
+
})
|
| 831 |
+
return out
|
| 832 |
+
|
| 833 |
+
|
| 834 |
+
_VENDOR_DOCS = "state = 'posted' AND move_type IN ('in_invoice','in_refund')"
|
| 835 |
+
|
| 836 |
+
|
| 837 |
+
def read_bills(cur):
|
| 838 |
+
"""[(row dict)] β posted vendor bills and refunds, keyed on the `account.move` id."""
|
| 839 |
+
have = columns(cur, "account_move")
|
| 840 |
+
if not have:
|
| 841 |
+
return []
|
| 842 |
+
sql = ("SELECT id, name, partner_id, partner_name, invoice_date, invoice_date_due, "
|
| 843 |
+
f" {_col(have, 'amount_untaxed_signed', '0')}, "
|
| 844 |
+
f" {_col(have, 'amount_residual_signed', '0')}, "
|
| 845 |
+
f" {_col(have, 'payment_state', chr(39) + chr(39))}, move_type "
|
| 846 |
+
f"FROM account_move WHERE {_VENDOR_DOCS} AND partner_id IS NOT NULL")
|
| 847 |
+
out = []
|
| 848 |
+
for r in cur.execute(sql).fetchall():
|
| 849 |
+
(mid, name, pid, pname, when, due, untaxed, residual, pay, mtype) = r
|
| 850 |
+
out.append({
|
| 851 |
+
"_id": str(mid),
|
| 852 |
+
"bill_no": str(name or ""),
|
| 853 |
+
"odoo_id": int(mid),
|
| 854 |
+
"vendor": str(pname or ""),
|
| 855 |
+
VENDOR_JOIN_KEY: int(pid),
|
| 856 |
+
"invoice_date": _as_date(when),
|
| 857 |
+
"due_date": _as_date(due),
|
| 858 |
+
"amount_untaxed": float(untaxed or 0.0),
|
| 859 |
+
"residual": float(residual or 0.0),
|
| 860 |
+
"payment_state": str(pay or ""),
|
| 861 |
+
"move_type": str(mtype or ""),
|
| 862 |
+
})
|
| 863 |
+
return out
|
| 864 |
+
|
| 865 |
+
|
| 866 |
+
def read_vendors(cur):
|
| 867 |
+
"""[(row dict)] β every partner carrying a posted vendor bill, keyed on the `res.partner` id.
|
| 868 |
+
|
| 869 |
+
β DERIVED FROM THE BILLS, unlike `read_customers` which is deliberately NOT derived from its
|
| 870 |
+
invoices. The asymmetry is intentional and the reason is what that function's own comment
|
| 871 |
+
says: a customer registry sourced from receivables is what kept most Odoo ids out of the store.
|
| 872 |
+
There is no second document universe for vendors β a partner with no bill has no payable
|
| 873 |
+
history to show β so the bill IS the population, and MEASURED it dangles nothing (0 bills
|
| 874 |
+
carry a null partner; all 393 vendors resolve in `res_partner`).
|
| 875 |
+
"""
|
| 876 |
+
have = columns(cur, "res_partner")
|
| 877 |
+
if not have or not columns(cur, "account_move"):
|
| 878 |
+
return []
|
| 879 |
+
sql = (f"SELECT p.id, p.name, {_col(have, 'p.country_name', chr(39) + chr(39))} "
|
| 880 |
+
"FROM res_partner p WHERE p.id IN "
|
| 881 |
+
f" (SELECT partner_id FROM account_move WHERE {_VENDOR_DOCS} "
|
| 882 |
+
" AND partner_id IS NOT NULL)")
|
| 883 |
+
out = []
|
| 884 |
+
for (pid, name, country) in cur.execute(sql).fetchall():
|
| 885 |
+
out.append({
|
| 886 |
+
"_id": str(pid),
|
| 887 |
+
"vendor": str(name or ""),
|
| 888 |
+
"odoo_id": int(pid),
|
| 889 |
+
VENDOR_JOIN_KEY: int(pid),
|
| 890 |
+
"country": str(country or ""),
|
| 891 |
+
})
|
| 892 |
+
return out
|
| 893 |
+
|
| 894 |
+
|
| 895 |
def customers_from(invoice_rows):
|
| 896 |
"""The partners carrying the given invoice rows β the pre-2026-08-09 population builder.
|
| 897 |
|
|
|
|
| 927 |
("products", PRODUCTS_KEY, "Odoo products", product_fields),
|
| 928 |
("invoices", INVOICES_KEY, "Odoo invoices", invoice_fields),
|
| 929 |
("orders", ORDERS_KEY, "Odoo orders", order_fields),
|
| 930 |
+
# β WAVE 28 / R1. Measured populations: 19 / 192 / 6,538 / 393 β every one of them two orders
|
| 931 |
+
# of magnitude inside `MAX_ROWS`, which is why the answer to "every unique id is a database"
|
| 932 |
+
# is four more spec rows and four readers rather than a new substrate.
|
| 933 |
+
("agents", AGENTS_KEY, "Odoo agents", agent_fields),
|
| 934 |
+
("accounts", ACCOUNTS_KEY, "Odoo GL accounts", account_fields),
|
| 935 |
+
("vendors", VENDORS_KEY, "Odoo vendors", vendor_fields),
|
| 936 |
+
("bills", BILLS_KEY, "Odoo vendor bills", bill_fields),
|
| 937 |
)
|
| 938 |
|
| 939 |
|
|
|
|
| 957 |
"products": read_products(cur),
|
| 958 |
"invoices": read_invoices(cur, excluded=excluded),
|
| 959 |
"orders": read_orders(cur, excluded=excluded),
|
| 960 |
+
"agents": read_agents(cur),
|
| 961 |
+
"accounts": read_accounts(cur),
|
| 962 |
+
"vendors": read_vendors(cur),
|
| 963 |
+
"bills": read_bills(cur),
|
| 964 |
}
|
| 965 |
problems = []
|
| 966 |
|
api/routes_tables.py
CHANGED
|
@@ -842,5 +842,29 @@ def patch_row(table_key: str, pid: int, body: dict = Body(default=None),
|
|
| 842 |
cleared = sorted(k for k in also if k in _ut().PROFILE_PRESET_KEYS)
|
| 843 |
if cleared:
|
| 844 |
out["cleared"] = cleared
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 845 |
_refresh_relations(session)
|
| 846 |
return out
|
|
|
|
| 842 |
cleared = sorted(k for k in also if k in _ut().PROFILE_PRESET_KEYS)
|
| 843 |
if cleared:
|
| 844 |
out["cleared"] = cleared
|
| 845 |
+
# ββ R9's SECOND RE-ARM DOOR β the one call that makes `engine.clear_gone` live (wave 28,
|
| 846 |
+
# amendment A5; SESSION B built and gated the function and correctly declared it INERT until
|
| 847 |
+
# this line existed, citing [[flag-shipped-without-its-writer]]).
|
| 848 |
+
#
|
| 849 |
+
# β R9 makes a `not_found` handle a TOMBSTONE, not a 30-day backoff: nothing re-buys a dead
|
| 850 |
+
# account on a timer any more. Door 1 β correcting the handle β needs no wiring, because the
|
| 851 |
+
# verdict is keyed on `(platform, handle)` and a corrected handle simply is not the verdict we
|
| 852 |
+
# recorded. THIS is door 2: a human re-typing the SAME handle, which is how somebody says "try
|
| 853 |
+
# it again, the account is back". Without this call that person has no way back at all, and
|
| 854 |
+
# the failure costs nothing and raises nothing β so no spend-shaped test would ever find it.
|
| 855 |
+
#
|
| 856 |
+
# β GATED ON `updates`, NOT ON `accepted`: re-typing the identical value is the whole case this
|
| 857 |
+
# door exists for, and a no-op write can be filtered out of `accepted`. What matters is that a
|
| 858 |
+
# human touched the handle cell.
|
| 859 |
+
# β NOT a bare `except: pass`. A swallowed AttributeError here would be exactly the optional-
|
| 860 |
+
# prop silence this wiring exists to prevent β if the engine ever loses `clear_gone`, that must
|
| 861 |
+
# be readable in the log rather than degrade into "the re-arm quietly stopped working".
|
| 862 |
+
_pf = _ut().profile_field(table_key, st=session.runtime)
|
| 863 |
+
if _pf and _pf["key"] in updates:
|
| 864 |
+
try:
|
| 865 |
+
import automation_engine as _engine
|
| 866 |
+
_engine.clear_gone(session.runtime, table_key, stored.get(_pf["key"]))
|
| 867 |
+
except Exception as e: # noqa: BLE001
|
| 868 |
+
print(f"[aios-api] clear_gone failed: {type(e).__name__}: {e}")
|
| 869 |
_refresh_relations(session)
|
| 870 |
return out
|
platform/core/user_tables.py
CHANGED
|
@@ -1047,6 +1047,21 @@ def _clean_field(raw, previous=None):
|
|
| 1047 |
# column it is editing. Clearing it is deliberate work, not a side effect of renaming a header.
|
| 1048 |
if prev.get('pinned') is True or raw.get('pinned') is True:
|
| 1049 |
out['pinned'] = True
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1050 |
# An automation column carries the binding the engine reads; it is preserved across a patch
|
| 1051 |
# rather than re-declared, because the automation editor owns it and this route does not.
|
| 1052 |
auto = raw.get('automation') if 'automation' in raw else prev.get('automation')
|
|
|
|
| 1047 |
# column it is editing. Clearing it is deliberate work, not a side effect of renaming a header.
|
| 1048 |
if prev.get('pinned') is True or raw.get('pinned') is True:
|
| 1049 |
out['pinned'] = True
|
| 1050 |
+
# β 2026-08-09 (wave 28) β `agg` DECIDES WHETHER THE GRID'S TOTALS ROW SUMS THE COLUMN, and
|
| 1051 |
+
# this door silently dropped it while `clean_fields` (:402) kept it. Two validators, one
|
| 1052 |
+
# question, opposite answers β D-91's exact class, found from the other end: the register's row
|
| 1053 |
+
# described `clean_fields` as the lax door, and by the time it was read that half had been
|
| 1054 |
+
# fixed while THIS one still diverged.
|
| 1055 |
+
# β THE CONSEQUENCE IS QUIET, WHICH IS WHY IT SURVIVED. Every currency and money-rollup column
|
| 1056 |
+
# in the four Odoo databases declares `agg: "sum"`; the spawn writes definitions straight to
|
| 1057 |
+
# the store, so the totals row worked β until anyone PATCHED one of those fields (a rename, a
|
| 1058 |
+
# width, a description), at which point the column silently stopped totalling and nothing
|
| 1059 |
+
# anywhere went red. Caught by a gate leg written to assert the LINK bag survives this door,
|
| 1060 |
+
# which found `agg` instead.
|
| 1061 |
+
# β Sticky across a patch for the same reason as `default` and `pinned`: a client PATCH that
|
| 1062 |
+
# omits the key must not un-total a column as a side effect of renaming its header.
|
| 1063 |
+
if str(raw.get('agg') or prev.get('agg') or '').strip() == 'sum':
|
| 1064 |
+
out['agg'] = 'sum'
|
| 1065 |
# An automation column carries the binding the engine reads; it is preserved across a patch
|
| 1066 |
# rather than re-declared, because the automation editor owns it and this route does not.
|
| 1067 |
auto = raw.get('automation') if 'automation' in raw else prev.get('automation')
|
web/src/automation/AutomationBuilder.tsx
CHANGED
|
@@ -587,7 +587,15 @@ export default function AutomationBuilder({
|
|
| 587 |
*/
|
| 588 |
if (kind === "enrich_instagram")
|
| 589 |
return {
|
| 590 |
-
tier: "anonymous"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 591 |
dryRun: false, maxPosts: 10,
|
| 592 |
fromView: "", sortField: "first_found", sortDir: "desc",
|
| 593 |
// β `skipRecent: true` MATCHES `_ensure_enrich_step`'s server seed, deliberately. One
|
|
@@ -2052,7 +2060,16 @@ function NewDatabase({
|
|
| 2052 |
);
|
| 2053 |
}
|
| 2054 |
|
| 2055 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2056 |
action,
|
| 2057 |
pinned,
|
| 2058 |
catalog,
|
|
@@ -2433,35 +2450,65 @@ function ActionProps({
|
|
| 2433 |
</div>
|
| 2434 |
) : null}
|
| 2435 |
|
| 2436 |
-
|
| 2437 |
-
|
| 2438 |
-
|
| 2439 |
-
|
| 2440 |
-
|
| 2441 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2442 |
disabled={busy}
|
| 2443 |
-
onChange={(e) => setCfg({
|
| 2444 |
-
>
|
| 2445 |
-
|
| 2446 |
-
|
| 2447 |
-
</select>
|
| 2448 |
-
{/* β THE MONEY SENTENCE IS ON THE MONEY OPTION. R7's rule about the machine steps'
|
| 2449 |
-
switches applies to their replacement: a control whose cost is invisible is a
|
| 2450 |
-
control nobody can consent to. */}
|
| 2451 |
-
<p className="auto-hint">
|
| 2452 |
-
{String((cfg as { tier?: string }).tier || "anonymous") === "brightdata"
|
| 2453 |
-
? "Each profile is billed by the provider."
|
| 2454 |
-
: "Free, and some profiles will come back thin."}
|
| 2455 |
-
</p>
|
| 2456 |
-
</div>
|
| 2457 |
<label className="auto-check">
|
| 2458 |
<input
|
|
|
|
| 2459 |
type="checkbox"
|
| 2460 |
-
checked={!!(cfg as {
|
| 2461 |
disabled={busy}
|
| 2462 |
-
onChange={(e) => setCfg({
|
| 2463 |
/>
|
| 2464 |
-
|
| 2465 |
</label>
|
| 2466 |
{/*
|
| 2467 |
β WAVE 26 Β· ITEM 7 / R2 + C5 β TWO CONTROLS, AND THEY ARE INDEPENDENT.
|
|
@@ -2497,41 +2544,19 @@ function ActionProps({
|
|
| 2497 |
At most 12 β the provider returns a profile’s top 12 posts and no more.
|
| 2498 |
</p>
|
| 2499 |
</div>
|
| 2500 |
-
|
| 2501 |
-
|
| 2502 |
-
|
| 2503 |
-
|
| 2504 |
-
|
| 2505 |
-
|
| 2506 |
-
|
| 2507 |
-
|
| 2508 |
-
|
| 2509 |
-
|
| 2510 |
-
|
| 2511 |
-
|
| 2512 |
-
|
| 2513 |
-
is billed and per what. A distinct badge would read as a different KIND of warning
|
| 2514 |
-
for the same kind of fact. */}
|
| 2515 |
-
{(cfg as { postMetrics?: boolean }).postMetrics ? (
|
| 2516 |
-
<p className="auto-hint">
|
| 2517 |
-
Billed per post, not per profile β a separate scrape for each post kept above.
|
| 2518 |
-
</p>
|
| 2519 |
-
) : null}
|
| 2520 |
-
<label className="auto-check">
|
| 2521 |
-
<input
|
| 2522 |
-
type="checkbox"
|
| 2523 |
-
checked={!!(cfg as { commentMetrics?: boolean }).commentMetrics}
|
| 2524 |
-
disabled={busy}
|
| 2525 |
-
onChange={(e) => setCfg({ commentMetrics: e.target.checked })}
|
| 2526 |
-
/>
|
| 2527 |
-
Also capture per-comment engagement
|
| 2528 |
-
</label>
|
| 2529 |
-
{(cfg as { commentMetrics?: boolean }).commentMetrics ? (
|
| 2530 |
-
<p className="auto-hint">
|
| 2531 |
-
Billed by the separate Comments dataset. This is off by default; embedded comments
|
| 2532 |
-
already returned with a paid post are retained without this additional request.
|
| 2533 |
-
</p>
|
| 2534 |
-
) : null}
|
| 2535 |
<label className="auto-check">
|
| 2536 |
<input
|
| 2537 |
type="checkbox"
|
|
|
|
| 587 |
*/
|
| 588 |
if (kind === "enrich_instagram")
|
| 589 |
return {
|
| 590 |
+
// β WAVE 28 Β· R5/R6 β `tier: "anonymous"` and `noFallback: false` are GONE from the seed.
|
| 591 |
+
// R5 retired the free ladder from enrichment, so a new action must not be born naming a
|
| 592 |
+
// source strategy the runner no longer consults. The server keeps ACCEPTING both keys on
|
| 593 |
+
// stored definitions and ignores them (C2, D-65's law) β but a SEED is what a new step
|
| 594 |
+
// starts with, and seeding a dead key is how a retired concept outlives its removal.
|
| 595 |
+
// β `postMetrics`/`commentMetrics` stay `false` here and that is R6's ruling verbatim
|
| 596 |
+
// ("always both off until toggled on"), matching `clean_action_config`'s reading of an
|
| 597 |
+
// absent key so the seed and the validator cannot disagree.
|
| 598 |
+
postMetrics: false, commentMetrics: false,
|
| 599 |
dryRun: false, maxPosts: 10,
|
| 600 |
fromView: "", sortField: "first_found", sortDir: "desc",
|
| 601 |
// β `skipRecent: true` MATCHES `_ensure_enrich_step`'s server seed, deliberately. One
|
|
|
|
| 2060 |
);
|
| 2061 |
}
|
| 2062 |
|
| 2063 |
+
/**
|
| 2064 |
+
* β WAVE 28 β EXPORTED so the include-row render suite can mount the real panel.
|
| 2065 |
+
*
|
| 2066 |
+
* β THE EXPORT IS THE POINT, not a convenience. R6's three include rows are markup whose defect
|
| 2067 |
+
* modes are invisible to a source grep: a row that renders with no key attached, a Profile
|
| 2068 |
+
* checkbox that paints unchecked, two rows bound to the same key. `ReviewProps` below is already
|
| 2069 |
+
* exported for the same reason, so this is the file's existing shape rather than a new one.
|
| 2070 |
+
* `_test/` never ships (`deploy_web.py` excludes `_test/` and `_`-prefixed files both).
|
| 2071 |
+
*/
|
| 2072 |
+
export function ActionProps({
|
| 2073 |
action,
|
| 2074 |
pinned,
|
| 2075 |
catalog,
|
|
|
|
| 2450 |
</div>
|
| 2451 |
) : null}
|
| 2452 |
|
| 2453 |
+
{/*
|
| 2454 |
+
ββ WAVE 28 Β· R5 / R6 / R7 β THE SOURCE QUESTION IS GONE, AND THREE INCLUDE AXES
|
| 2455 |
+
REPLACE IT. What stood here was a `Source` select writing `config.tier`
|
| 2456 |
+
(Anonymous / Paid provider) plus a sibling `noFallback` checkbox β ONE control in two
|
| 2457 |
+
pieces, asking the user to choose a VENDOR STRATEGY.
|
| 2458 |
+
|
| 2459 |
+
β R5 RETIRED THE QUESTION, not just the control. Enrichment routes per capability to
|
| 2460 |
+
the paid providers and reports blocked on a refusal; there is no thin anonymous row to
|
| 2461 |
+
fall back to, so "which source" and "stop if it fails" no longer have answers a person
|
| 2462 |
+
could give. `tier` and `noFallback` are accepted-and-ignored in stored configs (C2,
|
| 2463 |
+
and D-65's law: never 400 a definition that was legal when it was written) β which is
|
| 2464 |
+
why this panel simply stops writing them rather than migrating anything.
|
| 2465 |
+
|
| 2466 |
+
β THE ROWS BELOW ARE A TRANSFORM, NOT AN ADDITION, and that distinction is the whole
|
| 2467 |
+
defect risk in this change. `postMetrics` and `commentMetrics` ALREADY had checkboxes
|
| 2468 |
+
34 lines below this point ("Also capture per-post engagement" / "β¦per-commentβ¦").
|
| 2469 |
+
Building three NEW rows and leaving those would have put TWO controls on each key β
|
| 2470 |
+
which compiles, renders, and satisfies any check asking whether a Post-data switch
|
| 2471 |
+
exists. The old pair is deleted; these carry their keys.
|
| 2472 |
+
|
| 2473 |
+
β KEYS UNCHANGED ON PURPOSE (C2). Every stored enrich action round-trips untouched;
|
| 2474 |
+
only the labels move. R7: no cost sentence anywhere in this panel β the run log keeps
|
| 2475 |
+
honest spend reporting, and a warning printed permanently is chrome the eye stops
|
| 2476 |
+
reading (DESIGN.md Β§4).
|
| 2477 |
+
*/}
|
| 2478 |
+
{/* β A GROUP HEADING, NOT A `<label htmlFor>`. The first cut pointed it at the Post-data
|
| 2479 |
+
input, which is wrong twice over: it claims one row is "the" control for a group of
|
| 2480 |
+
three, and clicking the heading would toggle Posts. `auto-field-label` is this
|
| 2481 |
+
panel's own idiom for naming a group ("Records to enrich" above uses it). */}
|
| 2482 |
+
<p className="auto-field-label">Include</p>
|
| 2483 |
+
{/* β DISPLAY-ONLY, AND IT IS NOT DECORATION. The profile IS the unit of enrichment β
|
| 2484 |
+
there is no run that skips it β so a switch here would be a control that cannot be
|
| 2485 |
+
off ([[wrong-parent-not-broken-control]]). It carries NO config key: rendering it
|
| 2486 |
+
as state would invent a flag no cleaner reads.
|
| 2487 |
+
β `readOnly` beside `disabled`: a `checked` input with no `onChange` is a React
|
| 2488 |
+
warning, and `readOnly` says the honest thing about why. */}
|
| 2489 |
+
<label className="auto-check">
|
| 2490 |
+
<input id="autox-inc-profile" type="checkbox" checked readOnly disabled />
|
| 2491 |
+
Profile
|
| 2492 |
+
</label>
|
| 2493 |
+
<label className="auto-check">
|
| 2494 |
+
<input
|
| 2495 |
+
id="autox-inc-posts"
|
| 2496 |
+
type="checkbox"
|
| 2497 |
+
checked={!!(cfg as { postMetrics?: boolean }).postMetrics}
|
| 2498 |
disabled={busy}
|
| 2499 |
+
onChange={(e) => setCfg({ postMetrics: e.target.checked })}
|
| 2500 |
+
/>
|
| 2501 |
+
Post data
|
| 2502 |
+
</label>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2503 |
<label className="auto-check">
|
| 2504 |
<input
|
| 2505 |
+
id="autox-inc-comments"
|
| 2506 |
type="checkbox"
|
| 2507 |
+
checked={!!(cfg as { commentMetrics?: boolean }).commentMetrics}
|
| 2508 |
disabled={busy}
|
| 2509 |
+
onChange={(e) => setCfg({ commentMetrics: e.target.checked })}
|
| 2510 |
/>
|
| 2511 |
+
Comment data
|
| 2512 |
</label>
|
| 2513 |
{/*
|
| 2514 |
β WAVE 26 Β· ITEM 7 / R2 + C5 β TWO CONTROLS, AND THEY ARE INDEPENDENT.
|
|
|
|
| 2544 |
At most 12 β the provider returns a profile’s top 12 posts and no more.
|
| 2545 |
</p>
|
| 2546 |
</div>
|
| 2547 |
+
{/*
|
| 2548 |
+
β WAVE 28 Β· R6/R7 β THE TWO CHECKBOXES THAT STOOD HERE MOVED UP INTO THE INCLUDE
|
| 2549 |
+
GROUP, KEYS AND ALL (`postMetrics`, `commentMetrics`). They are not deleted features:
|
| 2550 |
+
they are the SAME two switches, relabelled "Post data" and "Comment data" and grouped
|
| 2551 |
+
with the always-on Profile row so the three capture axes read as one decision.
|
| 2552 |
+
β THEIR CONDITIONAL COST HINTS WENT WITH THEM AND DID NOT COME BACK (R7): "Billed per
|
| 2553 |
+
post, not per profile" and "Billed by the separate Comments dataset". R7 puts spend
|
| 2554 |
+
reporting in the RUN LOG, where it is a measured fact about work already done, rather
|
| 2555 |
+
than in the panel, where it was a permanent caption on a switch.
|
| 2556 |
+
β Leaving them here as well as above is the duplicate-writer trap this wave's item 1
|
| 2557 |
+
was one literal reading away from shipping β two controls on one key, both correct,
|
| 2558 |
+
neither authoritative.
|
| 2559 |
+
*/}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2560 |
<label className="auto-check">
|
| 2561 |
<input
|
| 2562 |
type="checkbox"
|
web/src/automation/AutomationDetail.tsx
CHANGED
|
@@ -1076,19 +1076,42 @@ export default function AutomationDetail({
|
|
| 1076 |
datacenter egress, "204K not 204,312", which rung answers when. The
|
| 1077 |
CHOICE a person makes is exact-or-estimated and what it costs; the rest
|
| 1078 |
was the architecture explaining itself. DESIGN.md Β§4. */}
|
| 1079 |
-
|
| 1080 |
-
|
| 1081 |
-
|
| 1082 |
-
|
| 1083 |
-
|
| 1084 |
-
|
| 1085 |
-
|
| 1086 |
-
|
| 1087 |
-
|
| 1088 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1089 |
<p className="auto-hint">
|
| 1090 |
-
|
| 1091 |
-
|
| 1092 |
</p>
|
| 1093 |
</>
|
| 1094 |
) : null}
|
|
|
|
| 1076 |
datacenter egress, "204K not 204,312", which rung answers when. The
|
| 1077 |
CHOICE a person makes is exact-or-estimated and what it costs; the rest
|
| 1078 |
was the architecture explaining itself. DESIGN.md Β§4. */}
|
| 1079 |
+
{/*
|
| 1080 |
+
ββ WAVE 28 Β· R5 + R7 β THREE HEADINGS DESCRIBED A LADDER THAT NO LONGER
|
| 1081 |
+
EXISTS. "Exact counts" / "Post engagement" / "Estimated counts" named the
|
| 1082 |
+
capture RUNGS, and R5 retired the free anonymous rung from enrichment
|
| 1083 |
+
entirely: exact numbers or nothing, a vendor refusal reports blocked. A
|
| 1084 |
+
panel offering to "switch it off to get exact numbers or nothing" describes
|
| 1085 |
+
a choice the runner stopped having β a picture of the engine that disagrees
|
| 1086 |
+
with the engine, which is the defect this module refuses everywhere else.
|
| 1087 |
+
β THE MONEY SENTENCES WENT WITH THEM (R7): "charged per profile" and
|
| 1088 |
+
"Charged per post, so it is its own switch". Spend is reported in the RUN
|
| 1089 |
+
LOG against work already done, never as a permanent caption on a control.
|
| 1090 |
+
|
| 1091 |
+
β `paidReady` SURVIVES, and R5 makes it MORE load-bearing rather than less.
|
| 1092 |
+
It is the one live reader of the flag, and with no free ladder underneath, a
|
| 1093 |
+
workspace with no provider connected gets nothing at all rather than a thin
|
| 1094 |
+
row β so the panel says so. That is a system fact, not a price.
|
| 1095 |
+
*/}
|
| 1096 |
+
{/*
|
| 1097 |
+
β THE SENTENCE MAKES NO CLAIM ABOUT WHAT THE SWITCHES DO, and the first cut
|
| 1098 |
+
did: it read "post and comment data follow the switches above", which is
|
| 1099 |
+
NOT what `postMetrics` gates. Measured against the engine β
|
| 1100 |
+
`automation_engine.py:5358` ("likes/comments per post are bought, or the
|
| 1101 |
+
engagement series does not grow") and `connectors_ig.py:710` ("a separate,
|
| 1102 |
+
opt-in, ~13x-cost rung rather than a free by-product") β the profile pull
|
| 1103 |
+
keeps up to `maxPosts` posts REGARDLESS of that switch; what the switch buys
|
| 1104 |
+
is per-post ENGAGEMENT. R6 rules the LABEL ("Post data"), and the label is
|
| 1105 |
+
the owner's to set; a caption asserting a mechanism is mine, and this one
|
| 1106 |
+
would have been a picture of the engine that disagrees with the engine β
|
| 1107 |
+
the defect this module refuses in five other places.
|
| 1108 |
+
β ASK ->B is open on whether R5's reshape changes that. Until it is answered
|
| 1109 |
+
the honest panel says the one thing that is true either way.
|
| 1110 |
+
*/}
|
| 1111 |
+
<h3>Included</h3>
|
| 1112 |
<p className="auto-hint">
|
| 1113 |
+
The profile is always read.
|
| 1114 |
+
{paidReady ? "" : " No provider is connected on this workspace yet."}
|
| 1115 |
</p>
|
| 1116 |
</>
|
| 1117 |
) : null}
|
web/src/automation/automationApi.ts
CHANGED
|
@@ -31,12 +31,17 @@ export type AutomationKind =
|
|
| 31 |
| "discover_instagram"
|
| 32 |
| "plain";
|
| 33 |
export type RunState = "idle" | "running" | "ok" | "error" | "partial";
|
| 34 |
-
/*
|
| 35 |
-
*
|
| 36 |
-
*
|
| 37 |
-
*
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 38 |
*/
|
| 39 |
-
export type CaptureTier = "anonymous" | "brightdata";
|
| 40 |
|
| 41 |
/**
|
| 42 |
* A node's dot. Deliberately WIDER than `RunState` β a single step can be
|
|
@@ -1012,10 +1017,21 @@ export const COUNT_LABELS: [string, string][] = [
|
|
| 1012 |
// (profile read, media not anonymously readable) but this table did not, so a run whose
|
| 1013 |
// summary said "1/1 profiles read" showed a chip row reading "0 Read" β the chips
|
| 1014 |
// contradicting the sentence directly above them. A count the summary names must have a chip.
|
|
|
|
|
|
|
|
|
|
| 1015 |
["partial", "Profile only"],
|
| 1016 |
["blocked", "Blocked"],
|
| 1017 |
["error", "Errors"],
|
| 1018 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1019 |
["posts", "Posts"],
|
| 1020 |
["new_posts", "New posts"],
|
| 1021 |
// The engagement series only grows while the per-post rung is on, so its
|
|
|
|
| 31 |
| "discover_instagram"
|
| 32 |
| "plain";
|
| 33 |
export type RunState = "idle" | "running" | "ok" | "error" | "partial";
|
| 34 |
+
/*
|
| 35 |
+
* β WAVE 28 Β· R5 β `CaptureTier` ("anonymous" | "brightdata") IS DELETED. It was the client's
|
| 36 |
+
* name for the capture-rung choice, and R5 retired that choice: enrichment routes per capability
|
| 37 |
+
* to the paid providers and reports blocked on a refusal, so there is no tier for a client to
|
| 38 |
+
* hold. MEASURED before deleting: the type had ZERO usages anywhere under `src/` β it was already
|
| 39 |
+
* a declaration nothing read, which is what a retired concept looks like on the way out.
|
| 40 |
+
* β THE SERVER'S OWN VOCABULARY IS UNTOUCHED, deliberately. `TIERS`, `TIER_ALIASES` and the
|
| 41 |
+
* `brightdata` vendor name stay in the engine (`verify_automation.py:6510` fences them), because
|
| 42 |
+
* a stored config still carries `tier` and is accepted-and-ignored rather than refused (C2,
|
| 43 |
+
* D-65's law). What died is the USER CONCEPT, not the wire's tolerance for it.
|
| 44 |
*/
|
|
|
|
| 45 |
|
| 46 |
/**
|
| 47 |
* A node's dot. Deliberately WIDER than `RunState` β a single step can be
|
|
|
|
| 1017 |
// (profile read, media not anonymously readable) but this table did not, so a run whose
|
| 1018 |
// summary said "1/1 profiles read" showed a chip row reading "0 Read" β the chips
|
| 1019 |
// contradicting the sentence directly above them. A count the summary names must have a chip.
|
| 1020 |
+
// β WAVE 28 Β· R5 β KEPT, and the label is unchanged on purpose. "Profile only" describes WHAT
|
| 1021 |
+
// WAS CAPTURED (the profile answered, its media did not), which is still a state a single paid
|
| 1022 |
+
// pipeline reaches; it was never tier vocabulary the way "Exact counts" was.
|
| 1023 |
["partial", "Profile only"],
|
| 1024 |
["blocked", "Blocked"],
|
| 1025 |
["error", "Errors"],
|
| 1026 |
+
// β WAVE 28 Β· R5/R7 β WAS "Exact counts", WHICH NOW DISTINGUISHES NOTHING. The word only meant
|
| 1027 |
+
// anything against the free rung's "estimated" counts, and R5 retired that rung: every number a
|
| 1028 |
+
// run reports is exact, so a chip claiming exactness is a label with no complement.
|
| 1029 |
+
// β THE CHIP ITSELF STAYS, and R7 is why: cost prose leaves the PANEL and spend reporting lives
|
| 1030 |
+
// in the RUN LOG, against work already done. This is that reporting β the count of profiles the
|
| 1031 |
+
// run actually bought. Deleting it would take the honest half out with the prose.
|
| 1032 |
+
// β SAFE IN BOTH DIRECTIONS if B's R5 work stops emitting the count: the renderer filters on
|
| 1033 |
+
// `typeof r.counts?.[k] === "number"`, so an absent count paints no chip rather than a zero.
|
| 1034 |
+
["paid", "Paid profiles"],
|
| 1035 |
["posts", "Posts"],
|
| 1036 |
["new_posts", "New posts"],
|
| 1037 |
// The engagement series only grows while the per-post rung is on, so its
|
web/src/automation/steps.ts
CHANGED
|
@@ -85,12 +85,15 @@ export interface Step {
|
|
| 85 |
/**
|
| 86 |
* Number the server's nodes for display.
|
| 87 |
*
|
| 88 |
-
* β THE NUMBER IS `col + 1`, NOT the array index, and that is the honest one.
|
| 89 |
-
*
|
| 90 |
-
*
|
| 91 |
-
*
|
| 92 |
-
*
|
| 93 |
-
*
|
|
|
|
|
|
|
|
|
|
| 94 |
*
|
| 95 |
* "Step 1 is always the Trigger" (R9) therefore falls out of the payload β the
|
| 96 |
* trigger is the node at `col: 0` β instead of being asserted by this client. If
|
|
@@ -150,10 +153,14 @@ export interface PanelGroup {
|
|
| 150 |
*
|
| 151 |
* β GROUPED, NEVER MAPPED ONE-TO-ONE, and the difference is a defect rather than a nicety.
|
| 152 |
* `panel` is MANY-TO-ONE over nodes, which is easy to miss because the old surface hid it: you
|
| 153 |
-
* clicked ONE card and got ONE panel. `
|
| 154 |
-
* `
|
| 155 |
-
*
|
| 156 |
-
*
|
|
|
|
|
|
|
|
|
|
|
|
|
| 157 |
* loses work β mount the 21-toggle discovery filter TWICE against a single `preds` array, two
|
| 158 |
* editors writing one piece of state where whichever blurred last silently wins.
|
| 159 |
*
|
|
|
|
| 85 |
/**
|
| 86 |
* Number the server's nodes for display.
|
| 87 |
*
|
| 88 |
+
* β THE NUMBER IS `col + 1`, NOT the array index, and that is the honest one. It exists
|
| 89 |
+
* because a graph can FORK: two nodes at the same `col` are the SAME position in the flow
|
| 90 |
+
* reached two ways, and numbering them 4 and 5 would state a sequence that never happens.
|
| 91 |
+
* They share a number and the second is marked `alt`, so the list reads "either of these"
|
| 92 |
+
* rather than "then".
|
| 93 |
+
* β 2026-08-09 (wave 28, R5): the fork this was WRITTEN for is gone β `capture_paid` and
|
| 94 |
+
* `capture_anon` were the paid-rung/anonymous-ladder pair, and the ladder is retired, so
|
| 95 |
+
* today's graph is linear. The `col`-based rule STAYS because it is about forks in general,
|
| 96 |
+
* not about those two nodes; this note records that no shipped graph currently exercises it.
|
| 97 |
*
|
| 98 |
* "Step 1 is always the Trigger" (R9) therefore falls out of the payload β the
|
| 99 |
* trigger is the node at `col: 0` β instead of being asserted by this client. If
|
|
|
|
| 153 |
*
|
| 154 |
* β GROUPED, NEVER MAPPED ONE-TO-ONE, and the difference is a defect rather than a nicety.
|
| 155 |
* `panel` is MANY-TO-ONE over nodes, which is easy to miss because the old surface hid it: you
|
| 156 |
+
* clicked ONE card and got ONE panel. `discover_instagram` gives both of its nodes
|
| 157 |
+
* `panel: "find"`, and `field_instagram` gives every capture node `panel: "capture"`.
|
| 158 |
+
* β 2026-08-09 (wave 28, R5/C3): this used to name `capture`, `capture_paid`, `capture_anon`
|
| 159 |
+
* AND `capture_metrics` with a line citation β all four ids are DELETED and the citation was
|
| 160 |
+
* stale, which is worse than vague, because a stale line number reads as authoritative. The
|
| 161 |
+
* grouping rule is unchanged: the capture nodes are now `capture_posts` + `capture_comments`.
|
| 162 |
+
* So a body rendered per NODE would print the capture prose once per node, and β the one that
|
| 163 |
+
* actually
|
| 164 |
* loses work β mount the 21-toggle discovery filter TWICE against a single `preds` array, two
|
| 165 |
* editors writing one piece of state where whichever blurred last silently wins.
|
| 166 |
*
|
web/src/customer-grid/ColumnMenu.tsx
CHANGED
|
@@ -5,9 +5,10 @@ import { FieldTypeIcon, MenuLabel } from "./icons";
|
|
| 5 |
import { FieldSelectButton } from "./FieldSelect";
|
| 6 |
import { CODE_LANGUAGE_LABELS, CODE_LANGUAGES, CREATABLE_TYPES, choiceOptions, choiceRenames,
|
| 7 |
codeLanguageOf, directionLabel, isMachineOwned,
|
| 8 |
-
isProfileField, parseOptions, ratingMax, ROLLUP_FN_LABELS, ROLLUP_FNS
|
|
|
|
| 9 |
import type { Field, FieldFormat, FieldScope, FieldType, Measure, RollupCondition, RollupFn,
|
| 10 |
-
RollupSource, Viewer } from "./types";
|
| 11 |
import type { LinkTarget, RollupSourceOffer } from "./apiBridge";
|
| 12 |
import type { WindowSpec } from "./windows";
|
| 13 |
import { normalizeWindow, windowLabel } from "./windows";
|
|
@@ -279,6 +280,41 @@ interface OptionDraft {
|
|
| 279 |
}
|
| 280 |
|
| 281 |
const DEFAULT_OPTION_LABELS = ["Not started", "In progress", "Blocked", "Done"];
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 282 |
let optionDraftSequence = 0;
|
| 283 |
|
| 284 |
function nextOptionDraftId(): string {
|
|
@@ -1234,6 +1270,37 @@ function ExtraTypeEditor({
|
|
| 1234 |
</select>
|
| 1235 |
</label>
|
| 1236 |
) : null}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1237 |
<div className="cg-rollup-conditions">
|
| 1238 |
<div className="cg-rollup-conditions__head">
|
| 1239 |
<span>Conditions</span>
|
|
@@ -1276,34 +1343,117 @@ function ExtraTypeEditor({
|
|
| 1276 |
<option key={targetField.key} value={targetField.key}>{targetField.label}</option>
|
| 1277 |
))}
|
| 1278 |
</select>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1279 |
<select
|
| 1280 |
className="cg-input"
|
| 1281 |
aria-label={`Condition ${index + 1} operator`}
|
| 1282 |
-
value={condition.op}
|
| 1283 |
-
onChange={(event) =>
|
| 1284 |
-
|
| 1285 |
-
|
| 1286 |
-
|
| 1287 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1288 |
>
|
| 1289 |
-
<
|
| 1290 |
-
|
| 1291 |
-
|
| 1292 |
-
|
| 1293 |
-
|
| 1294 |
-
|
| 1295 |
-
|
| 1296 |
-
|
| 1297 |
-
|
| 1298 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1299 |
</select>
|
| 1300 |
-
{
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1301 |
<input
|
| 1302 |
className="cg-input"
|
| 1303 |
aria-label={`Condition ${index + 1} value`}
|
| 1304 |
value={condition.value ?? ""}
|
| 1305 |
onChange={(event) => onRollupConditions?.(rollupConditions.map((item, i) =>
|
| 1306 |
-
i === index ? { .
|
| 1307 |
))}
|
| 1308 |
/>
|
| 1309 |
) : null}
|
|
|
|
| 5 |
import { FieldSelectButton } from "./FieldSelect";
|
| 6 |
import { CODE_LANGUAGE_LABELS, CODE_LANGUAGES, CREATABLE_TYPES, choiceOptions, choiceRenames,
|
| 7 |
codeLanguageOf, directionLabel, isMachineOwned,
|
| 8 |
+
isProfileField, parseOptions, ratingMax, ROLLUP_FN_LABELS, ROLLUP_FNS,
|
| 9 |
+
ROLLUP_REF_OPS } from "./types";
|
| 10 |
import type { Field, FieldFormat, FieldScope, FieldType, Measure, RollupCondition, RollupFn,
|
| 11 |
+
RollupRefOp, RollupSource, Viewer } from "./types";
|
| 12 |
import type { LinkTarget, RollupSourceOffer } from "./apiBridge";
|
| 13 |
import type { WindowSpec } from "./windows";
|
| 14 |
import { normalizeWindow, windowLabel } from "./windows";
|
|
|
|
| 280 |
}
|
| 281 |
|
| 282 |
const DEFAULT_OPTION_LABELS = ["Not started", "In progress", "Blocked", "Done"];
|
| 283 |
+
|
| 284 |
+
/**
|
| 285 |
+
* β WAVE 28 Β· R4 / C1 β how a SET-STATISTIC comparison reads in the operator list.
|
| 286 |
+
*
|
| 287 |
+
* β KEYED BY `RollupRefOp`, so the day `core.user_tables.ROLLUP_REF_OPS` gains or loses an
|
| 288 |
+
* operator this map fails to compile rather than rendering a bare identifier β the same
|
| 289 |
+
* discipline `ROLLUP_FN_LABELS` uses, and the reason the Ο picker maps `ROLLUP_REF_OPS` instead
|
| 290 |
+
* of hand-filtering the full operator list.
|
| 291 |
+
* β The wording names the STATISTIC, not the arithmetic: "2 Ο above the mean" is the sentence a
|
| 292 |
+
* person is trying to write, and "is greater than mean + kΒ·stdev" is the implementation of it.
|
| 293 |
+
*/
|
| 294 |
+
/**
|
| 295 |
+
* β MIRRORS `core.user_tables.ROLLUP_MAX_SIGMAS`, and `verify_rollup_editor.py` HOLDS THE TWO IN
|
| 296 |
+
* STEP by importing the server's constant and comparing (a real enforcer this time β `types.ts`
|
| 297 |
+
* spent a wave claiming one that contained zero occurrences of the word it guarded,
|
| 298 |
+
* [[limit-with-no-enforcer]]).
|
| 299 |
+
*
|
| 300 |
+
* β WHY THE CONTROL MUST KNOW THE BOUND AT ALL. `_clean_rollup` REFUSES the whole field outside
|
| 301 |
+
* this range β not the leaf, the FIELD β so an unbounded box invites a value and then answers a
|
| 302 |
+
* 400 naming a cap the control never mentioned. That is `maxPosts` exactly: it offered 200 while
|
| 303 |
+
* the save door refused anything over 12 ([[default-must-pass-its-own-guard]]). A control must
|
| 304 |
+
* display what it will SEND.
|
| 305 |
+
*/
|
| 306 |
+
const MAX_SIGMAS = 10;
|
| 307 |
+
|
| 308 |
+
/** The default when a Ο row is born or its box is left empty β C1's worked example uses 2. */
|
| 309 |
+
const DEFAULT_SIGMAS = 2;
|
| 310 |
+
|
| 311 |
+
const REF_OP_LABELS: Record<RollupRefOp, string> = {
|
| 312 |
+
gt: "is more than β¦ Ο above the mean",
|
| 313 |
+
gte: "is at least β¦ Ο above the mean",
|
| 314 |
+
lt: "is less than β¦ Ο above the mean",
|
| 315 |
+
lte: "is at most β¦ Ο above the mean",
|
| 316 |
+
};
|
| 317 |
+
|
| 318 |
let optionDraftSequence = 0;
|
| 319 |
|
| 320 |
function nextOptionDraftId(): string {
|
|
|
|
| 1270 |
</select>
|
| 1271 |
</label>
|
| 1272 |
) : null}
|
| 1273 |
+
{/*
|
| 1274 |
+
ββ 2026-08-09 Β· D-99 β THE STATE THE OWNER'S REPORT DESCRIBES, SAID OUT LOUD.
|
| 1275 |
+
Owner: *"if we want to use Rollup field to apply to the related database, it doesn't
|
| 1276 |
+
work. Because the linked Field can't be selected yet."*
|
| 1277 |
+
|
| 1278 |
+
β WHAT IS ACTUALLY REACHABLE HERE, measured rather than guessed. The `Through` picker
|
| 1279 |
+
offers EVERY link column β `linkFields` filters on `f.type === "link"` and excludes
|
| 1280 |
+
nothing preset, derived or `automation:`-tagged, so the scouted hypothesis ("the picker
|
| 1281 |
+
omits preset links") is refuted at source. What CAN happen is one step later: the
|
| 1282 |
+
target is resolved out of `linkTargets`, which is `GET /tables` filtered by
|
| 1283 |
+
`user_tables.may_open` (creator, admin, or an explicit share). An automation-OWNED child
|
| 1284 |
+
dataset is stamped with a machine owner, so for a user who is neither its creator nor an
|
| 1285 |
+
admin it is simply ABSENT from that list β and then `targetFields` is `[]`, this Column
|
| 1286 |
+
picker holds nothing but its placeholder, and `Add condition` below is disabled.
|
| 1287 |
+
|
| 1288 |
+
β AND UNTIL NOW IT SAID NOTHING. An empty picker beside a dead button reads as "this
|
| 1289 |
+
feature is broken"; it is in fact a permission fact about ONE database, and the fix for
|
| 1290 |
+
it is a share, not a bug report. This is the absent-vs-empty third state the automation
|
| 1291 |
+
panel already distinguishes ("No view list was offered for that database") β the same
|
| 1292 |
+
discipline, applied to the surface that actually got reported.
|
| 1293 |
+
β IT NAMES THE TABLE RATHER THAN APOLOGISING. `chosenLink.link.table` is the key the
|
| 1294 |
+
bag points at, and printing it is what turns "it doesn't work" into a thing to go and
|
| 1295 |
+
grant.
|
| 1296 |
+
*/}
|
| 1297 |
+
{chosenLink && !target ? (
|
| 1298 |
+
<div className="cg-field-hint">
|
| 1299 |
+
This link points at <b>{chosenLink.link?.table}</b>, which is not among the databases
|
| 1300 |
+
you can open β so its columns cannot be listed here and the conditions below stay
|
| 1301 |
+
unavailable. Ask an admin to share that database with you, then reopen this editor.
|
| 1302 |
+
</div>
|
| 1303 |
+
) : null}
|
| 1304 |
<div className="cg-rollup-conditions">
|
| 1305 |
<div className="cg-rollup-conditions__head">
|
| 1306 |
<span>Conditions</span>
|
|
|
|
| 1343 |
<option key={targetField.key} value={targetField.key}>{targetField.label}</option>
|
| 1344 |
))}
|
| 1345 |
</select>
|
| 1346 |
+
{/*
|
| 1347 |
+
ββ WAVE 28 Β· R4 / C1 β THE THRESHOLD MAY NOW BE THE SET'S OWN STATISTICS.
|
| 1348 |
+
Owner's request (Nurilab): an average of the last 10 posts with outliers trimmed β
|
| 1349 |
+
"anything beyond 2 sigma". That is not a new aggregator, it is a CONDITION whose
|
| 1350 |
+
threshold is computed from the scoped rows themselves: mean + k*stdev of the same
|
| 1351 |
+
column, over the window `sortBy`+`limit` kept, BEFORE the conditions filter.
|
| 1352 |
+
|
| 1353 |
+
β ONE CONTROL, NOT TWO, AND THAT IS WHAT MAKES THE XOR SAFE. `value` and `ref` may
|
| 1354 |
+
never both be present β the server refuses the whole FIELD, not just the leaf β so
|
| 1355 |
+
a separate "compare againstβ¦" switch beside the operator would leave the editor
|
| 1356 |
+
free to write a `ref` while a stale `value` still rode along, producing a 400 on
|
| 1357 |
+
Save that names a key the user never typed. Folding the choice into the operator
|
| 1358 |
+
makes the illegal state unrepresentable: picking a plain op WRITES `value` and
|
| 1359 |
+
drops `ref`, picking a statistic op does the reverse, and there is no third path.
|
| 1360 |
+
|
| 1361 |
+
β THE OPS ARE `ROLLUP_REF_OPS`, IMPORTED, never a hand-filtered copy of the full
|
| 1362 |
+
list. `eq`/`neq` against a computed float is a coin flip on binary representation
|
| 1363 |
+
and `contains` against a number never matches β the server draws that line and a
|
| 1364 |
+
second copy here would be free to drift from it.
|
| 1365 |
+
β THE `value` IS COMPOUND (`ref:gt`) so the select always resolves to exactly one
|
| 1366 |
+
option. A `<select>` whose value matches nothing renders the FIRST option while
|
| 1367 |
+
the stored key says otherwise, and the next patch writes the lie back
|
| 1368 |
+
([[cg-condition-builder-items]]).
|
| 1369 |
+
*/}
|
| 1370 |
<select
|
| 1371 |
className="cg-input"
|
| 1372 |
aria-label={`Condition ${index + 1} operator`}
|
| 1373 |
+
value={condition.ref ? `ref:${condition.op}` : condition.op}
|
| 1374 |
+
onChange={(event) => {
|
| 1375 |
+
const raw = event.target.value;
|
| 1376 |
+
const isRef = raw.startsWith("ref:");
|
| 1377 |
+
const op = (isRef ? raw.slice(4) : raw) as RollupCondition["op"];
|
| 1378 |
+
// β THE LEAF IS REBUILT, NEVER SPREAD-AND-PATCHED. `{ ...item, ref: undefined }`
|
| 1379 |
+
// leaves the key present-and-undefined; `JSON.stringify` drops it today, but the
|
| 1380 |
+
// contract is "the key is absent", and relying on a serialiser's treatment of
|
| 1381 |
+
// `undefined` to enforce a server-side XOR is a guarantee held by accident.
|
| 1382 |
+
const next: RollupCondition = isRef
|
| 1383 |
+
? { field: condition.field, op,
|
| 1384 |
+
ref: { sigmas: condition.ref?.sigmas ?? DEFAULT_SIGMAS } }
|
| 1385 |
+
: { field: condition.field, op, value: condition.value ?? "" };
|
| 1386 |
+
onRollupConditions?.(rollupConditions.map((item, i) => (i === index ? next : item)));
|
| 1387 |
+
}}
|
| 1388 |
>
|
| 1389 |
+
<optgroup label="Compared with a value">
|
| 1390 |
+
<option value="eq">is</option>
|
| 1391 |
+
<option value="neq">is not</option>
|
| 1392 |
+
<option value="contains">contains</option>
|
| 1393 |
+
<option value="not_contains">does not contain</option>
|
| 1394 |
+
<option value="is_empty">is empty</option>
|
| 1395 |
+
<option value="is_not_empty">is not empty</option>
|
| 1396 |
+
<option value="gt">is greater than</option>
|
| 1397 |
+
<option value="gte">is at least</option>
|
| 1398 |
+
<option value="lt">is less than</option>
|
| 1399 |
+
<option value="lte">is at most</option>
|
| 1400 |
+
</optgroup>
|
| 1401 |
+
<optgroup label="Compared with the set's own spread">
|
| 1402 |
+
{ROLLUP_REF_OPS.map((op) => (
|
| 1403 |
+
<option key={op} value={`ref:${op}`}>{REF_OP_LABELS[op]}</option>
|
| 1404 |
+
))}
|
| 1405 |
+
</optgroup>
|
| 1406 |
</select>
|
| 1407 |
+
{condition.ref ? (
|
| 1408 |
+
<label className="cg-cond-ref">
|
| 1409 |
+
{/* β A BARE NUMBER BOX WOULD BE A CONTROL THAT WILL NOT SAY WHAT IT SETS. The
|
| 1410 |
+
unit is sigmas and the sign is meaningful (negative selects the LOW tail), so
|
| 1411 |
+
both are printed beside the input rather than left to be inferred from a
|
| 1412 |
+
placeholder. */}
|
| 1413 |
+
<input
|
| 1414 |
+
className="cg-input"
|
| 1415 |
+
type="number"
|
| 1416 |
+
step="0.5"
|
| 1417 |
+
/* β THE SERVER'S OWN BOUND, BOTH WAYS. Outside it `_clean_rollup` refuses the
|
| 1418 |
+
whole FIELD, so an unbounded box would invite a 400 naming a cap it never
|
| 1419 |
+
showed β `maxPosts`'s scar, in a control built the same day it was quoted. */
|
| 1420 |
+
min={-MAX_SIGMAS}
|
| 1421 |
+
max={MAX_SIGMAS}
|
| 1422 |
+
aria-label={`Condition ${index + 1} standard deviations from the mean`}
|
| 1423 |
+
/* β UNCONTROLLED + COMMIT ON BLUR, the idiom `maxPosts` uses and for its
|
| 1424 |
+
reason: a controlled box patching per keystroke stores "-" as 0 and "1" on
|
| 1425 |
+
the way to "1.5" as a legal value, and `Number("")` is 0 β so simply
|
| 1426 |
+
CLEARING the box would silently store "0 sigma", a real but different
|
| 1427 |
+
query, and the field could never be retyped because it snaps back. */
|
| 1428 |
+
defaultValue={String(condition.ref.sigmas)}
|
| 1429 |
+
onBlur={(event) => {
|
| 1430 |
+
const raw = Number(event.target.value);
|
| 1431 |
+
// Empty/garbage falls back to the default rather than to 0 β 0 means "at
|
| 1432 |
+
// the mean" and is a value somebody must choose, never one they land on by
|
| 1433 |
+
// deleting a character.
|
| 1434 |
+
const parsed = event.target.value.trim() === "" || !Number.isFinite(raw)
|
| 1435 |
+
? DEFAULT_SIGMAS
|
| 1436 |
+
: raw;
|
| 1437 |
+
// CLAMPED, not refused: the box is a dial, and a dial that rejects is a
|
| 1438 |
+
// dead control. The clamp is what makes "displays what it will send" true.
|
| 1439 |
+
const sigmas = Math.max(-MAX_SIGMAS, Math.min(MAX_SIGMAS, parsed));
|
| 1440 |
+
if (String(sigmas) !== event.target.value) event.target.value = String(sigmas);
|
| 1441 |
+
onRollupConditions?.(rollupConditions.map((item, i) => (
|
| 1442 |
+
i === index
|
| 1443 |
+
? { field: item.field, op: item.op, ref: { sigmas } }
|
| 1444 |
+
: item
|
| 1445 |
+
)));
|
| 1446 |
+
}}
|
| 1447 |
+
/>
|
| 1448 |
+
<span>Ο from the mean of this column (negative reads below it)</span>
|
| 1449 |
+
</label>
|
| 1450 |
+
) : !(["is_empty", "is_not_empty"] as string[]).includes(condition.op) ? (
|
| 1451 |
<input
|
| 1452 |
className="cg-input"
|
| 1453 |
aria-label={`Condition ${index + 1} value`}
|
| 1454 |
value={condition.value ?? ""}
|
| 1455 |
onChange={(event) => onRollupConditions?.(rollupConditions.map((item, i) =>
|
| 1456 |
+
i === index ? { field: item.field, op: item.op, value: event.target.value } : item
|
| 1457 |
))}
|
| 1458 |
/>
|
| 1459 |
) : null}
|
web/src/index.css
CHANGED
|
@@ -1478,6 +1478,29 @@ body {
|
|
| 1478 |
width: 30px;
|
| 1479 |
height: 30px;
|
| 1480 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1481 |
/* why a control is unavailable, in the place the control would have been. An empty popover
|
| 1482 |
reads as broken; a sentence reads as a decision (CG-3). */
|
| 1483 |
.cg-pop-note {
|
|
|
|
| 1478 |
width: 30px;
|
| 1479 |
height: 30px;
|
| 1480 |
}
|
| 1481 |
+
|
| 1482 |
+
/* ββ WAVE 28 Β· R4 / C1 (session C) β the set-statistic threshold's input ββββββββββββββββββββββ
|
| 1483 |
+
A condition row is a 3-column grid (field, operator, remove) and the ordinary value input
|
| 1484 |
+
already spans the first two on a second line. The sigma input needs the same span PLUS its
|
| 1485 |
+
unit beside it: the number is meaningless alone, and the sign is load-bearing (negative reads
|
| 1486 |
+
BELOW the mean), so a bare box would be a control that will not say what it sets
|
| 1487 |
+
([[wrong-parent-not-broken-control]]).
|
| 1488 |
+
β `grid-column` on the LABEL, not on its input: the input is now a grandchild of the row, so
|
| 1489 |
+
the `.cg-rollup-condition > input` rule above no longer reaches it β which is the trap that
|
| 1490 |
+
turns an extra child into an extra column. */
|
| 1491 |
+
.cg-rollup-condition > .cg-cond-ref {
|
| 1492 |
+
grid-column: 1 / span 2;
|
| 1493 |
+
display: grid;
|
| 1494 |
+
grid-template-columns: 84px minmax(0, 1fr);
|
| 1495 |
+
align-items: center;
|
| 1496 |
+
gap: 8px;
|
| 1497 |
+
}
|
| 1498 |
+
|
| 1499 |
+
.cg-rollup-condition > .cg-cond-ref > span {
|
| 1500 |
+
color: var(--cg-muted);
|
| 1501 |
+
font-size: var(--lp-fs-2xs);
|
| 1502 |
+
line-height: 1.35;
|
| 1503 |
+
}
|
| 1504 |
/* why a control is unavailable, in the place the control would have been. An empty popover
|
| 1505 |
reads as broken; a sentence reads as a decision (CG-3). */
|
| 1506 |
.cg-pop-note {
|