Deploy AIOS web (React glide grid + FastAPI slice)

RELEASES.json

@@ -1,5 +1,5 @@
 {
- "current": "43c0c66",
+ "current": "b859a84",
  "releases": [
   {
    "version": "v14",

VERSION

@@ -1 +1 @@
-43c0c66
+b859a84

api/automation_engine.py

@@ -2695,6 +2695,76 @@ def bd_scrape(dataset_id, urls, wait=None):
                 f"or collect {sid} from the control panel")
 
 
+#: ⭐⭐ 2026-08-07 — HOW LONG THE CORPUS RUNG MAY WAIT. A corpus query is a scan over 620M rows
+#: and takes MINUTES, not seconds (§2d MEASURED ~4 min for a narrow one). It is a FALLBACK behind
+#: a scrape that already failed, so the budget is generous — but bounded, because an enrich step
+#: that never returns is an automation that never finishes.
+BD_CORPUS_WAIT = float(os.environ.get("AIOS_BD_CORPUS_WAIT") or 420)
+BD_CORPUS_POLL = float(os.environ.get("AIOS_BD_CORPUS_POLL") or 15)
+
+
+def bd_corpus_profile(handle, log=print):
+    """One profile out of the PRE-COLLECTED corpus. `(node|None, note)`.
+
+    ⭐⭐ WHY THIS RUNG EXISTS, and it is measured rather than defensive. On 2026-08-07 the SCRAPE
+    path answered `Parse error: Invalid URL` for `inayma` on **four consecutive attempts and
+    through both input shapes** (a URL with and without its trailing slash, and the `user_name`
+    discovery input, which the vendor converts to the same URL internally) — while a control
+    profile in the SAME batch came back with 36 keys. So the URL form was not the problem and
+    neither was the account: the anonymous ladder read it fine. The vendor simply could not
+    scrape that profile.
+    ⇒ **The same dataset answered instantly by the OTHER route.** `POST /datasets/filter` with
+    `account = inayma` returned the full 33-key row — exact followers (203,818 against the
+    anonymous rung's rounded 204,000), biography, `is_verified`, `avg_engagement`, `fbid`,
+    `highlights_count`. One dataset, two routes, and only one of them was broken.
+
+    ⚠ THE ROWS ARE PRE-COLLECTED, so this is not a live read — it is the vendor's own last
+    capture, and its population differs from the scrape path's (§2d: `avg_engagement` 88%
+    populated on the corpus vs 0% on scrape). That is why it is a FALLBACK and not the default:
+    fresher-but-thinner beats staler-but-fuller when both work.
+    ⛔ THE SNAPSHOT NAMESPACES ARE NOT INTERCHANGEABLE. A corpus query mints `snap_…` and is read
+    at `/datasets/snapshot/{id}`; the scraper mints `sd_…` and is read at
+    `/datasets/v3/snapshot/{id}`. Crossing them returns a flat 404 about a snapshot that is alive
+    and building — the trap §2c records, and the reason these paths are spelled out here.
+    """
+    h = str(handle or "").strip().lstrip("@").lower()
+    if not h:
+        return None, "no handle to look up"
+    payload, note = bd_call(BD_PATH_FILTER, None,
+                            {"dataset_id": BD_DS_PROFILES, "records_limit": 1,
+                             "filter": {"name": "account", "operator": "=", "value": h}})
+    if note:
+        return None, note
+    snap = str((payload or {}).get("snapshot_id") or "").strip()
+    if not snap:
+        return None, "the profile corpus accepted the query but named no snapshot"
+    deadline = time.time() + BD_CORPUS_WAIT
+    while time.time() < deadline:
+        time.sleep(BD_CORPUS_POLL)
+        got, perr = bd_call(f"{BD_PATH_FILTER_SNAPSHOT}/{snap}", None, None)
+        status = str((got or {}).get("status") or "").strip()
+        if perr or status in ("failed", "error"):
+            return None, (perr or f"the corpus query failed ({(got or {}).get('error') or status})")
+        if status == "ready":
+            # ⚠ READY IS NOT DELIVERABLE. §2d MEASURED `/download` answering "Snapshot is
+            # building. Try again in a few minutes" while status already said `ready` — delivery
+            # lags readiness. So the download is polled too, inside the same budget.
+            while time.time() < deadline:
+                rows, derr = bd_call(f"{BD_PATH_FILTER_SNAPSHOT}/{snap}/download",
+                                     {"format": "json"}, None)
+                if derr:
+                    time.sleep(BD_CORPUS_POLL)
+                    continue
+                if isinstance(rows, dict):
+                    rows = [rows]
+                if isinstance(rows, list) and rows:
+                    return rows[0], ""
+                time.sleep(BD_CORPUS_POLL)
+            break
+    return None, (f"the profile corpus did not deliver {h!r} within "
+                  f"{int(BD_CORPUS_WAIT)}s (snapshot {snap} is still building — it is not lost)")
+
+
 def pull_profile_bd(url, max_posts=DEFAULT_POSTS_PER_PULL, post_metrics=False, log=print):
     """The paid rung: one profile, EXACT counts, plus the top posts the anonymous surface hides.
 
@@ -2711,11 +2781,41 @@ def pull_profile_bd(url, max_posts=DEFAULT_POSTS_PER_PULL, post_metrics=False, l
         return {"state": "error", "note": f"{url!r} is not an Instagram profile URL",
                 "profile": {}, "posts": [], "via": ""}
     rows, note = bd_scrape(BD_DS_PROFILES, [f"https://www.instagram.com/{handle}/"])
-    if note:
-        return {"state": "blocked", "profile": {}, "posts": [], "via": "brightdata",
-                "note": note}
     node = rows[0] if rows else {}
-    profile = _bd_profile(node, handle)
+    profile = _bd_profile(node, handle) if node else {}
+    unreadable = profile.get("followers") is None and profile.get("following") is None
+    via = "brightdata"
+    if note or unreadable:
+        # ⭐⭐ 2026-08-07 — THE CORPUS IS TRIED BEFORE THIS RUNG GIVES UP, and it is the difference
+        # between a blank row and a full one. MEASURED: the scraper answered `Parse error: Invalid
+        # URL` for `inayma` four times running and through both input shapes, while a control
+        # profile in the same batch returned 36 keys — and `POST /datasets/filter` on the SAME
+        # dataset returned that profile's complete 33-key row. A rung that reports `blocked`
+        # while the vendor is holding the answer on another route is a rung that gave up early.
+        # ⚠ The FIRST failure is kept in the note either way: if the corpus also misses, the
+        # person needs to know the scrape was tried and what it said, not only the fallback.
+        # ⚠ THE UNREADABLE-ROW DIAGNOSTIC IS CARRIED, NOT DROPPED. "It answered 200 with something
+        # we could not parse" and "it refused" are different problems with different fixes, and
+        # the field-names hint is the one that tells the next reader to go and diff a real row
+        # against the schema. Losing it behind the corpus fallback would make a mapping
+        # regression look like a vendor outage.
+        scrape_note = note or ("the scrape answered, but no follower/following counts were "
+                               "readable in it (the field names may have moved — see "
+                               "instagram-capture.md)")
+        cnode, cnote = bd_corpus_profile(handle, log=log)
+        if cnode:
+            node = cnode
+            profile = _bd_profile(node, handle)
+            via = "brightdata:corpus"
+            # ⚠ The corpus is PRE-COLLECTED, so the row is the vendor's last capture rather than
+            # a read taken now. Said out loud on the attempt, because "when was this true" is the
+            # one question a stored measurement must always be able to answer.
+            note = (f"the live scrape did not answer ({_s(scrape_note, 120)}), so this came from "
+                    f"the vendor's pre-collected profile corpus — the values are its last "
+                    f"capture, not a read taken just now")
+        else:
+            return {"state": "blocked", "profile": {}, "posts": [], "via": "brightdata",
+                    "note": f"{scrape_note}; the profile corpus did not answer either ({cnote})"}
     if profile.get("followers") is None and profile.get("following") is None:
         # It answered 200 with something we could not read. Say THAT — not "0 followers".
         # ⚠ `posts_count` is deliberately NOT part of this test: it arrives as a fabricated 0 and
@@ -2754,8 +2854,12 @@ def pull_profile_bd(url, max_posts=DEFAULT_POSTS_PER_PULL, post_metrics=False, l
     # IDENTITY WITHOUT MEDIA IS STILL `partial`, on the paid rung too. The rule does not soften
     # because we are paying: a run that wrote a follower count and no posts must not paint green
     # over a posts table that did not grow.
+    # ⚠ `via` CARRIES WHICH ROUTE ANSWERED — `brightdata` or `brightdata:corpus`. It lands in the
+    # snapshot row's `source` column, so a stored measurement can always say whether it was read
+    # live or taken from the vendor's pre-collected corpus. Two routes writing one indisting-
+    # uishable `source` would make the series unauditable exactly where it matters most.
     return {"state": "ok" if posts else "partial", "profile": profile, "posts": posts,
-            "via": "brightdata",
+            "via": via,
             "note": note or ("" if posts else "profile read; no posts were returned")}