fsanyoto commited on
Commit
e4d9f48
Β·
verified Β·
1 Parent(s): 5bfda9a

Deploy AIOS web (React glide grid + FastAPI slice)

Browse files
RELEASES.json CHANGED
@@ -1,5 +1,5 @@
1
  {
2
- "current": "b859a84",
3
  "releases": [
4
  {
5
  "version": "v14",
 
1
  {
2
+ "current": "139bf21",
3
  "releases": [
4
  {
5
  "version": "v14",
VERSION CHANGED
@@ -1 +1 @@
1
- b859a84
 
1
+ 139bf21
api/automation_engine.py CHANGED
@@ -23,8 +23,9 @@ the skill version takes a URL from a developer on a CLI; this takes one from a r
23
  """
24
  from __future__ import annotations
25
 
26
- import datetime as _dt
27
- import hmac
 
28
  import ipaddress
29
  import json
30
  import os
@@ -92,7 +93,12 @@ IG_TABLE_PREFIX = "ut_ig_"
92
  #: have inherited a 200,000-row ceiling, and with it the measured 35.8 MB single-bucket
93
  #: serialisation cost, purely by an accident of naming. Naming the append tables is the version
94
  #: that stays true when the next `ut_ig_*` table is not one.
95
- APPEND_TABLES = frozenset({"ut_ig_snapshots", "ut_ig_post_snapshots"})
 
 
 
 
 
96
 
97
  #: ⭐ WAVE 24 (owner ruling R6) β€” `plain` IS WHAT AN AUTOMATION IS NOW, and it is the DEFAULT.
98
  #: The create wizard is deleted, so nobody picks a kind any more: a new automation is a trigger
@@ -622,7 +628,7 @@ def next_fire(expr, now=None, lookahead_days=400):
622
  # THE UPSERT β€” pure, so the arithmetic is testable without a store or a network
623
  # ---------------------------------------------------------------------------------------------
624
 
625
- def upsert_rows(existing, incoming, key_field, cap=None):
626
  """Merge scraped rows into a user table's rows BY KEY. Returns `(rows, counts)`.
627
 
628
  THE RULE THAT MATTERS: **an orphan is COUNTED, NEVER DELETED.** A row that has stopped
@@ -691,7 +697,41 @@ def upsert_rows(existing, incoming, key_field, cap=None):
691
  counts["duplicates"] = incoming_dupes + len(dupe_ids)
692
  counts["orphans"] = sum(
693
  1 for kv, rid in by_key.items() if kv not in seen_keys and rid not in dupe_ids)
694
- return rows, counts
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
695
 
696
 
697
  # ---------------------------------------------------------------------------------------------
@@ -797,7 +837,8 @@ def ut_key_for(label, key=None):
797
  MACHINE_OWNERS = ("automation", "scheduler")
798
 
799
 
800
- def ut_ensure(rt, label, fields, username="automation", key=None, flow_tag=""):
 
801
  """Create the table if it is missing; return its key. Idempotent β€” a re-run of an automation
802
  that owns a table must not spawn `ut_x_2`, so the key is DERIVED from the label (or given)
803
  and an existing table with that key is adopted, not duplicated.
@@ -838,21 +879,34 @@ def ut_ensure(rt, label, fields, username="automation", key=None, flow_tag=""):
838
  # A migration that cannot run must not stop the automation from writing its rows.
839
  # The old schema still reads; a refused write loses the pull we just paid for.
840
  pass
841
- wanted = [({**f, "automation": {"flowId": str(flow_tag)}}
842
- if flow_tag and not f.get("automation") else dict(f))
843
- for f in (fields or [])]
 
 
 
 
 
 
 
 
844
  created = _iso()
845
  human = username if username and username not in MACHINE_OWNERS else ""
846
 
847
  # ⚠ SKIP THE WRITE WHEN NOTHING WOULD CHANGE. Without this, every re-run spends a store
848
  # commit re-writing an identical definition β€” against a 20 s flush floor and a 256/hr repo
849
  # budget, an idempotent helper that always writes is the same defect as a per-row insert.
850
- have = ut_get(rt, key)
851
- if (have is not None
852
- and not [f for f in wanted
853
- if f.get("key") not in {g.get("key") for g in (have.get("fields") or [])}]
854
- and not (human and (have.get("createdBy") or "") in MACHINE_OWNERS)):
855
- return key
 
 
 
 
 
856
 
857
  def _up(cur):
858
  cur = cur if isinstance(cur, dict) else {}
@@ -860,21 +914,34 @@ def ut_ensure(rt, label, fields, username="automation", key=None, flow_tag=""):
860
  if t is None:
861
  if len(cur) >= MAX_UT_TABLES:
862
  return cur
863
- cur[key] = {"key": key, "label": str(label)[:60], "source": "Automation",
864
- "createdBy": username, "created": created,
865
- "fields": wanted, "rows": {}}
866
- return cur
 
 
867
  have = {f.get("key") for f in (t.get("fields") or [])}
868
- for f in wanted:
869
- if f.get("key") not in have:
870
- t.setdefault("fields", []).append(f)
871
- have.add(f.get("key"))
 
 
 
 
 
 
 
 
 
872
  # ADOPTION: a machine name is not an owner. It never overwrites a human one.
873
  if human and (t.get("createdBy") or "") in MACHINE_OWNERS:
874
  t["createdBy"] = human
875
  # An automation's table SAYS an automation owns it β€” the nav badge reads from this.
876
- t["source"] = t.get("source") or "Automation"
877
- return cur
 
 
878
 
879
  rt.update(UT_STORE_KEY, _up, flush="sync")
880
  return key
@@ -1759,15 +1826,26 @@ def _presets_after_write(rt, defn, username):
1759
  arrives later. The run-time path reports that condition LOUDLY (`ut_missing`, D-11), so the
1760
  honest failure still has exactly one home.
1761
  """
1762
- if (defn.get("trigger") or {}).get("key") != "ig_profile_match":
1763
- return
1764
- cfg = defn.get("config") or {}
1765
- target = str(cfg.get("targetTable") or "")
1766
- if not target:
1767
- return
1768
- try:
1769
- ut_ensure(rt, cfg.get("targetLabel") or "IG candidates", CANDIDATE_FIELDS, username,
1770
- key=target, flow_tag=str(defn.get("id") or ""))
 
 
 
 
 
 
 
 
 
 
 
1771
  except Exception as e: # noqa: BLE001
1772
  # β›” NEVER FAILS THE SAVE. The definition is already committed by the time this runs, so
1773
  # raising here would answer 500 for an automation that IS stored β€” the caller would retry
@@ -1945,8 +2023,7 @@ SNAPSHOT_FIELDS = [
1945
  field_def("is_professional", "Professional account"),
1946
  field_def("is_private", "Private"),
1947
  field_def("highlights_count", "Highlights"),
1948
- field_def("bio_hashtags", "Bio hashtags"),
1949
- field_def("post_hashtags", "Post hashtags"),
1950
  field_def("pronouns", "Pronouns"),
1951
  # ⭐ 2026-08-07 β€” the rest of the vendor's Profiles schema (see `_bd_profile`). The snapshot
1952
  # row maps the WHOLE schema by design, so these belong here the moment the map reads them;
@@ -1988,8 +2065,12 @@ POST_FIELDS = [
1988
  # ⚠ BLANK, NEVER ZERO. `postMetrics` is off by default (it buys one vendor record per post),
1989
  # so on most pulls these stay empty β€” and empty means "not read", which is what makes an
1990
  # honest average possible at all. A 0 here would claim a post nobody watched.
1991
- field_def("views", "Views", "int"),
1992
- field_def("likes", "Likes", "int"),
 
 
 
 
1993
  field_def("comments", "Comments", "int"),
1994
  field_def("measured_at", "Engagement read at", "date"),
1995
  # ⭐ 2026-08-07 β€” the second half of "spawn relevant Post/Comment database that is LINKED":
@@ -1998,8 +2079,12 @@ POST_FIELDS = [
1998
  # ⚠ It resolves to nothing until comment capture is switched on, which is the honest state
1999
  # for a relation whose far side is empty β€” the same standing `ut_ig_post_snapshots` has when
2000
  # `postMetrics` is off.
2001
- field_def("comments_link", "Comment rows", "link",
2002
- link={"table": "ut_ig_comments", "on": "shortcode", "from": "shortcode"}),
 
 
 
 
2003
  ]
2004
 
2005
  #: ⭐⭐ 2026-08-07 (owner instruction) β€” THE COMMENT DATABASE.
@@ -2031,21 +2116,29 @@ COMMENT_FIELDS = [
2031
  field_def("comment_key", "Comment"), field_def("shortcode", "Shortcode"),
2032
  field_def("influencer_key", "Influencer"),
2033
  field_def("commented_at", "Commented at", "date"),
2034
- field_def("likes", "Likes", "int"),
2035
- field_def("replies", "Replies", "int"),
 
 
 
2036
  # ⚠ NO AUTHOR COLUMN AND NO TEXT COLUMN, and their absence is the ruling rather than an
2037
  # oversight. `comment_user` is vendor-flagged PII and the text is the payload D-22 priced.
2038
  # What is left is the SHAPE of a thread β€” how many, how recent, how engaged β€” which is the
2039
  # part that informs an influencer decision without ingesting a stranger.
2040
  ]
2041
  POST_SNAPSHOT_FIELDS = [
2042
- field_def("post_snapshot_key", "Snapshot"), field_def("shortcode", "Shortcode"),
2043
- field_def("pulled_at", "Pulled at"), field_def("likes", "Likes"),
2044
- field_def("comments", "Comments"),
 
2045
  # Plays/views move independently of likes on video, so it is its own series rather than a
2046
  # thing to derive. Paid rung only; blank means not read.
2047
- field_def("views", "Views"),
2048
- ]
 
 
 
 
2049
 
2050
 
2051
  def ig_handle(url):
@@ -2191,7 +2284,8 @@ def _loose_count(txt):
2191
  BD_BASE_DEFAULT = "https://api.brightdata.com"
2192
  BD_DS_PROFILES = "gd_l1vikfch901nx3by4" # Instagram – Profiles. 36 fields, 620M records
2193
  BD_DS_POSTS = "gd_lk5ns7kz21pck8jpis" # Instagram – Posts. 43 fields
2194
- BD_PATH_SCRAPE = "/datasets/v3/scrape" # SYNC: rows come back inline. param: dataset_id
 
2195
  BD_PATH_TRIGGER = "/datasets/v3/trigger" # ASYNC: -> {"snapshot_id": "sd_…"}
2196
  BD_PATH_SNAPSHOT = "/datasets/v3/snapshot" # /<sd_id>?format=json -> the rows
2197
  BD_PATH_FILTER = "/datasets/filter" # ⚠ NO /v3/ β€” the CORPUS query (discovery)
@@ -2501,8 +2595,7 @@ def _bd_profile(node, handle):
2501
  "is_professional": _bd_flag(node, "is_professional_account"),
2502
  "is_private": _bd_flag(node, "is_private"),
2503
  "highlights_count": _ig_int(_first(node, "highlights_count")),
2504
- "bio_hashtags": _bd_list(node, "bio_hashtags"),
2505
- "post_hashtags": _bd_list(node, "post_hashtags"),
2506
  "pronouns": str(_first(node, "pronouns", default="") or ""),
2507
  # ⭐ 2026-08-07 (owner instruction: *"have ALL Fields available to us from Bright Data to
2508
  # be pre-set Fields for us and populated"*) β€” THE REST OF THE PROFILES SCHEMA.
@@ -2558,8 +2651,13 @@ def _bd_post_identity(p):
2558
  code = ig_shortcode(p.get("url")) if isinstance(p, dict) else ""
2559
  if not code:
2560
  return None
2561
- out = {"shortcode": code, "url": f"https://www.instagram.com/p/{code}/",
2562
- "type": _bd_type(p.get("content_type"), p.get("type"))}
 
 
 
 
 
2563
  caption = _first(p, "caption", "description", default="")
2564
  if isinstance(caption, dict):
2565
  caption = _first(caption, "text", default="")
@@ -2599,8 +2697,11 @@ def _bd_post_metrics(row):
2599
  "caption": str(_first(row, "description", "caption", default="") or ""),
2600
  "likes": _ig_int(_first(row, "likes", "like_count")),
2601
  "comments": _ig_int(_first(row, "num_comments", "comment_count", "comments")),
2602
- "views": _ig_int(_first(row, "video_play_count", "video_view_count",
2603
- "engagement_score_view", "play_count")),
 
 
 
2604
  # Sponsored-post detection. MEASURED populated (`True`, with the brand alongside) β€” and
2605
  # it is the one field here that answers a commercial question the counts cannot.
2606
  "paid_partnership": _bd_flag(row, "is_paid_partnership"),
@@ -2829,15 +2930,31 @@ def pull_profile_bd(url, max_posts=DEFAULT_POSTS_PER_PULL, post_metrics=False, l
2829
  if got:
2830
  posts.append(got)
2831
 
2832
- if post_metrics and posts:
2833
- time.sleep(min(PACE_SECONDS, 1.0)) # the vendor is paid, but it is still someone's API
2834
- rows2, note2 = bd_scrape(BD_DS_POSTS, [p["url"] for p in posts])
2835
- if note2:
2836
- # The engagement half refusing does NOT lose the identity half β€” the profile and the
2837
- # post rows still land, and the run says which part was not readable.
2838
- note = f"post metrics unavailable: {note2}"
2839
- else:
2840
- by_code = {}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2841
  for r in rows2:
2842
  got = _bd_post_metrics(r)
2843
  if got:
@@ -2848,8 +2965,10 @@ def pull_profile_bd(url, max_posts=DEFAULT_POSTS_PER_PULL, post_metrics=False, l
2848
  # The metrics row is RICHER (it knows posted_at); merge it over the identity
2849
  # row, dropping keys it could not read so a blank never overwrites a value.
2850
  p.update({k: v for k, v in extra.items() if v not in (None, "")})
2851
- if not by_code:
2852
- note = "post metrics were requested but the Posts dataset returned no rows"
 
 
2853
 
2854
  # IDENTITY WITHOUT MEDIA IS STILL `partial`, on the paid rung too. The rule does not soften
2855
  # because we are paying: a run that wrote a follower count and no posts must not paint green
@@ -3300,7 +3419,6 @@ PRESET_PROFILE_FIELDS = [
3300
  field_def("business_category", "Business category"),
3301
  field_def("is_private", "Private account", "checkbox"),
3302
  field_def("bio_hashtags", "Bio hashtags"),
3303
- field_def("post_hashtags", "Post hashtags"),
3304
  field_def("pronouns", "Pronouns"),
3305
  field_def("profile_name", "Profile name"),
3306
  field_def("is_joined_recently", "Joined recently", "checkbox"),
@@ -3329,7 +3447,13 @@ PRESET_PROFILE_FIELDS = [
3329
  # identically-named columns is unreadable. Caught by the preset set's own label-collision
3330
  # gate rather than on screen, which is what that gate is for.
3331
  field_def("posts_link", "Post rows", "link",
3332
- link={"table": "ut_ig_posts", "on": "influencer_key"}),
 
 
 
 
 
 
3333
  # ⚠ THE ROLLUPS READ `ut_ig_posts`' OWN LATEST COLUMNS, which is why those exist β€” a rollup
3334
  # is ONE HOP (Airtable's rule and ours), and the engagement SERIES lives one table further
3335
  # out in `ut_ig_post_snapshots`.
@@ -3339,20 +3463,36 @@ PRESET_PROFILE_FIELDS = [
3339
  # has to name what makes one post later than another.
3340
  field_def("avg_views_12", "Avg views Β· last 12 posts", "rollup",
3341
  rollup={"link": "posts_link", "field": "views", "fn": "average",
3342
- "limit": MAX_POSTS_PER_PULL, "sortBy": "posted_at", "sortDir": "desc"}),
 
 
 
 
 
3343
  field_def("avg_likes_12", "Avg likes Β· last 12 posts", "rollup",
3344
  rollup={"link": "posts_link", "field": "likes", "fn": "average",
3345
- "limit": MAX_POSTS_PER_PULL, "sortBy": "posted_at", "sortDir": "desc"}),
 
3346
  field_def("avg_comments_12", "Avg comments Β· last 12 posts", "rollup",
3347
  rollup={"link": "posts_link", "field": "comments", "fn": "average",
3348
- "limit": MAX_POSTS_PER_PULL, "sortBy": "posted_at", "sortDir": "desc"}),
 
3349
  # ⭐ THE ONE HONEST POST COUNT WE HAVE. D-82: the vendor's `posts_count` is a FABRICATED ZERO
3350
  # on the paid rung (49/49 rows measured), so it is discarded and that column reads blank after
3351
  # a paid enrich. This counts the post rows actually captured β€” a different number and a true
3352
  # one, which is why it gets its own column and its own label rather than quietly filling
3353
  # `posts_count` with something that is not what that field means.
3354
  field_def("posts_captured", "Posts captured", "rollup",
3355
- rollup={"link": "posts_link", "fn": "countall"}),
 
 
 
 
 
 
 
 
 
3356
  # ⭐ R3's stamp. WITHOUT IT THE WHOLE SET IS UNREADABLE: a blank `followers` means "never
3357
  # enriched" and a stale one means "enriched in March", and no cell on the row can tell them
3358
  # apart. It is the single field that turns the other fifteen from numbers into measurements.
@@ -3364,8 +3504,7 @@ PRESET_PROFILE_FIELDS = [
3364
  # views/likes/comments at a `pulled_at`). This cell is a DERIVED window over those two,
3365
  # rewritten each run, so a person reading the profile row can see the recent posts without a
3366
  # join β€” and deleting it would cost a convenience, never a measurement. Shape: contract C2.
3367
- field_def("posts", "Recent posts", "json"),
3368
- ]
3369
  #: The keys the preset set owns β€” derived, so a field added above cannot be forgotten here.
3370
  PRESET_PROFILE_KEYS = tuple(f["key"] for f in PRESET_PROFILE_FIELDS)
3371
 
@@ -3385,7 +3524,7 @@ PRESET_FLAG_KEY = next((f["key"] for f in PRESET_PROFILE_FIELDS if f.get("profil
3385
  #: the failure C1 exists to prevent. The five below are facts about the SEARCH (how often we found
3386
  #: them, who for, and a human's decision) rather than about the profile, so they are discovery's
3387
  #: and not part of the cross-tenant set.
3388
- CANDIDATE_FIELDS = [
3389
  *PRESET_PROFILE_FIELDS,
3390
  # ⭐ WAVE 26 Β· R3 β€” `first_found` / `last_found` ARE DATES, not ISO strings in a text cell.
3391
  # They were written by `_iso()`, so the cell read `2026-08-05T14:03:11+07:00` and the owner
@@ -3408,7 +3547,153 @@ CANDIDATE_FIELDS = [
3408
  # column: every candidate carried BOTH a `stage_<auto>` select saying where it was AND a
3409
  # boolean saying whether it was kept β€” two progress fields that could disagree, with no rule
3410
  # about which one won. The stage field is the one progress column. Nothing replaces this.
3411
- ]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3412
 
3413
 
3414
  # ── ⭐ WAVE 26 Β· THE MIGRATION (owner rulings R3 + R4/R5, contracts C1-a and C3) ───────────────
@@ -3558,7 +3843,8 @@ def migrate_ig_tables(rt, log=print, only=""):
3558
  # ⭐ 2026-08-07 (owner ruling) β€” the primary-column half. Counted separately from
3559
  # `stamped` because they answer different questions: that one is "how many ROWS got a
3560
  # platform", these are "how many TABLES changed shape".
3561
- "pinned": 0, "flagged": 0, "droppedName": 0}
 
3562
  want = {f["key"]: f["type"] for f in CANDIDATE_FIELDS}
3563
  # β›” `only` NARROWS THE SET, IT DOES NOT BYPASS THE TEST β€” and skipping that cost six red
3564
  # checks the first time. `ut_ensure` calls this with the key it is ABOUT TO CREATE, so on a
@@ -3600,12 +3886,13 @@ def migrate_ig_tables(rt, log=print, only=""):
3600
  other_profile = next((f for f in fields if f is not flag_f
3601
  and isinstance(f.get("profile"), dict)), None)
3602
  want_pin = pin_f is not None and pin_f.get("pinned") is not True
3603
- want_flag = (flag_f is not None and other_profile is None
3604
- and not isinstance(flag_f.get("profile"), dict))
3605
- vestigial = _vestigial_name_field(fields, rows)
3606
- if (not stale_keys and has_platform and not needs_stamp
3607
- and not want_pin and not want_flag and vestigial is None):
3608
- continue
 
3609
  stats["tables"] += 1
3610
  if want_pin:
3611
  pin_f["pinned"] = True
@@ -3613,14 +3900,21 @@ def migrate_ig_tables(rt, log=print, only=""):
3613
  if want_flag:
3614
  flag_f["profile"] = {"source": PROFILE_SOURCE_IG}
3615
  stats["flagged"] += 1
3616
- if vestigial is not None:
3617
  # ⚠ THE CELLS GO WITH THE COLUMN. A row dict keeping a `name` key whose field no longer
3618
  # exists is invisible everywhere except the next export, where it reappears as a column
3619
  # nobody declared. `_vestigial_name_field` has already proven every one of them blank.
3620
  fields = [f for f in fields if f is not vestigial]
3621
  for r in rows.values():
3622
  r.pop("name", None)
3623
- stats["droppedName"] += 1
 
 
 
 
 
 
 
3624
 
3625
  for key in stale_keys:
3626
  conv = _MIGRATE_CONVERT.get(key)
@@ -4598,7 +4892,7 @@ def capture_rows(res, pulled):
4598
  ("alt_text", 400)):
4599
  if p.get(k):
4600
  ident[k] = _s(p.get(k), n)
4601
- metrics = {k: p.get(k) for k in ("likes", "comments", "views")}
4602
  # ⭐⭐ 2026-08-07 β€” THE LATEST ENGAGEMENT VALUES, ONTO THE POST ROW ITSELF.
4603
  #
4604
  # This is what keeps a rollup at ONE HOP (Airtable's rule and ours): without it, "average
@@ -4624,11 +4918,12 @@ def capture_rows(res, pulled):
4624
  ident["measured_at"] = _day(pulled)
4625
  idents.append(ident)
4626
  if any(v is not None for v in metrics.values()):
4627
- metrics_rows.append({
4628
- "post_snapshot_key": f"{p['shortcode']}@{pulled}",
4629
- "shortcode": p["shortcode"], "pulled_at": pulled,
4630
- "likes": _s(metrics["likes"]), "comments": _s(metrics["comments"]),
4631
- "views": _s(metrics["views"])})
 
4632
  return snap_row, idents, metrics_rows
4633
 
4634
 
@@ -4651,7 +4946,7 @@ PRESET_FROM_PROFILE = {
4651
  # `PRESET_PROFILE_KEYS` entry must either be written by this map or be explicitly declared
4652
  # as written elsewhere (`platform`, `enriched_at`, `posts` are stamped by `preset_cells`).
4653
  "business_category": "business_category", "is_private": "is_private",
4654
- "bio_hashtags": "bio_hashtags", "post_hashtags": "post_hashtags",
4655
  "pronouns": "pronouns", "profile_name": "profile_name",
4656
  "is_joined_recently": "is_joined_recently", "has_channel": "has_channel",
4657
  "partner_id": "partner_id", "external_url_title": "external_url_title",
@@ -4666,95 +4961,15 @@ PRESET_FROM_PROFILE = {
4666
  #: ⚠ The five relational columns are written by `compute_relation_cells` on the tick, NOT by an
4667
  #: enrichment run β€” which is the whole point of them: they stay true when the LINKED table
4668
  #: changes, and a pull that touched no profile still updates a profile's post count.
4669
- PRESET_WRITTEN_ELSEWHERE = ("platform", "handle", "enriched_at", "posts",
4670
- "posts_link", "avg_views_12", "avg_likes_12", "avg_comments_12",
4671
- "posts_captured")
4672
-
4673
-
4674
- #: The keys one post carries into the window. Deliberately SHORTER than `POST_FIELDS`: this is a
4675
- #: readable summary on a profile row, not a second copy of the post record. `ut_ig_posts` keeps
4676
- #: everything, including the fields nobody reads at a glance (`alt_text`, `hashtags`, `partner`).
4677
- _WINDOW_POST_KEYS = ("shortcode", "url", "posted_at", "type", "caption")
4678
- #: The engagement half, present only when the PAID rung bought it (R2). ABSENT, never zero.
4679
- _WINDOW_METRIC_KEYS = ("views", "likes", "comments")
4680
- #: A caption is the one unbounded field here, and 32 KB is the cell ceiling. Bounded per post so
4681
- #: the window cannot approach it: 12 posts x 280 chars is ~4 KB, comfortably inside `MAX_JSON_CELL`.
4682
- WINDOW_CAPTION_CHARS = 280
4683
- #: β›” THE FIELD LAYER'S CEILING, READ LAZILY β€” AND THE LAZINESS IS NOT STYLE.
4684
- #:
4685
- #: `core.user_tables.MAX_JSON_CELL` is what actually refuses an oversized cell, so reading it beats
4686
- #: restating 32 KB here (a local copy drifts the day that constant moves, and the engine would then
4687
- #: build a document the door rejects).
4688
- #: ⚠ BUT IMPORTING IT AT MODULE LEVEL CHANGED BEHAVIOUR SOMEWHERE ELSE ENTIRELY. Pulling
4689
- #: `core.user_tables` in at import time initialises the store layer earlier than this module used
4690
- #: to, and a gate asserting "the upsert ran THREE times for the WHOLE run" went to FOUR: the
4691
- #: platform master's write had been failing before its first upsert and now failed after it. Same
4692
- #: outcome, one more call against a 256-commits/hr budget β€” caught by a COUNT, invisible to every
4693
- #: behavioural check. A top-level import is a side effect; this one buys a constant.
4694
- def _max_json_cell():
4695
- try:
4696
- from core.user_tables import MAX_JSON_CELL
4697
- return MAX_JSON_CELL
4698
- except Exception: # noqa: BLE001
4699
- return 32 * 1024
4700
-
4701
-
4702
- def posts_window(res, pulled):
4703
- """⭐ WAVE 26 Β· R1 / contract C2 β€” the last-N posts, as a DERIVED json cell.
4704
 
4705
- β›” THIS IS A VIEW, AND R3 IS UNTOUCHED. The authoritative post record is `ut_ig_posts` (keyed
4706
- by shortcode, so it ACCUMULATES across runs) and the authoritative engagement series is
4707
- `ut_ig_post_snapshots` (one appended row per pull, carrying views/likes/comments at a
4708
- `pulled_at`). Nothing here is a second home for either: the cell is rewritten every run, and
4709
- deleting it would cost a convenience rather than a measurement. That is exactly what makes it
4710
- compatible with *"one store for one series"* β€” the owner asked for the posts to be visible ON
4711
- the profile row AND to keep accumulating, and those are two different jobs.
4712
 
4713
- ⚠ AN ABSENT METRIC IS AN ABSENT KEY, NEVER A ZERO. `postMetrics` is off by default (R2 β€” it
4714
- buys one extra vendor record per post), so most windows carry identity and no engagement. A
4715
- `views: 0` would claim we measured a post nobody watched; the key simply is not there, and the
4716
- client renders nothing for it. This module's blank-never-zero law, at one more seam.
4717
- """
4718
- posts = [p for p in ((res or {}).get("posts") or []) if isinstance(p, dict)]
4719
- if not posts:
4720
- return ""
4721
- out = []
4722
- for p in posts:
4723
- row = {}
4724
- for k in _WINDOW_POST_KEYS:
4725
- v = p.get(k)
4726
- if v is None or str(v).strip() == "":
4727
- continue
4728
- row[k] = _s(v, WINDOW_CAPTION_CHARS) if k == "caption" else _s(v, 300)
4729
- for k in _WINDOW_METRIC_KEYS:
4730
- v = p.get(k)
4731
- if v is None or str(v).strip() == "":
4732
- continue # NOT read -> not a key. Never 0.
4733
- n = _ig_int(v)
4734
- if n is not None:
4735
- row[k] = n
4736
- if row:
4737
- out.append(row)
4738
- if not out:
4739
- return ""
4740
- # Newest first, so "the last 10 posts" reads top-down. A post with no date sorts last rather
4741
- # than first: an unknown date is not a recent one.
4742
- out.sort(key=lambda r: str(r.get("posted_at") or ""), reverse=True)
4743
- doc = {"n": len(out), "metrics": any(k in r for r in out for k in _WINDOW_METRIC_KEYS),
4744
- "as_of": _day(pulled), "posts": out}
4745
- text = json.dumps(doc, ensure_ascii=False, separators=(",", ":"))
4746
- # β›” THE CELL CEILING IS THE FIELD LAYER'S, and blowing it would have the write REFUSED β€” so
4747
- # the window sheds posts until it fits rather than losing the whole cell. Shedding is safe
4748
- # precisely because this is a view: `ut_ig_posts` still holds every one of them.
4749
- ceiling = _max_json_cell()
4750
- while len(text.encode("utf-8")) > ceiling and len(doc["posts"]) > 1:
4751
- doc["posts"] = doc["posts"][:-1]
4752
- doc["n"] = len(doc["posts"])
4753
- text = json.dumps(doc, ensure_ascii=False, separators=(",", ":"))
4754
- return text
4755
-
4756
-
4757
- def preset_cells(res, pulled):
4758
  """C4/R3: one pull β†’ the LATEST-value cells written onto the enriched record.
4759
 
4760
  β›” `enriched_at` IS ALWAYS WRITTEN when a pull succeeded, and it is the field that makes the
@@ -4776,9 +4991,6 @@ def preset_cells(res, pulled):
4776
  # arrived with no platform at all β€” and a blank half of the dedup key is how one account
4777
  # becomes two rows.
4778
  cells["platform"] = PLATFORM_INSTAGRAM
4779
- win = posts_window(res, pulled)
4780
- if win:
4781
- cells["posts"] = win
4782
  # R3: `enriched_at` is a `date` column now. It answers "how stale is this number", which is a
4783
  # question in days; the full stamp keeps its precision on the snapshot series.
4784
  cells["enriched_at"] = _day(pulled)
@@ -4816,21 +5028,13 @@ def run_field_instagram(rt, defn, username="automation", log=print, step=_no_ste
4816
  # the relational tables the pull lands in (R7): every row timestamped for time-range filters
4817
  if dry:
4818
  # The Write node is off: resolve the keys, create nothing. (`ut_ensure` writes.)
4819
- snap_key, post_key, ps_key = ("ut_ig_snapshots", "ut_ig_posts", "ut_ig_post_snapshots")
4820
- else:
4821
- snap_key = ut_ensure(rt, "IG snapshots", SNAPSHOT_FIELDS, username,
4822
- key="ut_ig_snapshots", flow_tag=str(defn.get("id") or ""))
4823
- post_key = ut_ensure(rt, "IG posts", POST_FIELDS, username, key="ut_ig_posts",
4824
- flow_tag=str(defn.get("id") or ""))
4825
- ps_key = ut_ensure(rt, "IG post snapshots", POST_SNAPSHOT_FIELDS, username,
4826
- key="ut_ig_post_snapshots", flow_tag=str(defn.get("id") or ""))
4827
- # ⭐ 2026-08-07 β€” the COMMENT database, spawned beside the other three so a post's
4828
- # `comments_link` names a table that exists rather than one that might. It stays EMPTY
4829
- # until comment capture is switched on β€” the same honest state `ut_ig_post_snapshots`
4830
- # holds while `postMetrics` is off, which is why it is ensured on the same line rather
4831
- # than being made conditional.
4832
- ut_ensure(rt, "IG comments", COMMENT_FIELDS, username, key="ut_ig_comments",
4833
- flow_tag=str(defn.get("id") or ""))
4834
  missing = [] if dry else ut_missing(rt, snap_key, post_key, ps_key)
4835
  snaps = dict((ut_get(rt, snap_key) or {}).get("rows") or {})
4836
  posts = dict((ut_get(rt, post_key) or {}).get("rows") or {})
@@ -4902,7 +5106,9 @@ def run_field_instagram(rt, defn, username="automation", log=print, step=_no_ste
4902
 
4903
  # --- THE THREE UPSERTS. Once each, over the whole run's accumulated rows.
4904
  snaps, c_snap = upsert_rows(snaps, in_snaps, "snapshot_key", cap=row_cap(snap_key))
4905
- posts, c_post = upsert_rows(posts, in_posts, "shortcode", cap=row_cap(post_key))
 
 
4906
  psnaps, c_ps = upsert_rows(psnaps, in_psnaps, "post_snapshot_key", cap=row_cap(ps_key))
4907
  counts["new_posts"] = c_post["inserted"]
4908
  capped = [(snap_key, c_snap["capped"]), (post_key, c_post["capped"]),
@@ -4936,14 +5142,15 @@ def run_field_instagram(rt, defn, username="automation", log=print, step=_no_ste
4936
  tt.setdefault("rows", {}).setdefault(str(rid), {})[fkey] = val
4937
  for rid, vals in stage_writes.items():
4938
  tt.setdefault("rows", {}).setdefault(str(rid), {}).update(vals)
4939
- for k, rws in ((snap_key, snaps), (post_key, posts), (ps_key, psnaps)):
4940
- tgt = cur.get(k)
4941
- if tgt is not None:
4942
- tgt["rows"] = rws
4943
- return cur
4944
-
4945
- rt.update(UT_STORE_KEY, _up, flush="sync") # ONE coalesced update for all four tables
4946
- if lanes:
 
4947
  ensure_stage_field(rt, table_key, defn, username)
4948
 
4949
  # --- C6 (R2): WRITE-THROUGH to the platform master. Three postures, never conflated:
@@ -5191,10 +5398,10 @@ def run_discover_instagram(rt, defn, username="automation", log=print, step=_no_
5191
  # down the rows path and parses as one unusable row instead of an error.
5192
  counts["dropped"] = max(0, len(rows) - len(incoming))
5193
  label = cfg.get("targetLabel") or "IG candidates"
5194
- table_key = (ut_key_for(label, cfg.get("targetTable") or DISCOVER_TABLE) if dry
5195
- else ut_ensure(rt, label, CANDIDATE_FIELDS, username,
5196
- key=cfg.get("targetTable") or DISCOVER_TABLE,
5197
- flow_tag=str(defn.get("id") or "")))
5198
  existing = dict((ut_get(rt, table_key) or {}).get("rows") or {})
5199
  # --- ⭐ WAVE 26 Β· C3 / owner ruling R4 β€” THE UPSERT KEY IS `(platform, handle)`.
5200
  #
@@ -6873,7 +7080,7 @@ def enrich_selection(rt, table_key, cfg, profile_key, today=None):
6873
  return picked, "; ".join(notes)
6874
 
6875
 
6876
- def _has_action(actions, kind):
6877
  """Does this flow contain `kind` ANYWHERE, forks included?
6878
 
6879
  β›” FORKS ARE THE WHOLE REASON THIS IS A FUNCTION. A group's children live under
@@ -6890,7 +7097,20 @@ def _has_action(actions, kind):
6890
  for br in ((a.get("config") or {}).get("branches") or []):
6891
  if _has_action((br or {}).get("actions") or [], kind):
6892
  return True
6893
- return False
 
 
 
 
 
 
 
 
 
 
 
 
 
6894
 
6895
 
6896
  def apply_actions(rt, defn, table_key, row_ids, username="automation", log=print):
@@ -6960,20 +7180,33 @@ def apply_actions(rt, defn, table_key, row_ids, username="automation", log=print
6960
  # step at it, and (b) add a SECOND profile column to a table that already flagged a different
6961
  # one, which `user_tables` forbids at both write doors. A table with no profile column is an
6962
  # UNBOUND enrich: it must stay unbound and say so, which is a different defect (D-79) with its
6963
- # own honest refusal, not something to paper over by inventing the binding.
6964
- _tbl_now = ut_get(rt, table) or {}
6965
- _bound = next((f for f in (_tbl_now.get("fields") or [])
6966
- if isinstance(f.get("profile"), dict)), None)
6967
- if _bound and _has_action(actions, "enrich_instagram"):
 
 
 
 
 
 
 
6968
  # ⚠ AND THE DECLARATIONS ARE STRIPPED FROM ANYTHING NEW. The binding already exists β€”
6969
  # `_bound` is it β€” so a preset arriving now must carry DATA, never a second identity: a
6970
  # table whose profile column is `ig_handle` would otherwise gain a rival `handle`.
6971
  topup = [({k: v for k, v in f.items() if k not in ("profile", "pinned")}
6972
  if f.get("key") != _bound.get("key") else dict(f))
6973
  for f in PRESET_PROFILE_FIELDS]
6974
- try:
6975
- ut_ensure(rt, _tbl_now.get("label") or table, topup, username, key=table,
6976
- flow_tag=str(defn.get("id") or ""))
 
 
 
 
 
 
6977
  except Exception as exc: # noqa: BLE001
6978
  # A schema top-up that cannot run must not stop the enrichment: the cells that DO
6979
  # have columns still land, which is strictly better than the pull being thrown away.
@@ -7256,32 +7489,30 @@ def _enrich_flush(rt, defn, username, acc, log):
7256
  if acc.get("dry"):
7257
  out["enrichDryRun"] = acc["profiles"]
7258
  return out
7259
- tag = str(defn.get("id") or "")
7260
- snap_key = ut_ensure(rt, "IG snapshots", SNAPSHOT_FIELDS, username,
7261
- key="ut_ig_snapshots", flow_tag=tag)
7262
- post_key = ut_ensure(rt, "IG posts", POST_FIELDS, username, key="ut_ig_posts", flow_tag=tag)
7263
- ps_key = ut_ensure(rt, "IG post snapshots", POST_SNAPSHOT_FIELDS, username,
7264
- key="ut_ig_post_snapshots", flow_tag=tag)
7265
- # ⭐ 2026-08-07 β€” the COMMENT database. Spawned on BOTH enrich paths, not just one: the two
7266
- # writers already ensure the same three tables independently, and a fourth that only one of
7267
- # them knows about is how a database exists for automations created after Tuesday.
7268
- ut_ensure(rt, "IG comments", COMMENT_FIELDS, username, key="ut_ig_comments", flow_tag=tag)
7269
  missing = ut_missing(rt, snap_key, post_key, ps_key)
7270
  snaps, c_snap = upsert_rows(dict((ut_get(rt, snap_key) or {}).get("rows") or {}),
7271
  acc["snaps"], "snapshot_key", cap=row_cap(snap_key))
7272
- posts, c_post = upsert_rows(dict((ut_get(rt, post_key) or {}).get("rows") or {}),
7273
- acc["posts"], "shortcode", cap=row_cap(post_key))
 
 
7274
  psnaps, c_ps = upsert_rows(dict((ut_get(rt, ps_key) or {}).get("rows") or {}),
7275
  acc["psnaps"], "post_snapshot_key", cap=row_cap(ps_key))
7276
 
7277
  def _up(cur):
7278
  cur = cur if isinstance(cur, dict) else {}
7279
- for k, rws in ((snap_key, snaps), (post_key, posts), (ps_key, psnaps)):
7280
- if cur.get(k) is not None:
7281
- cur[k]["rows"] = rws
7282
- return cur
7283
-
7284
- rt.update(UT_STORE_KEY, _up, flush="sync") # ONE coalesced update for all three tables
 
7285
  # LOUD, never silent (D-11): a full append table means the SERIES has stopped growing, which
7286
  # is the failure a chart cannot show you.
7287
  capped = c_snap["capped"] + c_post["capped"] + c_ps["capped"]
@@ -8067,7 +8298,7 @@ def _ut():
8067
 
8068
  Pulling this module in at import time initialises the store layer earlier than
8069
  `automation_engine` used to, and the last time that happened it moved a store-commit COUNT
8070
- from three to four on an unrelated gate (see `_max_json_cell`'s note). The relational pass
8071
  needs three constants and two predicates from the field layer; it does not need to change
8072
  when this module is imported.
8073
  """
@@ -8121,7 +8352,7 @@ def _sort_key(value, ftype):
8121
  return (raw.lower(),)
8122
 
8123
 
8124
- def _rollup_fold(fn, values):
8125
  """`values` (raw cells, in the order the window kept them) β†’ the aggregate, as a STRING.
8126
 
8127
  Returns `""` for "nothing to aggregate", NEVER `0`. β›” That distinction is this module's
@@ -8131,8 +8362,13 @@ def _rollup_fold(fn, values):
8131
  ⚠ The ONE exception is the count family, where zero IS the answer β€” "how many linked records"
8132
  over an empty set is genuinely 0, not unknown.
8133
  """
8134
- if fn == "countall":
8135
- return str(len(values))
 
 
 
 
 
8136
  if fn == "counta":
8137
  return str(len([v for v in values if str(v or "").strip() != ""]))
8138
  if fn == "count":
@@ -8187,7 +8423,7 @@ def _link_from_key(fields, bag):
8187
  return str(pin.get("key") or "") if pin else ""
8188
 
8189
 
8190
- def _linked_rows_by_join(linked, on_key):
8191
  """`{join value (lower-cased) -> [(row_id, row)]}` over one linked table, built ONCE.
8192
 
8193
  ⚠ Lower-cased because the join values this exists for are Instagram handles, which the
@@ -8199,16 +8435,46 @@ def _linked_rows_by_join(linked, on_key):
8199
  k = str((row or {}).get(on_key) or "").strip().lower()
8200
  if k:
8201
  idx.setdefault(k, []).append((str(rid), row or {}))
8202
- return idx
8203
-
8204
-
8205
- def compute_relation_cells(rt, table_key):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8206
  """Recompute every DERIVED LINK cell and every ROLLUP cell on ONE table. Returns rows touched.
8207
 
8208
  Zero store reads when the table declares neither kind β€” the same cheap-by-construction shape
8209
  `compute_metric_cells` has, so walking every table on a tick costs a dict scan per table.
8210
  """
8211
- t = ut_get(rt, table_key)
 
8212
  fields = list((t or {}).get("fields") or [])
8213
  links = [f for f in fields if _ut().is_derived_link(f)]
8214
  rollups = [f for f in fields if isinstance(f.get("rollup"), dict)]
@@ -8225,10 +8491,23 @@ def compute_relation_cells(rt, table_key):
8225
  if not lk_key or lk_key in resolved or not isinstance((f or {}).get("link"), dict):
8226
  continue
8227
  bag = f["link"]
8228
- linked = ut_get(rt, str(bag.get("table") or "")) or {}
8229
  linked_types[lk_key] = {str(lf.get("key")): str(lf.get("type") or "text")
8230
  for lf in (linked.get("fields") or [])}
8231
- if bag.get("on"):
 
 
 
 
 
 
 
 
 
 
 
 
 
8232
  idx = _linked_rows_by_join(linked, str(bag["on"]))
8233
  from_key = _link_from_key(fields, bag)
8234
  resolved[lk_key] = {
@@ -8261,15 +8540,25 @@ def compute_relation_cells(rt, table_key):
8261
  want = ",".join(i for i, _r in hits[:_ut().LINK_MAX_IDS])
8262
  if str(row.get(fk, "")) != want:
8263
  changes.setdefault(rid, {})[fk] = want
8264
- for f in rollups:
8265
  fk, bag = str(f["key"]), f["rollup"]
8266
  lk_key = str(bag.get("link") or "")
8267
- hits = list((resolved.get(lk_key) or {}).get(rid) or [])
8268
  # ⚠ A rollup whose link field does not exist (renamed, deleted) resolves to NOTHING
8269
  # and therefore to a blank cell β€” never to a stale number. A column that keeps
8270
  # printing yesterday's answer after its input is gone is the worst of the options.
8271
- sort_by = str(bag.get("sortBy") or "")
8272
- if sort_by:
 
 
 
 
 
 
 
 
 
 
8273
  ftype = (linked_types.get(lk_key) or {}).get(sort_by, "text")
8274
  # β›” PARTITION, THEN SORT. A row whose sort cell is blank or unparseable is not
8275
  # rankable, and it must land at the END whichever direction is asked for β€” which
@@ -8280,8 +8569,21 @@ def compute_relation_cells(rt, table_key):
8280
  rankable = [(p, k) for p, k in keyed if k is not None]
8281
  rankable.sort(key=lambda pk: pk[1],
8282
  reverse=str(bag.get("sortDir") or "desc") == "desc")
8283
- hits = [p for p, _k in rankable] + [p for p, k in keyed if k is None]
8284
- limit = int(bag.get("limit") or 0)
 
 
 
 
 
 
 
 
 
 
 
 
 
8285
  if limit:
8286
  hits = hits[:limit]
8287
  src = str(bag.get("field") or "")
@@ -8289,10 +8591,20 @@ def compute_relation_cells(rt, table_key):
8289
  [r.get(src) for _i, r in hits] if src else [1] * len(hits))
8290
  if str(row.get(fk, "")) != want:
8291
  changes.setdefault(rid, {})[fk] = want
8292
- if not changes:
8293
- return 0
8294
-
8295
- def _up(cur):
 
 
 
 
 
 
 
 
 
 
8296
  cur = cur if isinstance(cur, dict) else {}
8297
  tt = cur.get(table_key)
8298
  if tt is not None:
@@ -8304,28 +8616,44 @@ def compute_relation_cells(rt, table_key):
8304
  return len(changes)
8305
 
8306
 
8307
- def refresh_relations(rt, log=print):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8308
  """The tick half of the relational pass β€” the twin of `refresh_metrics`.
8309
 
8310
  ⚠ Runs for EVERY table, because a link can point anywhere: a rollup on table A goes stale
8311
  when table B gains a row, and A has no way to know that happened. Cheap by construction β€” a
8312
  table declaring neither kind costs one dict scan.
8313
  """
8314
- touched = 0
8315
- # ⚠ FILTER OFF THE DICT ALREADY IN HAND, exactly as `refresh_metrics` does. Calling
8316
- # `compute_relation_cells` for every table would make it re-`ut_get` each one just to
8317
- # discover it declares neither kind β€” so the "costs one dict scan" claim above is only true
8318
- # with this line, and without it the sibling two functions down does strictly less work for
8319
- # the same job.
8320
- for tk, t in ut_all(rt).items():
8321
- fs = (t or {}).get("fields") or []
8322
- if not any(f.get("type") == "rollup" or _ut().is_derived_link(f) for f in fs):
8323
- continue
8324
- try:
8325
- touched += compute_relation_cells(rt, tk)
8326
- except Exception as e: # noqa: BLE001
8327
- log(f"[aios-auto] relation refresh {tk} failed: {type(e).__name__}: {e}")
8328
- return touched
 
 
8329
 
8330
 
8331
  def refresh_metrics(rt, today=None, log=print):
 
23
  """
24
  from __future__ import annotations
25
 
26
+ import datetime as _dt
27
+ import hashlib
28
+ import hmac
29
  import ipaddress
30
  import json
31
  import os
 
93
  #: have inherited a 200,000-row ceiling, and with it the measured 35.8 MB single-bucket
94
  #: serialisation cost, purely by an accident of naming. Naming the append tables is the version
95
  #: that stays true when the next `ut_ig_*` table is not one.
96
+ IG_SNAPSHOTS_TABLE = "ut_ig_snapshots"
97
+ IG_POSTS_TABLE = "ut_ig_posts"
98
+ IG_POST_SNAPSHOTS_TABLE = "ut_ig_post_snapshots"
99
+ IG_COMMENTS_TABLE = "ut_ig_comments"
100
+ AUTOMATION_RECORD_MODE = "automation"
101
+ APPEND_TABLES = frozenset({IG_SNAPSHOTS_TABLE, IG_POST_SNAPSHOTS_TABLE})
102
 
103
  #: ⭐ WAVE 24 (owner ruling R6) β€” `plain` IS WHAT AN AUTOMATION IS NOW, and it is the DEFAULT.
104
  #: The create wizard is deleted, so nobody picks a kind any more: a new automation is a trigger
 
628
  # THE UPSERT β€” pure, so the arithmetic is testable without a store or a network
629
  # ---------------------------------------------------------------------------------------------
630
 
631
+ def upsert_rows(existing, incoming, key_field, cap=None):
632
  """Merge scraped rows into a user table's rows BY KEY. Returns `(rows, counts)`.
633
 
634
  THE RULE THAT MATTERS: **an orphan is COUNTED, NEVER DELETED.** A row that has stopped
 
697
  counts["duplicates"] = incoming_dupes + len(dupe_ids)
698
  counts["orphans"] = sum(
699
  1 for kv, rid in by_key.items() if kv not in seen_keys and rid not in dupe_ids)
700
+ return rows, counts
701
+
702
+
703
+ def dedupe_canonical_rows(existing, key_field, newest_by=""):
704
+ """Collapse duplicate logical rows while preserving the lowest stable row id.
705
+
706
+ Canonical entity tables use this before every upsert. Snapshot tables deliberately do not:
707
+ repeated shortcodes there are new timestamped observations, not duplicate posts. Values are
708
+ taken newest-first and then filled from older rows, so a sparse fresh projection does not
709
+ erase a field an earlier row knew.
710
+ """
711
+ rows = {str(k): dict(v or {}) for k, v in (existing or {}).items()}
712
+ groups = {}
713
+ for rid, row in rows.items():
714
+ identity = str(row.get(key_field) or "").strip()
715
+ if identity:
716
+ groups.setdefault(identity, []).append((rid, row))
717
+ removed = 0
718
+ for members in groups.values():
719
+ if len(members) < 2:
720
+ continue
721
+ keep = min((rid for rid, _row in members), key=lambda r: (not r.isdigit(), int(r) if r.isdigit() else r))
722
+ ordered = sorted(members, key=lambda item: str(item[1].get(newest_by) or ""), reverse=True) \
723
+ if newest_by else members
724
+ merged = {}
725
+ for _rid, row in ordered:
726
+ for key, value in row.items():
727
+ if key not in merged or str(merged.get(key) or "").strip() == "":
728
+ merged[key] = value
729
+ rows[keep] = merged
730
+ for rid, _row in members:
731
+ if rid != keep:
732
+ rows.pop(rid, None)
733
+ removed += 1
734
+ return rows, removed
735
 
736
 
737
  # ---------------------------------------------------------------------------------------------
 
837
  MACHINE_OWNERS = ("automation", "scheduler")
838
 
839
 
840
+ def ut_ensure(rt, label, fields, username="automation", key=None, flow_tag="",
841
+ record_mode="", lock_fields=False):
842
  """Create the table if it is missing; return its key. Idempotent β€” a re-run of an automation
843
  that owns a table must not spawn `ut_x_2`, so the key is DERIVED from the label (or given)
844
  and an existing table with that key is adopted, not duplicated.
 
879
  # A migration that cannot run must not stop the automation from writing its rows.
880
  # The old schema still reads; a refused write loses the pull we just paid for.
881
  pass
882
+ wanted = []
883
+ for raw_field in (fields or []):
884
+ field = dict(raw_field)
885
+ if flow_tag or lock_fields:
886
+ automation = dict(field.get("automation") or {})
887
+ if flow_tag:
888
+ automation.setdefault("flowId", str(flow_tag))
889
+ if lock_fields:
890
+ automation["preset"] = True
891
+ field["automation"] = automation
892
+ wanted.append(field)
893
  created = _iso()
894
  human = username if username and username not in MACHINE_OWNERS else ""
895
 
896
  # ⚠ SKIP THE WRITE WHEN NOTHING WOULD CHANGE. Without this, every re-run spends a store
897
  # commit re-writing an identical definition β€” against a 20 s flush floor and a 256/hr repo
898
  # budget, an idempotent helper that always writes is the same defect as a per-row insert.
899
+ have = ut_get(rt, key)
900
+ have_fields = {str(f.get("key") or ""): f for f in ((have or {}).get("fields") or [])}
901
+ missing = [f for f in wanted if f.get("key") not in have_fields]
902
+ missing_locks = [f for f in wanted
903
+ if lock_fields and f.get("key") in have_fields
904
+ and (not isinstance(have_fields[f.get("key")].get("automation"), dict)
905
+ or have_fields[f.get("key")]["automation"].get("preset") is not True)]
906
+ if (have is not None and not missing and not missing_locks
907
+ and not (record_mode and have.get("recordMode") != record_mode)
908
+ and not (human and (have.get("createdBy") or "") in MACHINE_OWNERS)):
909
+ return key
910
 
911
  def _up(cur):
912
  cur = cur if isinstance(cur, dict) else {}
 
914
  if t is None:
915
  if len(cur) >= MAX_UT_TABLES:
916
  return cur
917
+ cur[key] = {"key": key, "label": str(label)[:60], "source": "Automation",
918
+ "createdBy": username, "created": created,
919
+ "fields": wanted, "rows": {}}
920
+ if record_mode:
921
+ cur[key]["recordMode"] = record_mode
922
+ return cur
923
  have = {f.get("key") for f in (t.get("fields") or [])}
924
+ for f in wanted:
925
+ if f.get("key") not in have:
926
+ t.setdefault("fields", []).append(f)
927
+ have.add(f.get("key"))
928
+ elif lock_fields:
929
+ stored = next((g for g in (t.get("fields") or [])
930
+ if g.get("key") == f.get("key")), None)
931
+ if stored is not None:
932
+ automation = dict(stored.get("automation") or {})
933
+ if flow_tag:
934
+ automation.setdefault("flowId", str(flow_tag))
935
+ automation["preset"] = True
936
+ stored["automation"] = automation
937
  # ADOPTION: a machine name is not an owner. It never overwrites a human one.
938
  if human and (t.get("createdBy") or "") in MACHINE_OWNERS:
939
  t["createdBy"] = human
940
  # An automation's table SAYS an automation owns it β€” the nav badge reads from this.
941
+ t["source"] = t.get("source") or "Automation"
942
+ if record_mode:
943
+ t["recordMode"] = record_mode
944
+ return cur
945
 
946
  rt.update(UT_STORE_KEY, _up, flush="sync")
947
  return key
 
1826
  arrives later. The run-time path reports that condition LOUDLY (`ut_missing`, D-11), so the
1827
  honest failure still has exactly one home.
1828
  """
1829
+ cfg = defn.get("config") or {}
1830
+ target = str(_flow_table(defn) or cfg.get("targetTable") or "")
1831
+ if not target:
1832
+ return
1833
+ try:
1834
+ if (defn.get("trigger") or {}).get("key") == "ig_profile_match":
1835
+ ut_ensure(rt, cfg.get("targetLabel") or "IG candidates", CANDIDATE_FIELDS, username,
1836
+ key=target, flow_tag=str(defn.get("id") or ""), lock_fields=True)
1837
+ return
1838
+ enrich_actions = _actions_of_kind(((defn.get("flow") or {}).get("actions") or []),
1839
+ "enrich_instagram")
1840
+ table = ut_get(rt, target) or {}
1841
+ named = next((str((a.get("config") or {}).get("profileField") or "").strip()
1842
+ for a in enrich_actions
1843
+ if str((a.get("config") or {}).get("profileField") or "").strip()), "")
1844
+ bound = next((str(f.get("key") or "") for f in table.get("fields") or []
1845
+ if isinstance(f.get("profile"), dict)), "") or named
1846
+ if enrich_actions and bound:
1847
+ ut_ensure(rt, table.get("label") or target, _profile_schema_for(bound), username,
1848
+ key=target, flow_tag=str(defn.get("id") or ""), lock_fields=True)
1849
  except Exception as e: # noqa: BLE001
1850
  # β›” NEVER FAILS THE SAVE. The definition is already committed by the time this runs, so
1851
  # raising here would answer 500 for an automation that IS stored β€” the caller would retry
 
2023
  field_def("is_professional", "Professional account"),
2024
  field_def("is_private", "Private"),
2025
  field_def("highlights_count", "Highlights"),
2026
+ field_def("bio_hashtags", "Bio hashtags"),
 
2027
  field_def("pronouns", "Pronouns"),
2028
  # ⭐ 2026-08-07 β€” the rest of the vendor's Profiles schema (see `_bd_profile`). The snapshot
2029
  # row maps the WHOLE schema by design, so these belong here the moment the map reads them;
 
2065
  # ⚠ BLANK, NEVER ZERO. `postMetrics` is off by default (it buys one vendor record per post),
2066
  # so on most pulls these stay empty β€” and empty means "not read", which is what makes an
2067
  # honest average possible at all. A 0 here would claim a post nobody watched.
2068
+ field_def("views", "Views", "int"),
2069
+ # Bright Data's Reels schema exposes BOTH views and video play count. They are not aliases:
2070
+ # its own example returns 388 views and 1,890 plays for one Reel. Keeping two columns avoids
2071
+ # a plausible-looking "Views" value that is actually repeat plays.
2072
+ field_def("plays", "Plays", "int"),
2073
+ field_def("likes", "Likes", "int"),
2074
  field_def("comments", "Comments", "int"),
2075
  field_def("measured_at", "Engagement read at", "date"),
2076
  # ⭐ 2026-08-07 β€” the second half of "spawn relevant Post/Comment database that is LINKED":
 
2079
  # ⚠ It resolves to nothing until comment capture is switched on, which is the honest state
2080
  # for a relation whose far side is empty β€” the same standing `ut_ig_post_snapshots` has when
2081
  # `postMetrics` is off.
2082
+ field_def("comments_link", "Comment rows", "link",
2083
+ link={"table": IG_COMMENTS_TABLE, "on": "shortcode", "from": "shortcode"}),
2084
+ field_def("post_snapshots_link", "Measurement rows", "link",
2085
+ link={"table": IG_POST_SNAPSHOTS_TABLE, "on": "shortcode", "from": "shortcode"}),
2086
+ field_def("measurements_captured", "Measurements captured", "rollup",
2087
+ rollup={"link": "post_snapshots_link", "fn": "countall"}),
2088
  ]
2089
 
2090
  #: ⭐⭐ 2026-08-07 (owner instruction) β€” THE COMMENT DATABASE.
 
2116
  field_def("comment_key", "Comment"), field_def("shortcode", "Shortcode"),
2117
  field_def("influencer_key", "Influencer"),
2118
  field_def("commented_at", "Commented at", "date"),
2119
+ field_def("likes", "Likes", "int"),
2120
+ field_def("replies", "Replies", "int"),
2121
+ field_def("post_link", "Post", "link",
2122
+ link={"table": IG_POSTS_TABLE, "on": "shortcode", "from": "shortcode",
2123
+ "single": True}),
2124
  # ⚠ NO AUTHOR COLUMN AND NO TEXT COLUMN, and their absence is the ruling rather than an
2125
  # oversight. `comment_user` is vendor-flagged PII and the text is the payload D-22 priced.
2126
  # What is left is the SHAPE of a thread β€” how many, how recent, how engaged β€” which is the
2127
  # part that informs an influencer decision without ingesting a stranger.
2128
  ]
2129
  POST_SNAPSHOT_FIELDS = [
2130
+ field_def("post_snapshot_key", "Snapshot"), field_def("shortcode", "Shortcode"),
2131
+ field_def("influencer_key", "Influencer"),
2132
+ field_def("pulled_at", "Pulled at"), field_def("likes", "Likes"),
2133
+ field_def("comments", "Comments"),
2134
  # Plays/views move independently of likes on video, so it is its own series rather than a
2135
  # thing to derive. Paid rung only; blank means not read.
2136
+ field_def("views", "Views", "int"),
2137
+ field_def("plays", "Plays", "int"),
2138
+ field_def("post_link", "Post", "link",
2139
+ link={"table": IG_POSTS_TABLE, "on": "shortcode", "from": "shortcode",
2140
+ "single": True}),
2141
+ ]
2142
 
2143
 
2144
  def ig_handle(url):
 
2284
  BD_BASE_DEFAULT = "https://api.brightdata.com"
2285
  BD_DS_PROFILES = "gd_l1vikfch901nx3by4" # Instagram – Profiles. 36 fields, 620M records
2286
  BD_DS_POSTS = "gd_lk5ns7kz21pck8jpis" # Instagram – Posts. 43 fields
2287
+ BD_DS_REELS = "gd_lyclm20il4r5helnj" # Instagram Reels: views and play counts
2288
+ BD_PATH_SCRAPE = "/datasets/v3/scrape" # SYNC: rows come back inline. param: dataset_id
2289
  BD_PATH_TRIGGER = "/datasets/v3/trigger" # ASYNC: -> {"snapshot_id": "sd_…"}
2290
  BD_PATH_SNAPSHOT = "/datasets/v3/snapshot" # /<sd_id>?format=json -> the rows
2291
  BD_PATH_FILTER = "/datasets/filter" # ⚠ NO /v3/ β€” the CORPUS query (discovery)
 
2595
  "is_professional": _bd_flag(node, "is_professional_account"),
2596
  "is_private": _bd_flag(node, "is_private"),
2597
  "highlights_count": _ig_int(_first(node, "highlights_count")),
2598
+ "bio_hashtags": _bd_list(node, "bio_hashtags"),
 
2599
  "pronouns": str(_first(node, "pronouns", default="") or ""),
2600
  # ⭐ 2026-08-07 (owner instruction: *"have ALL Fields available to us from Bright Data to
2601
  # be pre-set Fields for us and populated"*) β€” THE REST OF THE PROFILES SCHEMA.
 
2651
  code = ig_shortcode(p.get("url")) if isinstance(p, dict) else ""
2652
  if not code:
2653
  return None
2654
+ kind = _bd_type(p.get("content_type"), p.get("type"))
2655
+ # The URL route is part of Bright Data's endpoint contract. A Reel sent through the Posts
2656
+ # scraper can return likes/comments while omitting its view fields; `/reel/{shortcode}` is
2657
+ # the input documented for the Reels scraper.
2658
+ route = "reel" if kind == "video" else "p"
2659
+ out = {"shortcode": code, "url": f"https://www.instagram.com/{route}/{code}/",
2660
+ "type": kind}
2661
  caption = _first(p, "caption", "description", default="")
2662
  if isinstance(caption, dict):
2663
  caption = _first(caption, "text", default="")
 
2697
  "caption": str(_first(row, "description", "caption", default="") or ""),
2698
  "likes": _ig_int(_first(row, "likes", "like_count")),
2699
  "comments": _ig_int(_first(row, "num_comments", "comment_count", "comments")),
2700
+ # Reels expose views and plays separately (and the counts can differ materially). Keep
2701
+ # them separate so a repeat-play count never wears the "Views" label.
2702
+ "views": _ig_int(_first(row, "views", "view_count", "video_view_count",
2703
+ "engagement_score_view")),
2704
+ "plays": _ig_int(_first(row, "video_play_count", "play_count")),
2705
  # Sponsored-post detection. MEASURED populated (`True`, with the brand alongside) β€” and
2706
  # it is the one field here that answers a commercial question the counts cannot.
2707
  "paid_partnership": _bd_flag(row, "is_paid_partnership"),
 
2930
  if got:
2931
  posts.append(got)
2932
 
2933
+ if post_metrics and posts:
2934
+ time.sleep(min(PACE_SECONDS, 1.0)) # the vendor is paid, but it is still someone's API
2935
+ # Posts and Reels are different Bright Data endpoints. Sending every permalink to the
2936
+ # Posts endpoint was why Inayma's five video rows got likes/comments but no views: the
2937
+ # documented Reels response is where `views` and `video_play_count` live. Split the batch
2938
+ # without duplicating any record, so enabling post metrics still buys one row per post.
2939
+ ordinary = [p for p in posts if p.get("type") != "video"]
2940
+ videos = [p for p in posts if p.get("type") == "video"]
2941
+ rows2, metric_notes = [], []
2942
+ if ordinary:
2943
+ got_rows, got_note = bd_scrape(BD_DS_POSTS, [p["url"] for p in ordinary])
2944
+ rows2.extend(got_rows)
2945
+ if got_note:
2946
+ metric_notes.append(f"posts: {got_note}")
2947
+ if videos:
2948
+ got_rows, got_note = bd_scrape(BD_DS_REELS, [p["url"] for p in videos])
2949
+ rows2.extend(got_rows)
2950
+ if got_note:
2951
+ metric_notes.append(f"reels: {got_note}")
2952
+ if metric_notes and not rows2:
2953
+ # The engagement half refusing does NOT lose the identity half β€” the profile and the
2954
+ # post rows still land, and the run says which part was not readable.
2955
+ note = f"post metrics unavailable: {'; '.join(metric_notes)}"
2956
+ else:
2957
+ by_code = {}
2958
  for r in rows2:
2959
  got = _bd_post_metrics(r)
2960
  if got:
 
2965
  # The metrics row is RICHER (it knows posted_at); merge it over the identity
2966
  # row, dropping keys it could not read so a blank never overwrites a value.
2967
  p.update({k: v for k, v in extra.items() if v not in (None, "")})
2968
+ if not by_code:
2969
+ note = "post metrics were requested but the Posts dataset returned no rows"
2970
+ elif metric_notes:
2971
+ note = f"some post metrics were unavailable: {'; '.join(metric_notes)}"
2972
 
2973
  # IDENTITY WITHOUT MEDIA IS STILL `partial`, on the paid rung too. The rule does not soften
2974
  # because we are paying: a run that wrote a follower count and no posts must not paint green
 
3419
  field_def("business_category", "Business category"),
3420
  field_def("is_private", "Private account", "checkbox"),
3421
  field_def("bio_hashtags", "Bio hashtags"),
 
3422
  field_def("pronouns", "Pronouns"),
3423
  field_def("profile_name", "Profile name"),
3424
  field_def("is_joined_recently", "Joined recently", "checkbox"),
 
3447
  # identically-named columns is unreadable. Caught by the preset set's own label-collision
3448
  # gate rather than on screen, which is what that gate is for.
3449
  field_def("posts_link", "Post rows", "link",
3450
+ link={"table": IG_POSTS_TABLE, "on": "influencer_key"}),
3451
+ field_def("profile_snapshots_link", "Profile history", "link",
3452
+ link={"table": IG_SNAPSHOTS_TABLE, "on": "influencer_key"}),
3453
+ field_def("post_snapshots_link", "Post measurement rows", "link",
3454
+ link={"table": IG_POST_SNAPSHOTS_TABLE, "on": "influencer_key"}),
3455
+ field_def("comments_link", "Comment rows", "link",
3456
+ link={"table": IG_COMMENTS_TABLE, "on": "influencer_key"}),
3457
  # ⚠ THE ROLLUPS READ `ut_ig_posts`' OWN LATEST COLUMNS, which is why those exist β€” a rollup
3458
  # is ONE HOP (Airtable's rule and ours), and the engagement SERIES lives one table further
3459
  # out in `ut_ig_post_snapshots`.
 
3463
  # has to name what makes one post later than another.
3464
  field_def("avg_views_12", "Avg views Β· last 12 posts", "rollup",
3465
  rollup={"link": "posts_link", "field": "views", "fn": "average",
3466
+ "limit": MAX_POSTS_PER_PULL, "sortBy": "posted_at", "sortDir": "desc",
3467
+ "distinctBy": "shortcode"}),
3468
+ field_def("avg_plays_12", "Avg plays - last 12 posts", "rollup",
3469
+ rollup={"link": "posts_link", "field": "plays", "fn": "average",
3470
+ "limit": MAX_POSTS_PER_PULL, "sortBy": "posted_at", "sortDir": "desc",
3471
+ "distinctBy": "shortcode"}),
3472
  field_def("avg_likes_12", "Avg likes Β· last 12 posts", "rollup",
3473
  rollup={"link": "posts_link", "field": "likes", "fn": "average",
3474
+ "limit": MAX_POSTS_PER_PULL, "sortBy": "posted_at", "sortDir": "desc",
3475
+ "distinctBy": "shortcode"}),
3476
  field_def("avg_comments_12", "Avg comments Β· last 12 posts", "rollup",
3477
  rollup={"link": "posts_link", "field": "comments", "fn": "average",
3478
+ "limit": MAX_POSTS_PER_PULL, "sortBy": "posted_at", "sortDir": "desc",
3479
+ "distinctBy": "shortcode"}),
3480
  # ⭐ THE ONE HONEST POST COUNT WE HAVE. D-82: the vendor's `posts_count` is a FABRICATED ZERO
3481
  # on the paid rung (49/49 rows measured), so it is discarded and that column reads blank after
3482
  # a paid enrich. This counts the post rows actually captured β€” a different number and a true
3483
  # one, which is why it gets its own column and its own label rather than quietly filling
3484
  # `posts_count` with something that is not what that field means.
3485
  field_def("posts_captured", "Posts captured", "rollup",
3486
+ rollup={"link": "posts_link", "fn": "countall", "distinctBy": "shortcode"}),
3487
+ field_def("profile_reads", "Profile reads", "rollup",
3488
+ rollup={"link": "profile_snapshots_link", "fn": "countall",
3489
+ "distinctBy": "snapshot_key"}),
3490
+ field_def("post_measurements_captured", "Post measurements captured", "rollup",
3491
+ rollup={"link": "post_snapshots_link", "fn": "countall",
3492
+ "distinctBy": "post_snapshot_key"}),
3493
+ field_def("comments_captured", "Comments captured", "rollup",
3494
+ rollup={"link": "comments_link", "fn": "countall",
3495
+ "distinctBy": "comment_key"}),
3496
  # ⭐ R3's stamp. WITHOUT IT THE WHOLE SET IS UNREADABLE: a blank `followers` means "never
3497
  # enriched" and a stale one means "enriched in March", and no cell on the row can tell them
3498
  # apart. It is the single field that turns the other fifteen from numbers into measurements.
 
3504
  # views/likes/comments at a `pulled_at`). This cell is a DERIVED window over those two,
3505
  # rewritten each run, so a person reading the profile row can see the recent posts without a
3506
  # join β€” and deleting it would cost a convenience, never a measurement. Shape: contract C2.
3507
+ ]
 
3508
  #: The keys the preset set owns β€” derived, so a field added above cannot be forgotten here.
3509
  PRESET_PROFILE_KEYS = tuple(f["key"] for f in PRESET_PROFILE_FIELDS)
3510
 
 
3524
  #: the failure C1 exists to prevent. The five below are facts about the SEARCH (how often we found
3525
  #: them, who for, and a human's decision) rather than about the profile, so they are discovery's
3526
  #: and not part of the cross-tenant set.
3527
+ CANDIDATE_FIELDS = [
3528
  *PRESET_PROFILE_FIELDS,
3529
  # ⭐ WAVE 26 Β· R3 β€” `first_found` / `last_found` ARE DATES, not ISO strings in a text cell.
3530
  # They were written by `_iso()`, so the cell read `2026-08-05T14:03:11+07:00` and the owner
 
3547
  # column: every candidate carried BOTH a `stage_<auto>` select saying where it was AND a
3548
  # boolean saying whether it was kept β€” two progress fields that could disagree, with no rule
3549
  # about which one won. The stage field is the one progress column. Nothing replaces this.
3550
+ ]
3551
+
3552
+
3553
+ def _profile_backlink_field(table_key, table_label, profile_key):
3554
+ """A deterministic reciprocal link from one canonical IG table to one profile database.
3555
+
3556
+ The key includes the target table identity, so ten separate Profile databases can all point
3557
+ into the same canonical Posts/Comments/history tables without one relation overwriting the
3558
+ next. The join is derived in both directions and therefore needs no cross-table fan-out write.
3559
+ """
3560
+ digest = hashlib.sha1(str(table_key).encode("utf-8")).hexdigest()[:10]
3561
+ return field_def(
3562
+ f"profiles_{digest}", f"Profiles - {str(table_label or table_key)[:48]}", "link",
3563
+ link={"table": str(table_key), "on": str(profile_key), "from": "influencer_key"},
3564
+ )
3565
+
3566
+
3567
+ def _profile_schema_for(bound_key):
3568
+ """Preset fields for an existing Profile table without inventing a second identity column."""
3569
+ out = []
3570
+ for field in PRESET_PROFILE_FIELDS:
3571
+ item = dict(field)
3572
+ if item.get("key") == PRESET_FLAG_KEY and bound_key != PRESET_FLAG_KEY:
3573
+ item.pop("profile", None)
3574
+ item.pop("pinned", None)
3575
+ out.append(item)
3576
+ return out
3577
+
3578
+
3579
+ def _reconcile_ig_graph_fields(rt, wanted_by_table, drop_by_table=None):
3580
+ """Repair machine-owned IG field declarations in one coalesced store update.
3581
+
3582
+ `ut_ensure` deliberately merges only missing keys. That is correct for user columns but not
3583
+ sufficient for a canonical relation: a stale `link.table`, rollup function, or type would
3584
+ survive forever. This pass overwrites the contract keys for the fields we own, preserves
3585
+ unrelated/user fields, and removes only explicitly retired preset columns plus their cells.
3586
+ """
3587
+ drop_by_table = drop_by_table or {}
3588
+ changed = False
3589
+
3590
+ def _up(cur):
3591
+ nonlocal changed
3592
+ cur = cur if isinstance(cur, dict) else {}
3593
+ for table_key, wanted_fields in wanted_by_table.items():
3594
+ table = cur.get(table_key)
3595
+ if table is None:
3596
+ continue
3597
+ wanted = {str(f.get("key")): dict(f) for f in wanted_fields}
3598
+ drops = set(drop_by_table.get(table_key) or ())
3599
+ fields, seen = [], set()
3600
+ for stored in table.get("fields") or []:
3601
+ key = str(stored.get("key") or "")
3602
+ if key in drops:
3603
+ changed = True
3604
+ continue
3605
+ desired = wanted.get(key)
3606
+ if desired is None:
3607
+ fields.append(stored)
3608
+ continue
3609
+ repaired = dict(stored)
3610
+ repaired.update(desired)
3611
+ # These bags define behaviour, not decoration. If the desired field does not use
3612
+ # one, a stale bag from an earlier type must not remain attached to it.
3613
+ for bag in ("link", "rollup"):
3614
+ if bag not in desired:
3615
+ repaired.pop(bag, None)
3616
+ if repaired != stored:
3617
+ changed = True
3618
+ fields.append(repaired)
3619
+ seen.add(key)
3620
+ for key, desired in wanted.items():
3621
+ if key not in seen and not any(str(f.get("key") or "") == key for f in fields):
3622
+ fields.append(desired)
3623
+ changed = True
3624
+ if drops:
3625
+ for row in (table.get("rows") or {}).values():
3626
+ for key in drops:
3627
+ if key in row:
3628
+ row.pop(key, None)
3629
+ changed = True
3630
+ table["fields"] = fields
3631
+ return cur
3632
+
3633
+ # Avoid a commit when the declarations are already exact.
3634
+ snapshot = ut_all(rt)
3635
+ needs = False
3636
+ for table_key, wanted_fields in wanted_by_table.items():
3637
+ table = snapshot.get(table_key) or {}
3638
+ by_key = {str(f.get("key") or ""): f for f in table.get("fields") or []}
3639
+ drops = set(drop_by_table.get(table_key) or ())
3640
+ if drops & set(by_key):
3641
+ needs = True
3642
+ break
3643
+ for desired in wanted_fields:
3644
+ stored = by_key.get(str(desired.get("key") or ""))
3645
+ if stored is None or any(stored.get(k) != v for k, v in desired.items()):
3646
+ needs = True
3647
+ break
3648
+ if needs:
3649
+ break
3650
+ if needs:
3651
+ rt.update(UT_STORE_KEY, _up, flush="sync")
3652
+ return changed
3653
+
3654
+
3655
+ def ensure_ig_graph(rt, username="automation", flow_tag="", profile_table="", profile_field=""):
3656
+ """Ensure the ONE per-tenant Instagram relational graph and return its canonical keys.
3657
+
3658
+ Every enrichment path calls this function. Fixed table keys prevent a flow aimed at a new
3659
+ Profile database from spawning `IG posts 2`; deterministic reciprocal link fields connect
3660
+ each Profile database to the same Posts, Comments, profile-history, and post-history stores.
3661
+ """
3662
+ profile = ut_get(rt, profile_table) if profile_table else None
3663
+ bound = str(profile_field or "").strip()
3664
+ if profile is not None and not bound:
3665
+ bound = next((str(f.get("key") or "") for f in profile.get("fields") or []
3666
+ if isinstance(f.get("profile"), dict)), "")
3667
+ profile_label = str((profile or {}).get("label") or profile_table)
3668
+ backlink = [_profile_backlink_field(profile_table, profile_label, bound)] \
3669
+ if profile_table and profile is not None and bound else []
3670
+
3671
+ graph = {
3672
+ IG_SNAPSHOTS_TABLE: ("IG snapshots", [*SNAPSHOT_FIELDS, *backlink]),
3673
+ IG_POSTS_TABLE: ("IG posts", [*POST_FIELDS, *backlink]),
3674
+ IG_POST_SNAPSHOTS_TABLE: ("IG post snapshots", [*POST_SNAPSHOT_FIELDS, *backlink]),
3675
+ IG_COMMENTS_TABLE: ("IG comments", [*COMMENT_FIELDS, *backlink]),
3676
+ }
3677
+ keys = {}
3678
+ for key, (label, fields) in graph.items():
3679
+ keys[key] = ut_ensure(rt, label, fields, username, key=key, flow_tag=flow_tag,
3680
+ record_mode=AUTOMATION_RECORD_MODE, lock_fields=True)
3681
+
3682
+ # `ut_ensure` stamps newly created fields with the first flow that introduced them. Do not
3683
+ # put that tag into the reconciliation template: these canonical tables are shared by many
3684
+ # Profile databases, and otherwise every run would rewrite their provenance from flow A to
3685
+ # flow B and back again.
3686
+ wanted = {key: [dict(f) for f in fields]
3687
+ for key, (_label, fields) in graph.items()}
3688
+ drops = {IG_SNAPSHOTS_TABLE: {"post_hashtags"}}
3689
+ if profile_table and profile is not None and bound:
3690
+ profile_fields = _profile_schema_for(bound)
3691
+ ut_ensure(rt, profile_label or profile_table, profile_fields, username,
3692
+ key=profile_table, flow_tag=flow_tag, lock_fields=True)
3693
+ wanted[profile_table] = [dict(f) for f in profile_fields]
3694
+ drops[profile_table] = {"posts", "post_hashtags"}
3695
+ _reconcile_ig_graph_fields(rt, wanted, drops)
3696
+ return keys
3697
 
3698
 
3699
  # ── ⭐ WAVE 26 Β· THE MIGRATION (owner rulings R3 + R4/R5, contracts C1-a and C3) ───────────────
 
3843
  # ⭐ 2026-08-07 (owner ruling) β€” the primary-column half. Counted separately from
3844
  # `stamped` because they answer different questions: that one is "how many ROWS got a
3845
  # platform", these are "how many TABLES changed shape".
3846
+ "pinned": 0, "flagged": 0, "droppedName": 0,
3847
+ "droppedRecentPosts": 0, "droppedPostHashtags": 0}
3848
  want = {f["key"]: f["type"] for f in CANDIDATE_FIELDS}
3849
  # β›” `only` NARROWS THE SET, IT DOES NOT BYPASS THE TEST β€” and skipping that cost six red
3850
  # checks the first time. `ut_ensure` calls this with the key it is ABOUT TO CREATE, so on a
 
3886
  other_profile = next((f for f in fields if f is not flag_f
3887
  and isinstance(f.get("profile"), dict)), None)
3888
  want_pin = pin_f is not None and pin_f.get("pinned") is not True
3889
+ want_flag = (flag_f is not None and other_profile is None
3890
+ and not isinstance(flag_f.get("profile"), dict))
3891
+ vestigial = _vestigial_name_field(fields, rows)
3892
+ retired = {f.get("key") for f in fields} & {"posts", "post_hashtags"}
3893
+ if (not stale_keys and has_platform and not needs_stamp
3894
+ and not want_pin and not want_flag and vestigial is None and not retired):
3895
+ continue
3896
  stats["tables"] += 1
3897
  if want_pin:
3898
  pin_f["pinned"] = True
 
3900
  if want_flag:
3901
  flag_f["profile"] = {"source": PROFILE_SOURCE_IG}
3902
  stats["flagged"] += 1
3903
+ if vestigial is not None:
3904
  # ⚠ THE CELLS GO WITH THE COLUMN. A row dict keeping a `name` key whose field no longer
3905
  # exists is invisible everywhere except the next export, where it reappears as a column
3906
  # nobody declared. `_vestigial_name_field` has already proven every one of them blank.
3907
  fields = [f for f in fields if f is not vestigial]
3908
  for r in rows.values():
3909
  r.pop("name", None)
3910
+ stats["droppedName"] += 1
3911
+ if retired:
3912
+ fields = [f for f in fields if f.get("key") not in retired]
3913
+ for r in rows.values():
3914
+ for key in retired:
3915
+ r.pop(key, None)
3916
+ stats["droppedRecentPosts"] += int("posts" in retired)
3917
+ stats["droppedPostHashtags"] += int("post_hashtags" in retired)
3918
 
3919
  for key in stale_keys:
3920
  conv = _MIGRATE_CONVERT.get(key)
 
4892
  ("alt_text", 400)):
4893
  if p.get(k):
4894
  ident[k] = _s(p.get(k), n)
4895
+ metrics = {k: p.get(k) for k in ("likes", "comments", "views", "plays")}
4896
  # ⭐⭐ 2026-08-07 β€” THE LATEST ENGAGEMENT VALUES, ONTO THE POST ROW ITSELF.
4897
  #
4898
  # This is what keeps a rollup at ONE HOP (Airtable's rule and ours): without it, "average
 
4918
  ident["measured_at"] = _day(pulled)
4919
  idents.append(ident)
4920
  if any(v is not None for v in metrics.values()):
4921
+ metrics_rows.append({
4922
+ "post_snapshot_key": f"{p['shortcode']}@{pulled}",
4923
+ "shortcode": p["shortcode"], "influencer_key": prof.get("username"),
4924
+ "pulled_at": pulled,
4925
+ "likes": _s(metrics["likes"]), "comments": _s(metrics["comments"]),
4926
+ "views": _s(metrics["views"]), "plays": _s(metrics["plays"])})
4927
  return snap_row, idents, metrics_rows
4928
 
4929
 
 
4946
  # `PRESET_PROFILE_KEYS` entry must either be written by this map or be explicitly declared
4947
  # as written elsewhere (`platform`, `enriched_at`, `posts` are stamped by `preset_cells`).
4948
  "business_category": "business_category", "is_private": "is_private",
4949
+ "bio_hashtags": "bio_hashtags",
4950
  "pronouns": "pronouns", "profile_name": "profile_name",
4951
  "is_joined_recently": "is_joined_recently", "has_channel": "has_channel",
4952
  "partner_id": "partner_id", "external_url_title": "external_url_title",
 
4961
  #: ⚠ The five relational columns are written by `compute_relation_cells` on the tick, NOT by an
4962
  #: enrichment run β€” which is the whole point of them: they stay true when the LINKED table
4963
  #: changes, and a pull that touched no profile still updates a profile's post count.
4964
+ PRESET_WRITTEN_ELSEWHERE = (
4965
+ "platform", "handle", "enriched_at",
4966
+ "posts_link", "profile_snapshots_link", "post_snapshots_link", "comments_link",
4967
+ "avg_views_12", "avg_plays_12", "avg_likes_12", "avg_comments_12",
4968
+ "posts_captured", "profile_reads", "post_measurements_captured", "comments_captured",
4969
+ )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4970
 
 
 
 
 
 
 
 
4971
 
4972
+ def preset_cells(res, pulled):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4973
  """C4/R3: one pull β†’ the LATEST-value cells written onto the enriched record.
4974
 
4975
  β›” `enriched_at` IS ALWAYS WRITTEN when a pull succeeded, and it is the field that makes the
 
4991
  # arrived with no platform at all β€” and a blank half of the dedup key is how one account
4992
  # becomes two rows.
4993
  cells["platform"] = PLATFORM_INSTAGRAM
 
 
 
4994
  # R3: `enriched_at` is a `date` column now. It answers "how stale is this number", which is a
4995
  # question in days; the full stamp keeps its precision on the snapshot series.
4996
  cells["enriched_at"] = _day(pulled)
 
5028
  # the relational tables the pull lands in (R7): every row timestamped for time-range filters
5029
  if dry:
5030
  # The Write node is off: resolve the keys, create nothing. (`ut_ensure` writes.)
5031
+ snap_key, post_key, ps_key = (IG_SNAPSHOTS_TABLE, IG_POSTS_TABLE,
5032
+ IG_POST_SNAPSHOTS_TABLE)
5033
+ else:
5034
+ graph = ensure_ig_graph(rt, username, str(defn.get("id") or ""),
5035
+ profile_table=table_key)
5036
+ snap_key, post_key, ps_key = (graph[IG_SNAPSHOTS_TABLE], graph[IG_POSTS_TABLE],
5037
+ graph[IG_POST_SNAPSHOTS_TABLE])
 
 
 
 
 
 
 
 
5038
  missing = [] if dry else ut_missing(rt, snap_key, post_key, ps_key)
5039
  snaps = dict((ut_get(rt, snap_key) or {}).get("rows") or {})
5040
  posts = dict((ut_get(rt, post_key) or {}).get("rows") or {})
 
5106
 
5107
  # --- THE THREE UPSERTS. Once each, over the whole run's accumulated rows.
5108
  snaps, c_snap = upsert_rows(snaps, in_snaps, "snapshot_key", cap=row_cap(snap_key))
5109
+ posts, collapsed_posts = dedupe_canonical_rows(posts, "shortcode", newest_by="measured_at")
5110
+ posts, c_post = upsert_rows(posts, in_posts, "shortcode", cap=row_cap(post_key))
5111
+ c_post["duplicates"] += collapsed_posts
5112
  psnaps, c_ps = upsert_rows(psnaps, in_psnaps, "post_snapshot_key", cap=row_cap(ps_key))
5113
  counts["new_posts"] = c_post["inserted"]
5114
  capped = [(snap_key, c_snap["capped"]), (post_key, c_post["capped"]),
 
5142
  tt.setdefault("rows", {}).setdefault(str(rid), {})[fkey] = val
5143
  for rid, vals in stage_writes.items():
5144
  tt.setdefault("rows", {}).setdefault(str(rid), {}).update(vals)
5145
+ for k, rws in ((snap_key, snaps), (post_key, posts), (ps_key, psnaps)):
5146
+ tgt = cur.get(k)
5147
+ if tgt is not None:
5148
+ tgt["rows"] = rws
5149
+ _refresh_relations_inplace(cur, log=log)
5150
+ return cur
5151
+
5152
+ rt.update(UT_STORE_KEY, _up, flush="sync") # ONE coalesced update for all four tables
5153
+ if lanes:
5154
  ensure_stage_field(rt, table_key, defn, username)
5155
 
5156
  # --- C6 (R2): WRITE-THROUGH to the platform master. Three postures, never conflated:
 
5398
  # down the rows path and parses as one unusable row instead of an error.
5399
  counts["dropped"] = max(0, len(rows) - len(incoming))
5400
  label = cfg.get("targetLabel") or "IG candidates"
5401
+ table_key = (ut_key_for(label, cfg.get("targetTable") or DISCOVER_TABLE) if dry
5402
+ else ut_ensure(rt, label, CANDIDATE_FIELDS, username,
5403
+ key=cfg.get("targetTable") or DISCOVER_TABLE,
5404
+ flow_tag=str(defn.get("id") or ""), lock_fields=True))
5405
  existing = dict((ut_get(rt, table_key) or {}).get("rows") or {})
5406
  # --- ⭐ WAVE 26 Β· C3 / owner ruling R4 β€” THE UPSERT KEY IS `(platform, handle)`.
5407
  #
 
7080
  return picked, "; ".join(notes)
7081
 
7082
 
7083
+ def _has_action(actions, kind):
7084
  """Does this flow contain `kind` ANYWHERE, forks included?
7085
 
7086
  β›” FORKS ARE THE WHOLE REASON THIS IS A FUNCTION. A group's children live under
 
7097
  for br in ((a.get("config") or {}).get("branches") or []):
7098
  if _has_action((br or {}).get("actions") or [], kind):
7099
  return True
7100
+ return False
7101
+
7102
+
7103
+ def _actions_of_kind(actions, kind):
7104
+ """Every action of `kind`, including actions nested inside If branches."""
7105
+ out = []
7106
+ for action in actions or []:
7107
+ if not isinstance(action, dict):
7108
+ continue
7109
+ if action.get("kind") == kind:
7110
+ out.append(action)
7111
+ for branch in ((action.get("config") or {}).get("branches") or []):
7112
+ out.extend(_actions_of_kind((branch or {}).get("actions") or [], kind))
7113
+ return out
7114
 
7115
 
7116
  def apply_actions(rt, defn, table_key, row_ids, username="automation", log=print):
 
7180
  # step at it, and (b) add a SECOND profile column to a table that already flagged a different
7181
  # one, which `user_tables` forbids at both write doors. A table with no profile column is an
7182
  # UNBOUND enrich: it must stay unbound and say so, which is a different defect (D-79) with its
7183
+ # own honest refusal, not something to paper over by inventing the binding.
7184
+ _tbl_now = ut_get(rt, table) or {}
7185
+ _enrich_actions = _actions_of_kind(actions, "enrich_instagram")
7186
+ _bound = next((f for f in (_tbl_now.get("fields") or [])
7187
+ if isinstance(f.get("profile"), dict)), None)
7188
+ if _bound is None:
7189
+ _named = next((str((a.get("config") or {}).get("profileField") or "").strip()
7190
+ for a in _enrich_actions
7191
+ if str((a.get("config") or {}).get("profileField") or "").strip()), "")
7192
+ _bound = next((f for f in (_tbl_now.get("fields") or [])
7193
+ if str(f.get("key") or "") == _named), None)
7194
+ if _bound and _enrich_actions:
7195
  # ⚠ AND THE DECLARATIONS ARE STRIPPED FROM ANYTHING NEW. The binding already exists β€”
7196
  # `_bound` is it β€” so a preset arriving now must carry DATA, never a second identity: a
7197
  # table whose profile column is `ig_handle` would otherwise gain a rival `handle`.
7198
  topup = [({k: v for k, v in f.items() if k not in ("profile", "pinned")}
7199
  if f.get("key") != _bound.get("key") else dict(f))
7200
  for f in PRESET_PROFILE_FIELDS]
7201
+ try:
7202
+ if any(not bool((a.get("config") or {}).get("dryRun")) for a in _enrich_actions):
7203
+ ensure_ig_graph(rt, username, str(defn.get("id") or ""),
7204
+ profile_table=table, profile_field=str(_bound.get("key") or ""))
7205
+ else:
7206
+ # Dry run keeps its original no-create contract; only the already-established
7207
+ # Profile table is topped up, and no canonical related database is spawned.
7208
+ ut_ensure(rt, _tbl_now.get("label") or table, topup, username, key=table,
7209
+ flow_tag=str(defn.get("id") or ""), lock_fields=True)
7210
  except Exception as exc: # noqa: BLE001
7211
  # A schema top-up that cannot run must not stop the enrichment: the cells that DO
7212
  # have columns still land, which is strictly better than the pull being thrown away.
 
7489
  if acc.get("dry"):
7490
  out["enrichDryRun"] = acc["profiles"]
7491
  return out
7492
+ tag = str(defn.get("id") or "")
7493
+ profile_table = str(_flow_table(defn) or "")
7494
+ graph = ensure_ig_graph(rt, username, tag, profile_table=profile_table)
7495
+ snap_key, post_key, ps_key = (graph[IG_SNAPSHOTS_TABLE], graph[IG_POSTS_TABLE],
7496
+ graph[IG_POST_SNAPSHOTS_TABLE])
 
 
 
 
 
7497
  missing = ut_missing(rt, snap_key, post_key, ps_key)
7498
  snaps, c_snap = upsert_rows(dict((ut_get(rt, snap_key) or {}).get("rows") or {}),
7499
  acc["snaps"], "snapshot_key", cap=row_cap(snap_key))
7500
+ old_posts, collapsed_posts = dedupe_canonical_rows(
7501
+ dict((ut_get(rt, post_key) or {}).get("rows") or {}), "shortcode", newest_by="measured_at")
7502
+ posts, c_post = upsert_rows(old_posts, acc["posts"], "shortcode", cap=row_cap(post_key))
7503
+ c_post["duplicates"] += collapsed_posts
7504
  psnaps, c_ps = upsert_rows(dict((ut_get(rt, ps_key) or {}).get("rows") or {}),
7505
  acc["psnaps"], "post_snapshot_key", cap=row_cap(ps_key))
7506
 
7507
  def _up(cur):
7508
  cur = cur if isinstance(cur, dict) else {}
7509
+ for k, rws in ((snap_key, snaps), (post_key, posts), (ps_key, psnaps)):
7510
+ if cur.get(k) is not None:
7511
+ cur[k]["rows"] = rws
7512
+ _refresh_relations_inplace(cur, log=log)
7513
+ return cur
7514
+
7515
+ rt.update(UT_STORE_KEY, _up, flush="sync") # ONE coalesced update for all three tables
7516
  # LOUD, never silent (D-11): a full append table means the SERIES has stopped growing, which
7517
  # is the failure a chart cannot show you.
7518
  capped = c_snap["capped"] + c_post["capped"] + c_ps["capped"]
 
8298
 
8299
  Pulling this module in at import time initialises the store layer earlier than
8300
  `automation_engine` used to, and the last time that happened it moved a store-commit COUNT
8301
+ from three to four on an unrelated gate. The relational pass
8302
  needs three constants and two predicates from the field layer; it does not need to change
8303
  when this module is imported.
8304
  """
 
8352
  return (raw.lower(),)
8353
 
8354
 
8355
+ def _rollup_fold(fn, values):
8356
  """`values` (raw cells, in the order the window kept them) β†’ the aggregate, as a STRING.
8357
 
8358
  Returns `""` for "nothing to aggregate", NEVER `0`. β›” That distinction is this module's
 
8362
  ⚠ The ONE exception is the count family, where zero IS the answer β€” "how many linked records"
8363
  over an empty set is genuinely 0, not unknown.
8364
  """
8365
+ if fn == "countall":
8366
+ return str(len(values))
8367
+ if fn == "latest":
8368
+ # Ordering belongs to the rollup bag (`sortBy` is mandatory for this function). Preserve
8369
+ # a blank on the newest row rather than reaching backwards and presenting an older value
8370
+ # as current.
8371
+ return "" if not values or values[0] is None else str(values[0])[:ROLLUP_MAX_CHARS]
8372
  if fn == "counta":
8373
  return str(len([v for v in values if str(v or "").strip() != ""]))
8374
  if fn == "count":
 
8423
  return str(pin.get("key") or "") if pin else ""
8424
 
8425
 
8426
+ def _linked_rows_by_join(linked, on_key):
8427
  """`{join value (lower-cased) -> [(row_id, row)]}` over one linked table, built ONCE.
8428
 
8429
  ⚠ Lower-cased because the join values this exists for are Instagram handles, which the
 
8435
  k = str((row or {}).get(on_key) or "").strip().lower()
8436
  if k:
8437
  idx.setdefault(k, []).append((str(rid), row or {}))
8438
+ return idx
8439
+
8440
+
8441
+ def _rollup_condition_matches(row, condition, field_types):
8442
+ """Evaluate one Airtable-style linked-record condition against a candidate row."""
8443
+ field = str((condition or {}).get("field") or "")
8444
+ op = str((condition or {}).get("op") or "")
8445
+ raw = (row or {}).get(field)
8446
+ text = str(raw or "").strip()
8447
+ if op == "is_empty":
8448
+ return text == ""
8449
+ if op == "is_not_empty":
8450
+ return text != ""
8451
+ wanted = str((condition or {}).get("value") or "").strip()
8452
+ if op == "contains":
8453
+ return wanted.casefold() in text.casefold()
8454
+ if op == "not_contains":
8455
+ return wanted.casefold() not in text.casefold()
8456
+ if op in ("eq", "neq"):
8457
+ left_num, right_num = _lane_num(text), _lane_num(wanted)
8458
+ equal = (left_num == right_num if left_num is not None and right_num is not None
8459
+ else text.casefold() == wanted.casefold())
8460
+ return equal if op == "eq" else not equal
8461
+ ftype = (field_types or {}).get(field, "text")
8462
+ left = _sort_key(text, ftype)
8463
+ right = _sort_key(wanted, ftype)
8464
+ if left is None or right is None:
8465
+ return False
8466
+ return ((op == "gt" and left > right) or (op == "gte" and left >= right)
8467
+ or (op == "lt" and left < right) or (op == "lte" and left <= right))
8468
+
8469
+
8470
+ def compute_relation_cells(rt, table_key, tables=None):
8471
  """Recompute every DERIVED LINK cell and every ROLLUP cell on ONE table. Returns rows touched.
8472
 
8473
  Zero store reads when the table declares neither kind β€” the same cheap-by-construction shape
8474
  `compute_metric_cells` has, so walking every table on a tick costs a dict scan per table.
8475
  """
8476
+ store = tables if tables is not None else ut_all(rt)
8477
+ t = (store or {}).get(table_key)
8478
  fields = list((t or {}).get("fields") or [])
8479
  links = [f for f in fields if _ut().is_derived_link(f)]
8480
  rollups = [f for f in fields if isinstance(f.get("rollup"), dict)]
 
8491
  if not lk_key or lk_key in resolved or not isinstance((f or {}).get("link"), dict):
8492
  continue
8493
  bag = f["link"]
8494
+ linked = (store or {}).get(str(bag.get("table") or "")) or {}
8495
  linked_types[lk_key] = {str(lf.get("key")): str(lf.get("type") or "text")
8496
  for lf in (linked.get("fields") or [])}
8497
+ if bag.get("inverse"):
8498
+ # Airtable's reciprocal side: this row is linked to every SOURCE row whose ordinary
8499
+ # link cell contains this row id. The source cell remains the one relationship truth.
8500
+ source_rows = linked.get("rows") or {}
8501
+ inverse_key = str(bag.get("inverse") or "")
8502
+ inverse_index = {}
8503
+ for source_id, source_row in source_rows.items():
8504
+ for target_id in [part.strip() for part in
8505
+ str((source_row or {}).get(inverse_key) or "").split(",")]:
8506
+ if target_id:
8507
+ inverse_index.setdefault(target_id, []).append(
8508
+ (str(source_id), source_row or {}))
8509
+ resolved[lk_key] = {str(rid): inverse_index.get(str(rid), []) for rid in rows}
8510
+ elif bag.get("on"):
8511
  idx = _linked_rows_by_join(linked, str(bag["on"]))
8512
  from_key = _link_from_key(fields, bag)
8513
  resolved[lk_key] = {
 
8540
  want = ",".join(i for i, _r in hits[:_ut().LINK_MAX_IDS])
8541
  if str(row.get(fk, "")) != want:
8542
  changes.setdefault(rid, {})[fk] = want
8543
+ for f in rollups:
8544
  fk, bag = str(f["key"]), f["rollup"]
8545
  lk_key = str(bag.get("link") or "")
8546
+ hits = list((resolved.get(lk_key) or {}).get(rid) or [])
8547
  # ⚠ A rollup whose link field does not exist (renamed, deleted) resolves to NOTHING
8548
  # and therefore to a blank cell β€” never to a stale number. A column that keeps
8549
  # printing yesterday's answer after its input is gone is the worst of the options.
8550
+ conditions = list(bag.get("conditions") or [])
8551
+ if conditions:
8552
+ matches = lambda pair: [
8553
+ _rollup_condition_matches(pair[1], condition,
8554
+ linked_types.get(lk_key) or {})
8555
+ for condition in conditions]
8556
+ if str(bag.get("conditionConj") or "and") == "or":
8557
+ hits = [pair for pair in hits if any(matches(pair))]
8558
+ else:
8559
+ hits = [pair for pair in hits if all(matches(pair))]
8560
+ sort_by = str(bag.get("sortBy") or "")
8561
+ if sort_by:
8562
  ftype = (linked_types.get(lk_key) or {}).get(sort_by, "text")
8563
  # β›” PARTITION, THEN SORT. A row whose sort cell is blank or unparseable is not
8564
  # rankable, and it must land at the END whichever direction is asked for β€” which
 
8569
  rankable = [(p, k) for p, k in keyed if k is not None]
8570
  rankable.sort(key=lambda pk: pk[1],
8571
  reverse=str(bag.get("sortDir") or "desc") == "desc")
8572
+ hits = [p for p, _k in rankable] + [p for p, k in keyed if k is None]
8573
+ distinct_by = str(bag.get("distinctBy") or "")
8574
+ if distinct_by:
8575
+ seen, unique = set(), []
8576
+ for pair in hits:
8577
+ identity = str(pair[1].get(distinct_by) or "").strip().lower()
8578
+ # A blank is not an identity. Keep it rather than collapsing every unknown
8579
+ # record into one synthetic duplicate.
8580
+ if identity and identity in seen:
8581
+ continue
8582
+ if identity:
8583
+ seen.add(identity)
8584
+ unique.append(pair)
8585
+ hits = unique
8586
+ limit = int(bag.get("limit") or 0)
8587
  if limit:
8588
  hits = hits[:limit]
8589
  src = str(bag.get("field") or "")
 
8591
  [r.get(src) for _i, r in hits] if src else [1] * len(hits))
8592
  if str(row.get(fk, "")) != want:
8593
  changes.setdefault(rid, {})[fk] = want
8594
+ if not changes:
8595
+ return 0
8596
+
8597
+ # A run that just wrote linked rows passes its in-flight user_tables bucket here so the
8598
+ # relation refresh joins the SAME coalesced commit. The standalone/tick path below keeps
8599
+ # the public helper's old persist-on-change behaviour.
8600
+ if tables is not None:
8601
+ tt = tables.get(table_key)
8602
+ if tt is not None:
8603
+ for r, vals in changes.items():
8604
+ tt.setdefault("rows", {}).setdefault(r, {}).update(vals)
8605
+ return len(changes)
8606
+
8607
+ def _up(cur):
8608
  cur = cur if isinstance(cur, dict) else {}
8609
  tt = cur.get(table_key)
8610
  if tt is not None:
 
8616
  return len(changes)
8617
 
8618
 
8619
+ def _refresh_relations_inplace(tables, log=print):
8620
+ """Refresh every relation against one mutable user_tables bucket; perform no store write."""
8621
+ touched = 0
8622
+ for tk, table in list((tables or {}).items()):
8623
+ fields = (table or {}).get("fields") or []
8624
+ if not any(f.get("type") == "rollup" or _ut().is_derived_link(f) for f in fields):
8625
+ continue
8626
+ try:
8627
+ touched += compute_relation_cells(None, tk, tables=tables)
8628
+ except Exception as e: # noqa: BLE001
8629
+ log(f"[aios-auto] relation refresh {tk} failed: {type(e).__name__}: {e}")
8630
+ return touched
8631
+
8632
+
8633
+ def refresh_relations(rt, log=print):
8634
  """The tick half of the relational pass β€” the twin of `refresh_metrics`.
8635
 
8636
  ⚠ Runs for EVERY table, because a link can point anywhere: a rollup on table A goes stale
8637
  when table B gains a row, and A has no way to know that happened. Cheap by construction β€” a
8638
  table declaring neither kind costs one dict scan.
8639
  """
8640
+ snapshot = {
8641
+ str(key): {**(table or {}),
8642
+ "rows": {str(rid): dict(row or {})
8643
+ for rid, row in ((table or {}).get("rows") or {}).items()}}
8644
+ for key, table in (ut_all(rt) or {}).items()
8645
+ }
8646
+ if not _refresh_relations_inplace(snapshot, log=log):
8647
+ return 0
8648
+ actual = [0]
8649
+
8650
+ def _up(cur):
8651
+ cur = cur if isinstance(cur, dict) else {}
8652
+ actual[0] = _refresh_relations_inplace(cur, log=log)
8653
+ return cur
8654
+
8655
+ rt.update(UT_STORE_KEY, _up, flush="sync")
8656
+ return actual[0]
8657
 
8658
 
8659
  def refresh_metrics(rt, today=None, log=print):
api/routes_tables.py CHANGED
@@ -36,6 +36,17 @@ def _ops(session, table_key):
36
  return table_store.make(f"{table_key}_table_workspace", st=session.runtime)
37
 
38
 
 
 
 
 
 
 
 
 
 
 
 
39
  def _defn_or_refuse(session, table_key):
40
  """The per-table wall: 404 for a key that does not exist, 403 for one this session may not
41
  open. 404-before-403 leaks nothing useful β€” ut keys are guessable slugs, and 'exists but
@@ -49,6 +60,16 @@ def _defn_or_refuse(session, table_key):
49
  return defn
50
 
51
 
 
 
 
 
 
 
 
 
 
 
52
  #: Tables whose ROWS ARE PER-USER (D-39, wave 22 C6's view half): a non-admin sees the rows
53
  #: they created plus any row nobody owns; an admin sees the whole pool.
54
  #:
@@ -174,6 +195,7 @@ def list_tables(session: Session = Depends(require_session)):
174
  continue
175
  out.append({"key": key, "label": ut_label(t, key, meta),
176
  "source": t.get("source") or "Blank",
 
177
  "createdBy": t.get("createdBy") or "",
178
  "created": t.get("created") or "",
179
  "fields": [dict(f) for f in (t.get("fields") or [])],
@@ -341,15 +363,13 @@ def table_rows(table_key: str, session: Session = Depends(require_session)):
341
  g = ut_assembly(session, table_key)
342
  merged = {str(r["pid"]): {k: v for k, v in r.items() if k != "pid"}
343
  for r in g["rows_src"]}
344
- for pid, cells in (g["ws"].get("overlays") or {}).items():
345
- if isinstance(cells, dict):
346
- merged.setdefault(str(pid), {}).update(cells)
347
  rows = aios_grid.rows_from_pool(
348
  g["rows_src"], g["fields"], merged, derived=g["derived"])
349
  return {"fields": g["fields"], "rows": rows, "today": g["today"],
350
  "pulled_at": time.strftime("%Y-%m-%d %H:%M"),
351
  "identity": {"pid": "pid"},
352
- "scope": {"table": table_key, "rowCount": len(rows)}}
 
353
 
354
 
355
  @router.post("/tables/{table_key}/rows", status_code=201)
@@ -361,7 +381,7 @@ def add_row(table_key: str, body: dict = Body(default=None),
361
  that requested `rid: 7` and got 12 because 7 had been re-used must find that out from the
362
  response rather than assume; the client re-anchors on what came back.
363
  """
364
- _defn_or_refuse(session, table_key)
365
  ut = _ut()
366
  values = (body or {}).get("values") or {}
367
  if not isinstance(values, dict):
@@ -385,6 +405,7 @@ def add_row(table_key: str, body: dict = Body(default=None),
385
  f"profile link (instagram.com/name)")
386
  raise err(400, "refused",
387
  f"row refused β€” the table may be at its {ut.MAX_ROWS}-row cap")
 
388
  return {"rid": rid, "pid": int(rid)}
389
 
390
 
@@ -411,6 +432,13 @@ def _field_or_refuse(session, table_key, fkey=""):
411
  "owns its own columns")
412
  if fkey and not ut.may_edit_field(table_key, fkey, session.uname, session.admin,
413
  st=session.runtime):
 
 
 
 
 
 
 
414
  raise err(403, "forbidden", "that column can only be changed by the database's creator "
415
  "or an admin")
416
  if not fkey and not (session.admin or defn.get("createdBy") == session.uname):
@@ -434,6 +462,10 @@ def add_field(table_key: str, body: dict = Body(default=None),
434
  # re-implements it (a second copy of the rule is how two doors start disagreeing).
435
  raise err(400, "refused",
436
  _refusal_sentence(ut, session, body or {}, table_key=table_key))
 
 
 
 
437
  return {"field": field}
438
 
439
 
@@ -516,6 +548,9 @@ def patch_field(table_key: str, fkey: str, body: dict = Body(default=None),
516
  # one thing that was never wrong (the D-46 lesson, one door over).
517
  raise err(400, "refused",
518
  _refusal_sentence(ut, session, body, table_key=table_key, fkey=fkey))
 
 
 
519
  out = {"field": field}
520
  if migrated is not None:
521
  out["migrated"] = migrated
@@ -528,18 +563,20 @@ def delete_field(table_key: str, fkey: str, session: Session = Depends(require_s
528
  if not _ut().delete_field(table_key, fkey, st=session.runtime):
529
  raise err(400, "refused",
530
  "that column could not be removed β€” a database must keep at least one")
 
531
  return {"deleted": fkey}
532
 
533
 
534
  @router.delete("/tables/{table_key}/rows/{rid}")
535
  def delete_row(table_key: str, rid: str, session: Session = Depends(require_session)):
536
- _defn_or_refuse(session, table_key)
537
  try:
538
  ok = _ut().delete_row(table_key, rid, st=session.runtime)
539
  except Exception:
540
  raise err(503, "store_unavailable", "the delete did not land β€” try again")
541
  if not ok:
542
  raise err(400, "refused", "rows can only be deleted from user-created databases")
 
543
  return {"ok": True}
544
 
545
 
@@ -554,6 +591,7 @@ def patch_row(table_key: str, pid: int, body: dict = Body(default=None),
554
  updates = dict(body or {})
555
  if not updates:
556
  raise err(400, "empty_patch", "no fields to update")
 
557
  g = ut_assembly(session, table_key, consume_corrections=False)
558
  if pid not in g["pids"]:
559
  raise err(403, "out_of_scope", "that row is not in this database")
@@ -576,8 +614,6 @@ def patch_row(table_key: str, pid: int, body: dict = Body(default=None),
576
  # rows envelope uses (`table_rows`): definition under, overlay over.
577
  stored = dict(((_ut().get(table_key, st=session.runtime) or {}).get("rows") or {})
578
  .get(str(pid)) or {})
579
- stored.update((grid_events.table_workspace(ctx, allowed_pids=None)
580
- .get("overlays") or {}).get(str(pid)) or {})
581
  accepted = {k: stored.get(k) for k in updates if k in stored}
582
 
583
  def _took(k):
@@ -615,4 +651,5 @@ def patch_row(table_key: str, pid: int, body: dict = Body(default=None),
615
  cleared = sorted(k for k in also if k in _ut().PROFILE_PRESET_KEYS)
616
  if cleared:
617
  out["cleared"] = cleared
 
618
  return out
 
36
  return table_store.make(f"{table_key}_table_workspace", st=session.runtime)
37
 
38
 
39
+ def _refresh_relations(session):
40
+ """Refresh reciprocal Links and Rollups after a human record/schema mutation."""
41
+ import automation_engine as engine
42
+ try:
43
+ engine.refresh_relations(session.runtime, log=lambda *_args: None)
44
+ except Exception as exc: # noqa: BLE001
45
+ # The source write already landed. A later automation/tick will repair materialised
46
+ # cells; never answer 503 and invite the browser to repeat a successful mutation.
47
+ print(f"[tables] relation refresh deferred: {type(exc).__name__}: {exc}")
48
+
49
+
50
  def _defn_or_refuse(session, table_key):
51
  """The per-table wall: 404 for a key that does not exist, 403 for one this session may not
52
  open. 404-before-403 leaks nothing useful β€” ut keys are guessable slugs, and 'exists but
 
60
  return defn
61
 
62
 
63
+ def _records_or_refuse(session, table_key):
64
+ """The human record-write wall for a database the automation engine owns."""
65
+ defn = _defn_or_refuse(session, table_key)
66
+ if not _ut().records_mutable(table_key, st=session.runtime):
67
+ raise err(403, "records_read_only",
68
+ "records in this automation-owned database are read-only β€” add Instagram "
69
+ "handles in a Profile database and let enrichment populate this database")
70
+ return defn
71
+
72
+
73
  #: Tables whose ROWS ARE PER-USER (D-39, wave 22 C6's view half): a non-admin sees the rows
74
  #: they created plus any row nobody owns; an admin sees the whole pool.
75
  #:
 
195
  continue
196
  out.append({"key": key, "label": ut_label(t, key, meta),
197
  "source": t.get("source") or "Blank",
198
+ "recordsMutable": ut.records_mutable(key, st=session.runtime),
199
  "createdBy": t.get("createdBy") or "",
200
  "created": t.get("created") or "",
201
  "fields": [dict(f) for f in (t.get("fields") or [])],
 
363
  g = ut_assembly(session, table_key)
364
  merged = {str(r["pid"]): {k: v for k, v in r.items() if k != "pid"}
365
  for r in g["rows_src"]}
 
 
 
366
  rows = aios_grid.rows_from_pool(
367
  g["rows_src"], g["fields"], merged, derived=g["derived"])
368
  return {"fields": g["fields"], "rows": rows, "today": g["today"],
369
  "pulled_at": time.strftime("%Y-%m-%d %H:%M"),
370
  "identity": {"pid": "pid"},
371
+ "scope": {"table": table_key, "rowCount": len(rows)},
372
+ "recordsMutable": _ut().records_mutable(table_key, st=session.runtime)}
373
 
374
 
375
  @router.post("/tables/{table_key}/rows", status_code=201)
 
381
  that requested `rid: 7` and got 12 because 7 had been re-used must find that out from the
382
  response rather than assume; the client re-anchors on what came back.
383
  """
384
+ _records_or_refuse(session, table_key)
385
  ut = _ut()
386
  values = (body or {}).get("values") or {}
387
  if not isinstance(values, dict):
 
405
  f"profile link (instagram.com/name)")
406
  raise err(400, "refused",
407
  f"row refused β€” the table may be at its {ut.MAX_ROWS}-row cap")
408
+ _refresh_relations(session)
409
  return {"rid": rid, "pid": int(rid)}
410
 
411
 
 
432
  "owns its own columns")
433
  if fkey and not ut.may_edit_field(table_key, fkey, session.uname, session.admin,
434
  st=session.runtime):
435
+ field = next((f for f in (defn.get("fields") or [])
436
+ if f.get("key") == str(fkey)), None)
437
+ if isinstance((field or {}).get("automation"), dict) \
438
+ and field["automation"].get("preset") is True:
439
+ raise err(403, "preset_field_locked",
440
+ "Instagram pre-set fields are locked; you may sort, filter or hide this "
441
+ "column, and add your own columns separately")
442
  raise err(403, "forbidden", "that column can only be changed by the database's creator "
443
  "or an admin")
444
  if not fkey and not (session.admin or defn.get("createdBy") == session.uname):
 
462
  # re-implements it (a second copy of the rule is how two doors start disagreeing).
463
  raise err(400, "refused",
464
  _refusal_sentence(ut, session, body or {}, table_key=table_key))
465
+ if field.get("type") == "link":
466
+ synced = ut.sync_reciprocal_link(table_key, field["key"], st=session.runtime)
467
+ field = synced.get("field") or field
468
+ _refresh_relations(session)
469
  return {"field": field}
470
 
471
 
 
548
  # one thing that was never wrong (the D-46 lesson, one door over).
549
  raise err(400, "refused",
550
  _refusal_sentence(ut, session, body, table_key=table_key, fkey=fkey))
551
+ synced = ut.sync_reciprocal_link(table_key, fkey, st=session.runtime)
552
+ field = synced.get("field") or field
553
+ _refresh_relations(session)
554
  out = {"field": field}
555
  if migrated is not None:
556
  out["migrated"] = migrated
 
563
  if not _ut().delete_field(table_key, fkey, st=session.runtime):
564
  raise err(400, "refused",
565
  "that column could not be removed β€” a database must keep at least one")
566
+ _refresh_relations(session)
567
  return {"deleted": fkey}
568
 
569
 
570
  @router.delete("/tables/{table_key}/rows/{rid}")
571
  def delete_row(table_key: str, rid: str, session: Session = Depends(require_session)):
572
+ _records_or_refuse(session, table_key)
573
  try:
574
  ok = _ut().delete_row(table_key, rid, st=session.runtime)
575
  except Exception:
576
  raise err(503, "store_unavailable", "the delete did not land β€” try again")
577
  if not ok:
578
  raise err(400, "refused", "rows can only be deleted from user-created databases")
579
+ _refresh_relations(session)
580
  return {"ok": True}
581
 
582
 
 
591
  updates = dict(body or {})
592
  if not updates:
593
  raise err(400, "empty_patch", "no fields to update")
594
+ _records_or_refuse(session, table_key)
595
  g = ut_assembly(session, table_key, consume_corrections=False)
596
  if pid not in g["pids"]:
597
  raise err(403, "out_of_scope", "that row is not in this database")
 
614
  # rows envelope uses (`table_rows`): definition under, overlay over.
615
  stored = dict(((_ut().get(table_key, st=session.runtime) or {}).get("rows") or {})
616
  .get(str(pid)) or {})
 
 
617
  accepted = {k: stored.get(k) for k in updates if k in stored}
618
 
619
  def _took(k):
 
651
  cleared = sorted(k for k in also if k in _ut().PROFILE_PRESET_KEYS)
652
  if cleared:
653
  out["cleared"] = cleared
654
+ _refresh_relations(session)
655
  return out
platform/core/grid_events.py CHANGED
@@ -1755,6 +1755,19 @@ def handle_one(event, ctx):
1755
  refused = True
1756
  continue
1757
  _auto = (_fdef or {}).get('automation')
 
 
 
 
 
 
 
 
 
 
 
 
 
1758
  if isinstance(_auto, dict) and _auto.get('stageField'):
1759
  _row = ((_tdef or {}).get('rows') or {}).get(str(pid)) or {}
1760
  _cur = str(_row.get(key) or '').strip()
@@ -1846,7 +1859,7 @@ def handle_one(event, ctx):
1846
  # `flowId` and not merely `automation`: a user-configured `automation` COLUMN
1847
  # carries its own gear bag (kind/source/urlField/settings) and no flowId, and it
1848
  # is not this ruling's subject.
1849
- if isinstance(_auto, dict) and _auto.get('flowId'):
1850
  refused = True
1851
  continue
1852
  # ⭐ WAVE 23 (C7) β€” the json wall, BEFORE the generic 10 000-char truncation, which
@@ -1904,7 +1917,13 @@ def handle_one(event, ctx):
1904
  clean = normalized
1905
  updates[key] = clean
1906
  if updates:
1907
- if _store_of(ctx).available():
 
 
 
 
 
 
1908
  _tops(ctx).patch_overlay(uname, pid, updates)
1909
  else:
1910
  _session_ready()
 
1755
  refused = True
1756
  continue
1757
  _auto = (_fdef or {}).get('automation')
1758
+ # An ordinary Link is a shared database relationship. Validate its selected
1759
+ # target ids and write it through to the definition row; a personal overlay
1760
+ # would be invisible to reciprocal links and Rollups.
1761
+ if (_fdef or {}).get('type') == 'link':
1762
+ _linked = _ut_mod.patch_link_cell(
1763
+ ctx.scope_key, pid, key, value, st=ctx.table.st)
1764
+ if _linked is None:
1765
+ refused = True
1766
+ continue
1767
+ if _linked != str(value):
1768
+ refused = True
1769
+ row_events.append((key, _linked))
1770
+ continue
1771
  if isinstance(_auto, dict) and _auto.get('stageField'):
1772
  _row = ((_tdef or {}).get('rows') or {}).get(str(pid)) or {}
1773
  _cur = str(_row.get(key) or '').strip()
 
1859
  # `flowId` and not merely `automation`: a user-configured `automation` COLUMN
1860
  # carries its own gear bag (kind/source/urlField/settings) and no flowId, and it
1861
  # is not this ruling's subject.
1862
+ if isinstance(_auto, dict) and (_auto.get('flowId') or _auto.get('preset')):
1863
  refused = True
1864
  continue
1865
  # ⭐ WAVE 23 (C7) β€” the json wall, BEFORE the generic 10 000-char truncation, which
 
1917
  clean = normalized
1918
  updates[key] = clean
1919
  if updates:
1920
+ if str(ctx.scope_key or '').startswith('ut_') and ctx.table is not None:
1921
+ # User-table cells are collaborative record data. Views/layout remain personal,
1922
+ # but record values live in the definition rows so Links, Rollups, automations,
1923
+ # and every viewer observe one truth.
1924
+ import core.user_tables as _ut_values
1925
+ _ut_values.patch_cells(ctx.scope_key, pid, updates, st=ctx.table.st)
1926
+ elif _store_of(ctx).available():
1927
  _tops(ctx).patch_overlay(uname, pid, updates)
1928
  else:
1929
  _session_ready()
platform/core/user_tables.py CHANGED
@@ -32,6 +32,7 @@ the Streamlit host, unchanged). The API passes the session's `TenantRuntime`, wh
32
  apply the tenant prefix / repo binding, so a Nurilab table lands in Nurilab's store.
33
  """
34
  import datetime as _dt
 
35
  import re
36
 
37
  import core.store as store
@@ -45,9 +46,21 @@ MAX_FIELDS = 60
45
  BLANK_SOURCE = 'Blank'
46
  #: automation-created tables carry their maker instead (wave 18 C4-AUTO)
47
  AUTOMATION_SOURCE = 'Automation'
 
48
  #: every user table's key is prefixed, so it can never collide with a registry module key
49
  KEY_PREFIX = 'ut_'
50
 
 
 
 
 
 
 
 
 
 
 
 
51
  #: The field types a user table may declare. ⚠ Kept a SUBSET of
52
  #: `aios_grid.CUSTOM_FIELD_TYPES` (gated in verify_api's W18-UT section). `formula` and
53
  #: `created_time` stay out (client-computed / row-datum kinds β€” a base column of either would
@@ -120,13 +133,17 @@ PROFILE_PRESET_KEYS = (
120
  # the profile facts themselves (CANDIDATE_FIELDS + the enrichment run_field_instagram pulls)
121
  'full_name', 'followers', 'following', 'posts_count', 'avg_engagement', 'bio',
122
  'external_url', 'verified', 'category', 'business_category', 'is_business',
123
- 'is_professional', 'is_private', 'highlights_count', 'bio_hashtags', 'post_hashtags',
124
  'pronouns', 'ig_id', 'profile_url',
125
  # ⭐ 2026-08-07 β€” the rest of the Bright Data profile schema, promoted to preset columns by
126
  # owner instruction. They are profile FACTS like the nineteen above, so blanking the handle
127
  # clears them for the same reason (R6): they describe an account this row no longer names.
128
  'profile_name', 'is_joined_recently', 'has_channel', 'partner_id', 'external_url_title',
129
  'fbid', 'related_accounts', 'country_code',
 
 
 
 
130
  # R3's LATEST-value stamp. The full series stays in `ut_ig_snapshots` β€” one store for one
131
  # series β€” which is why clearing here can never be a history delete.
132
  'enriched_at',
@@ -458,7 +475,7 @@ def add_row(table_key, values=None, username=None, st=None, rid=None):
458
  only when that id is FREE β€” an undo can never overwrite a row somebody has since created in
459
  the gap, and it never invents a non-numeric id, because `scoped_pool` reads row ids as ints.
460
  """
461
- if not is_user_table(table_key, st):
462
  return None # ⚠ never on a connector-backed table
463
  defn = get(table_key, st) or {}
464
  rows = defn.get('rows') or {}
@@ -601,7 +618,7 @@ def _clean_metric(raw):
601
  # So a link cell holds a COMMA-JOINED list of linked row ids β€” the shape `multiselect` already
602
  # uses for a set, so the grouping/filter/copy paths already know what to do with it.
603
  #
604
- # ⭐ TWO MODES, ONE KIND, AND `on` IS THE DISCRIMINATOR:
605
  # `on` DECLARED -> a DERIVED link. The linked rows are those whose `on` column equals this
606
  # row's `from` column. Machine-maintained, read-only, recomputed on the same
607
  # pass that already recomputes `metric` cells. THIS IS THE INSTAGRAM CASE, and
@@ -609,13 +626,10 @@ def _clean_metric(raw):
609
  # relation and is rewritten by the engine on every pull, so a second
610
  # user-editable copy of the same relation could only ever drift away from it.
611
  # `on` ABSENT -> an ORDINARY link. The user picks records; the cell is editable; `single` is
612
- # Airtable's `prefersSingleRecordLink`.
613
- #
614
- # ⚠ NO `inverse` (Airtable's symmetric back-link field) THIS WAVE, and it is an omission rather
615
- # than an oversight: maintaining one means every cell write fans out into a second table's rows
616
- # through `grid_events`' emit guard β€” the cross-table write-amplification class D-40 was
617
- # re-deferred BY RULING over. A derived link gets symmetry free (both sides re-derive from the
618
- # same join), so the gap only touches ordinary links, where a back-link is a convenience.
619
 
620
  #: How many linked ids one link cell may carry. Row ids are short numeric strings, so 500 ids is
621
  #: ~3 KB β€” comfortably inside a text cell, and `MAX_ROWS` bounds the absolute worst case anyway.
@@ -626,9 +640,13 @@ LINK_MAX_IDS = 500
626
  #: this product is a scalar β€” they could only ever return their own input, so offering them would
627
  #: mint columns that compute nothing. `arrayslice`'s real use ("just the first N") is what `limit`
628
  #: does below, honestly and by declared rank.
629
- ROLLUP_FNS = ('sum', 'average', 'min', 'max', 'count', 'counta', 'countall',
630
  'and', 'or', 'xor', 'concatenate', 'arrayjoin', 'arraycompact', 'arrayunique')
631
  ROLLUP_SORT_DIRS = ('asc', 'desc')
 
 
 
 
632
  #: The ceiling on `limit`. Not a performance bound β€” `MAX_ROWS` is that β€” but a refusal to let a
633
  #: column claim a window bigger than a table can hold.
634
  ROLLUP_MAX_LIMIT = MAX_ROWS
@@ -659,7 +677,15 @@ def _clean_link(raw):
659
  out = {'table': table}
660
  on = _field_key(raw.get('on'))
661
  frm = _field_key(raw.get('from'))
662
- if on:
 
 
 
 
 
 
 
 
663
  out['on'] = on
664
  # `from` is OPTIONAL and resolved at compute time (the profile-flagged column, then the
665
  # pinned one) β€” which is what makes an Instagram database link up with no configuration
@@ -671,6 +697,8 @@ def _clean_link(raw):
671
  # than dropped: a bag half of which is silently ignored is [[wrong-parent-not-broken-control]]
672
  # with the control still on screen.
673
  return None
 
 
674
  if raw.get('single') is True:
675
  out['single'] = True
676
  return out
@@ -706,10 +734,11 @@ def _clean_rollup(raw):
706
  except (TypeError, ValueError):
707
  return None
708
  sort_by = _field_key(raw.get('sortBy'))
 
709
  sort_dir = str(raw.get('sortDir') or '').strip().lower()
710
  if limit < 0 or limit > ROLLUP_MAX_LIMIT:
711
  return None
712
- if limit and not sort_by:
713
  return None
714
  if sort_dir and sort_dir not in ROLLUP_SORT_DIRS:
715
  return None
@@ -723,6 +752,35 @@ def _clean_rollup(raw):
723
  out['limit'] = limit
724
  elif sort_dir:
725
  return None
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
726
  return out
727
 
728
 
@@ -819,7 +877,7 @@ def is_derived_link(field):
819
  silently keeps taking writes after the other two stop.
820
  """
821
  lk = (field or {}).get('link')
822
- return isinstance(lk, dict) and bool(lk.get('on'))
823
 
824
 
825
  def is_computed_cell(field):
@@ -866,10 +924,18 @@ def clean_machine_fields(fields):
866
  def may_edit_field(table_key, fkey, viewer, is_admin=False, st=None):
867
  """May `viewer` change THIS column's definition? Creator/admin always; others only when the
868
  field itself says `editRole: 'everyone'`. Fail-closed on an unknown field."""
 
 
 
 
 
 
 
 
869
  if may_open(table_key, viewer, is_admin, st) and (
870
- bool(is_admin) or (get(table_key, st) or {}).get('createdBy') == viewer):
871
  return True
872
- for f in ((get(table_key, st) or {}).get('fields') or []):
873
  if f.get('key') == str(fkey):
874
  return f.get('editRole') == 'everyone'
875
  return False
@@ -922,6 +988,66 @@ def add_field(table_key, raw, st=None):
922
  return field
923
 
924
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
925
  def patch_field(table_key, fkey, raw, st=None):
926
  """Edit one column's definition IN PLACE. Returns the stored field, or None if refused.
927
 
@@ -973,11 +1099,39 @@ def delete_field(table_key, fkey, st=None):
973
  fields = (get(table_key, st) or {}).get('fields') or []
974
  if len(fields) <= 1 or not any(f.get('key') == str(fkey) for f in fields):
975
  return False
 
 
 
 
 
 
 
 
 
 
976
 
977
  def _drop(cur):
978
  t = cur.get(str(table_key))
979
  if t is not None:
980
  t['fields'] = [f for f in (t.get('fields') or []) if f.get('key') != str(fkey)]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
981
  return cur
982
 
983
  _st(st).update(STORE_KEY, _drop, flush='sync')
@@ -1075,6 +1229,50 @@ def patch_cells(table_key, row_id, values, st=None):
1075
  return True
1076
 
1077
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1078
  def patch_profile_cell(table_key, row_id, fkey, value, st=None):
1079
  """Write a PROFILE cell β€” and, when it is blanked, clear that row's preset cells IN THE SAME
1080
  WRITE (wave 25, contract C3 + owner ruling R6). Returns
@@ -1139,7 +1337,7 @@ def patch_profile_cell(table_key, row_id, fkey, value, st=None):
1139
 
1140
 
1141
  def delete_row(table_key, row_id, st=None):
1142
- if not is_user_table(table_key, st):
1143
  return False
1144
 
1145
  def _drop(cur):
 
32
  apply the tenant prefix / repo binding, so a Nurilab table lands in Nurilab's store.
33
  """
34
  import datetime as _dt
35
+ import hashlib
36
  import re
37
 
38
  import core.store as store
 
46
  BLANK_SOURCE = 'Blank'
47
  #: automation-created tables carry their maker instead (wave 18 C4-AUTO)
48
  AUTOMATION_SOURCE = 'Automation'
49
+ AUTOMATION_RECORD_MODE = 'automation'
50
  #: every user table's key is prefixed, so it can never collide with a registry module key
51
  KEY_PREFIX = 'ut_'
52
 
53
+
54
+ def records_mutable(table_key, st=None):
55
+ """May a human add/edit/delete records in this database?
56
+
57
+ Ordinary and Profile databases default open. Automation-owned child datasets opt out with a
58
+ table-level mode; engine writers use their direct coalesced path and are intentionally not
59
+ routed through this human-door predicate.
60
+ """
61
+ table = get(table_key, st) or {}
62
+ return bool(table) and table.get('recordMode') != AUTOMATION_RECORD_MODE
63
+
64
  #: The field types a user table may declare. ⚠ Kept a SUBSET of
65
  #: `aios_grid.CUSTOM_FIELD_TYPES` (gated in verify_api's W18-UT section). `formula` and
66
  #: `created_time` stay out (client-computed / row-datum kinds β€” a base column of either would
 
133
  # the profile facts themselves (CANDIDATE_FIELDS + the enrichment run_field_instagram pulls)
134
  'full_name', 'followers', 'following', 'posts_count', 'avg_engagement', 'bio',
135
  'external_url', 'verified', 'category', 'business_category', 'is_business',
136
+ 'is_professional', 'is_private', 'highlights_count', 'bio_hashtags',
137
  'pronouns', 'ig_id', 'profile_url',
138
  # ⭐ 2026-08-07 β€” the rest of the Bright Data profile schema, promoted to preset columns by
139
  # owner instruction. They are profile FACTS like the nineteen above, so blanking the handle
140
  # clears them for the same reason (R6): they describe an account this row no longer names.
141
  'profile_name', 'is_joined_recently', 'has_channel', 'partner_id', 'external_url_title',
142
  'fbid', 'related_accounts', 'country_code',
143
+ # relational projections and summaries owned by the Instagram graph
144
+ 'posts_link', 'profile_snapshots_link', 'post_snapshots_link', 'comments_link',
145
+ 'avg_views_12', 'avg_plays_12', 'avg_likes_12', 'avg_comments_12',
146
+ 'posts_captured', 'profile_reads', 'post_measurements_captured', 'comments_captured',
147
  # R3's LATEST-value stamp. The full series stays in `ut_ig_snapshots` β€” one store for one
148
  # series β€” which is why clearing here can never be a history delete.
149
  'enriched_at',
 
475
  only when that id is FREE β€” an undo can never overwrite a row somebody has since created in
476
  the gap, and it never invents a non-numeric id, because `scoped_pool` reads row ids as ints.
477
  """
478
+ if not is_user_table(table_key, st) or not records_mutable(table_key, st):
479
  return None # ⚠ never on a connector-backed table
480
  defn = get(table_key, st) or {}
481
  rows = defn.get('rows') or {}
 
618
  # So a link cell holds a COMMA-JOINED list of linked row ids β€” the shape `multiselect` already
619
  # uses for a set, so the grouping/filter/copy paths already know what to do with it.
620
  #
621
+ # ⭐ THREE MODES, ONE KIND:
622
  # `on` DECLARED -> a DERIVED link. The linked rows are those whose `on` column equals this
623
  # row's `from` column. Machine-maintained, read-only, recomputed on the same
624
  # pass that already recomputes `metric` cells. THIS IS THE INSTAGRAM CASE, and
 
626
  # relation and is rewritten by the engine on every pull, so a second
627
  # user-editable copy of the same relation could only ever drift away from it.
628
  # `on` ABSENT -> an ORDINARY link. The user picks records; the cell is editable; `single` is
629
+ # Airtable's `prefersSingleRecordLink`. Its value is the source of truth.
630
+ # `inverse` -> the COMPUTED reciprocal of one ordinary source link. It never fans writes
631
+ # into target rows: the relation pass derives it from the source cells, so the
632
+ # same shared-row truth powers both sides and every dependent Rollup.
 
 
 
633
 
634
  #: How many linked ids one link cell may carry. Row ids are short numeric strings, so 500 ids is
635
  #: ~3 KB β€” comfortably inside a text cell, and `MAX_ROWS` bounds the absolute worst case anyway.
 
640
  #: this product is a scalar β€” they could only ever return their own input, so offering them would
641
  #: mint columns that compute nothing. `arrayslice`'s real use ("just the first N") is what `limit`
642
  #: does below, honestly and by declared rank.
643
+ ROLLUP_FNS = ('sum', 'average', 'min', 'max', 'latest', 'count', 'counta', 'countall',
644
  'and', 'or', 'xor', 'concatenate', 'arrayjoin', 'arraycompact', 'arrayunique')
645
  ROLLUP_SORT_DIRS = ('asc', 'desc')
646
+ ROLLUP_CONDITION_OPS = ('eq', 'neq', 'contains', 'not_contains', 'is_empty', 'is_not_empty',
647
+ 'gt', 'gte', 'lt', 'lte')
648
+ ROLLUP_CONDITION_CONJ = ('and', 'or')
649
+ ROLLUP_MAX_CONDITIONS = 20
650
  #: The ceiling on `limit`. Not a performance bound β€” `MAX_ROWS` is that β€” but a refusal to let a
651
  #: column claim a window bigger than a table can hold.
652
  ROLLUP_MAX_LIMIT = MAX_ROWS
 
677
  out = {'table': table}
678
  on = _field_key(raw.get('on'))
679
  frm = _field_key(raw.get('from'))
680
+ inverse = _field_key(raw.get('inverse'))
681
+ reciprocal = _field_key(raw.get('reciprocal'))
682
+ if inverse:
683
+ # Airtable's reciprocal field: rows in `table` whose ordinary `inverse` link includes
684
+ # this row. It is computed, so `on`/`from` cannot simultaneously configure another join.
685
+ if on or frm:
686
+ return None
687
+ out['inverse'] = inverse
688
+ elif on:
689
  out['on'] = on
690
  # `from` is OPTIONAL and resolved at compute time (the profile-flagged column, then the
691
  # pinned one) β€” which is what makes an Instagram database link up with no configuration
 
697
  # than dropped: a bag half of which is silently ignored is [[wrong-parent-not-broken-control]]
698
  # with the control still on screen.
699
  return None
700
+ if reciprocal:
701
+ out['reciprocal'] = reciprocal
702
  if raw.get('single') is True:
703
  out['single'] = True
704
  return out
 
734
  except (TypeError, ValueError):
735
  return None
736
  sort_by = _field_key(raw.get('sortBy'))
737
+ distinct_by = _field_key(raw.get('distinctBy'))
738
  sort_dir = str(raw.get('sortDir') or '').strip().lower()
739
  if limit < 0 or limit > ROLLUP_MAX_LIMIT:
740
  return None
741
+ if (limit or fn == 'latest') and not sort_by:
742
  return None
743
  if sort_dir and sort_dir not in ROLLUP_SORT_DIRS:
744
  return None
 
752
  out['limit'] = limit
753
  elif sort_dir:
754
  return None
755
+ if distinct_by:
756
+ # A rollup may cross a table that already contains duplicate logical records. Keep the
757
+ # first row after ranking for each identity so "last 12 posts" means twelve POSTS, while
758
+ # a snapshot table can still keep every timestamped observation by omitting this option.
759
+ out['distinctBy'] = distinct_by
760
+ conditions = []
761
+ raw_conditions = raw.get('conditions') or []
762
+ if not isinstance(raw_conditions, list) or len(raw_conditions) > ROLLUP_MAX_CONDITIONS:
763
+ return None
764
+ for condition in raw_conditions:
765
+ if not isinstance(condition, dict):
766
+ return None
767
+ condition_field = _field_key(condition.get('field'))
768
+ op = str(condition.get('op') or '').strip().lower()
769
+ if not condition_field or op not in ROLLUP_CONDITION_OPS:
770
+ return None
771
+ item = {'field': condition_field, 'op': op}
772
+ if op not in ('is_empty', 'is_not_empty'):
773
+ value = condition.get('value')
774
+ if not isinstance(value, (str, int, float, bool)):
775
+ return None
776
+ item['value'] = str(value)[:1000]
777
+ conditions.append(item)
778
+ if conditions:
779
+ conj = str(raw.get('conditionConj') or 'and').strip().lower()
780
+ if conj not in ROLLUP_CONDITION_CONJ:
781
+ return None
782
+ out['conditions'] = conditions
783
+ out['conditionConj'] = conj
784
  return out
785
 
786
 
 
877
  silently keeps taking writes after the other two stop.
878
  """
879
  lk = (field or {}).get('link')
880
+ return isinstance(lk, dict) and bool(lk.get('on') or lk.get('inverse'))
881
 
882
 
883
  def is_computed_cell(field):
 
924
  def may_edit_field(table_key, fkey, viewer, is_admin=False, st=None):
925
  """May `viewer` change THIS column's definition? Creator/admin always; others only when the
926
  field itself says `editRole: 'everyone'`. Fail-closed on an unknown field."""
927
+ table = get(table_key, st) or {}
928
+ field = next((f for f in (table.get('fields') or []) if f.get('key') == str(fkey)), None)
929
+ # Instagram's pre-set schema is product contract, not tenant configuration. Even an admin
930
+ # may sort/filter/hide it, but cannot rename, retype, duplicate or delete it. Ordinary fields
931
+ # on the same database remain user-owned, including user-created Links and Rollups.
932
+ if isinstance((field or {}).get('automation'), dict) \
933
+ and field['automation'].get('preset') is True:
934
+ return False
935
  if may_open(table_key, viewer, is_admin, st) and (
936
+ bool(is_admin) or table.get('createdBy') == viewer):
937
  return True
938
+ for f in (table.get('fields') or []):
939
  if f.get('key') == str(fkey):
940
  return f.get('editRole') == 'everyone'
941
  return False
 
988
  return field
989
 
990
 
991
+ def _reciprocal_link_key(table_key, field_key):
992
+ digest = hashlib.sha1(f'{table_key}:{field_key}'.encode('utf-8')).hexdigest()[:10]
993
+ return f'linked_{digest}'
994
+
995
+
996
+ def sync_reciprocal_link(table_key, field_key, st=None):
997
+ """Create/repair Airtable's reciprocal link field for one ordinary link.
998
+
999
+ The source cell stores picked target row ids. The reciprocal is a computed inverse link on
1000
+ the target database; it lists source rows that include the current target id. Retargeting or
1001
+ retyping the source removes the obsolete inverse in the same store update.
1002
+ """
1003
+ table_key, field_key = str(table_key), str(field_key)
1004
+ result = {'field': None, 'reciprocal': None}
1005
+
1006
+ def _sync(cur):
1007
+ cur = cur if isinstance(cur, dict) else {}
1008
+ source = cur.get(table_key) or {}
1009
+ source_field = next((f for f in (source.get('fields') or [])
1010
+ if f.get('key') == field_key), None)
1011
+ # Remove every old inverse for this source identity first. This makes retargeting and
1012
+ # deleting deterministic instead of leaving a live-looking backlink on the old table.
1013
+ for candidate in cur.values():
1014
+ if not isinstance(candidate, dict):
1015
+ continue
1016
+ candidate['fields'] = [f for f in (candidate.get('fields') or [])
1017
+ if not (isinstance(f.get('link'), dict)
1018
+ and f['link'].get('table') == table_key
1019
+ and f['link'].get('inverse') == field_key)]
1020
+ if not source_field or source_field.get('type') != 'link':
1021
+ return cur
1022
+ bag = dict(source_field.get('link') or {})
1023
+ if bag.get('on') or bag.get('inverse'):
1024
+ return cur
1025
+ target_key = str(bag.get('table') or '')
1026
+ target = cur.get(target_key)
1027
+ if target is None:
1028
+ return cur
1029
+ reciprocal_key = _reciprocal_link_key(table_key, field_key)
1030
+ reciprocal = _clean_field({
1031
+ 'key': reciprocal_key,
1032
+ 'label': str(source.get('label') or table_key)[:80],
1033
+ 'type': 'link',
1034
+ 'link': {'table': table_key, 'inverse': field_key,
1035
+ 'reciprocal': field_key},
1036
+ 'editRole': 'admins',
1037
+ })
1038
+ if reciprocal is None:
1039
+ return cur
1040
+ target.setdefault('fields', []).append(reciprocal)
1041
+ bag['reciprocal'] = reciprocal_key
1042
+ source_field['link'] = bag
1043
+ result['field'] = dict(source_field)
1044
+ result['reciprocal'] = dict(reciprocal)
1045
+ return cur
1046
+
1047
+ _st(st).update(STORE_KEY, _sync, flush='sync')
1048
+ return result
1049
+
1050
+
1051
  def patch_field(table_key, fkey, raw, st=None):
1052
  """Edit one column's definition IN PLACE. Returns the stored field, or None if refused.
1053
 
 
1099
  fields = (get(table_key, st) or {}).get('fields') or []
1100
  if len(fields) <= 1 or not any(f.get('key') == str(fkey) for f in fields):
1101
  return False
1102
+ doomed = next((f for f in fields if f.get('key') == str(fkey)), {})
1103
+ doomed_link = doomed.get('link') if isinstance(doomed.get('link'), dict) else {}
1104
+ inverse_source = None
1105
+ if doomed_link.get('inverse'):
1106
+ source_table = str(doomed_link.get('table') or '')
1107
+ source_field = str(doomed_link.get('inverse') or '')
1108
+ source = get(source_table, st) or {}
1109
+ if len(source.get('fields') or []) <= 1:
1110
+ return False
1111
+ inverse_source = (source_table, source_field)
1112
 
1113
  def _drop(cur):
1114
  t = cur.get(str(table_key))
1115
  if t is not None:
1116
  t['fields'] = [f for f in (t.get('fields') or []) if f.get('key') != str(fkey)]
1117
+ if inverse_source:
1118
+ source_table, source_field = inverse_source
1119
+ source = cur.get(source_table)
1120
+ if isinstance(source, dict):
1121
+ source['fields'] = [f for f in (source.get('fields') or [])
1122
+ if f.get('key') != source_field]
1123
+ # An ordinary link owns its reciprocal field. Delete the reciprocal in the same schema
1124
+ # update so no target database can retain a live-looking backlink to a missing source.
1125
+ for candidate in cur.values():
1126
+ if not isinstance(candidate, dict):
1127
+ continue
1128
+ candidate['fields'] = [f for f in (candidate.get('fields') or [])
1129
+ if not (isinstance(f.get('link'), dict)
1130
+ and ((f['link'].get('table') == str(table_key)
1131
+ and f['link'].get('inverse') == str(fkey))
1132
+ or (inverse_source
1133
+ and f['link'].get('table') == inverse_source[0]
1134
+ and f['link'].get('inverse') == inverse_source[1])))]
1135
  return cur
1136
 
1137
  _st(st).update(STORE_KEY, _drop, flush='sync')
 
1229
  return True
1230
 
1231
 
1232
+ def patch_link_cell(table_key, row_id, fkey, value, st=None):
1233
+ """Persist one user-picked Link cell in the shared row and return its canonical id string.
1234
+
1235
+ A Link is a relationship in the database schema, not one user's visual overlay. Persisting
1236
+ it here makes the reciprocal field and every Rollup see the same source of truth. Derived
1237
+ joins and inverse fields are engine-owned and are refused by this door.
1238
+ """
1239
+ table_key, row_id, fkey = str(table_key), str(row_id), str(fkey)
1240
+ if not is_user_table(table_key, st) or not records_mutable(table_key, st):
1241
+ return None
1242
+ table = get(table_key, st) or {}
1243
+ if row_id not in (table.get('rows') or {}):
1244
+ return None
1245
+ field = next((f for f in (table.get('fields') or []) if f.get('key') == fkey), None)
1246
+ bag = (field or {}).get('link')
1247
+ if (field or {}).get('type') != 'link' or not isinstance(bag, dict) \
1248
+ or bag.get('on') or bag.get('inverse'):
1249
+ return None
1250
+ target = get(str(bag.get('table') or ''), st) or {}
1251
+ valid = set((target.get('rows') or {}).keys())
1252
+ raw_ids = value if isinstance(value, (list, tuple, set)) else str(value or '').split(',')
1253
+ picked, seen = [], set()
1254
+ for raw_id in raw_ids:
1255
+ rid = str(raw_id).strip()
1256
+ if not rid or rid in seen:
1257
+ continue
1258
+ if rid not in valid or len(picked) >= LINK_MAX_IDS:
1259
+ return None
1260
+ seen.add(rid)
1261
+ picked.append(rid)
1262
+ if bag.get('single'):
1263
+ break
1264
+ canonical = ','.join(picked)
1265
+
1266
+ def _set(cur):
1267
+ current = cur.get(table_key)
1268
+ if current is not None and row_id in (current.get('rows') or {}):
1269
+ current['rows'][row_id][fkey] = canonical
1270
+ return cur
1271
+
1272
+ _st(st).update(STORE_KEY, _set, flush='sync')
1273
+ return canonical
1274
+
1275
+
1276
  def patch_profile_cell(table_key, row_id, fkey, value, st=None):
1277
  """Write a PROFILE cell β€” and, when it is blanked, clear that row's preset cells IN THE SAME
1278
  WRITE (wave 25, contract C3 + owner ruling R6). Returns
 
1337
 
1338
 
1339
  def delete_row(table_key, row_id, st=None):
1340
+ if not is_user_table(table_key, st) or not records_mutable(table_key, st):
1341
  return False
1342
 
1343
  def _drop(cur):
web/src/customer-grid/ColumnMenu.tsx CHANGED
@@ -5,7 +5,7 @@ import { FieldTypeIcon, MenuLabel } from "./icons";
5
  import { FieldSelectButton } from "./FieldSelect";
6
  import { CREATABLE_TYPES, choiceOptions, choiceRenames, directionLabel, isMachineOwned,
7
  isProfileField, parseOptions, ratingMax, ROLLUP_FN_LABELS, ROLLUP_FNS } from "./types";
8
- import type { Field, FieldFormat, FieldScope, FieldType, Measure, RollupFn,
9
  Viewer } from "./types";
10
  import type { LinkTarget } from "./apiBridge";
11
  import type { WindowSpec } from "./windows";
@@ -63,6 +63,9 @@ interface FieldConfigExtra {
63
  rollup?: {
64
  link: string; field?: string; fn: string;
65
  limit?: number; sortBy?: string; sortDir?: "asc" | "desc";
 
 
 
66
  };
67
  }
68
 
@@ -83,6 +86,8 @@ interface ColumnMenuProps {
83
  */
84
  linkTargets?: LinkTarget[];
85
  locked: boolean;
 
 
86
  /** Wave-5 item 1 β€” who is looking. Gates the permissions entry (creator-or-admin). */
87
  viewer?: Viewer;
88
  /** Wave-5 item 3 β€” what the CURRENT VIEW does with this field, so the conditional
@@ -748,6 +753,12 @@ function ExtraTypeEditor({
748
  onRollupLimit,
749
  rollupSortBy = "",
750
  onRollupSortBy,
 
 
 
 
 
 
751
  }: {
752
  kind: FieldType | "measure";
753
  fields: Field[];
@@ -782,6 +793,12 @@ function ExtraTypeEditor({
782
  onRollupLimit?: (v: number) => void;
783
  rollupSortBy?: string;
784
  onRollupSortBy?: (v: string) => void;
 
 
 
 
 
 
785
  }) {
786
  if (kind === "formula") {
787
  // 2026-07-31 (owner item 2): other FORMULA fields are referencable now β€” evaluation is
@@ -1028,6 +1045,96 @@ function ExtraTypeEditor({
1028
  </select>
1029
  </label>
1030
  ) : null}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1031
  <label>
1032
  <span>Across</span>
1033
  <select
@@ -1058,6 +1165,21 @@ function ExtraTypeEditor({
1058
  />
1059
  </label>
1060
  ) : null}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1061
  {/* β›” THE RULE STATED AT THE CONTROL, because the server refuses the pair and a refusal
1062
  the user meets after pressing Create is a refusal they had no way to avoid. "The
1063
  last N" with no declared order is not a measurement β€” it is whichever N rows happen
@@ -1067,6 +1189,12 @@ function ExtraTypeEditor({
1067
  ? `The ${rollupLimit || 1} most recent linked records, newest first.`
1068
  : "Every linked record. Choose an order above to summarise just the most recent few."}
1069
  </div>
 
 
 
 
 
 
1070
  <div className="cg-field-hint">
1071
  Computed for you and refreshed as the linked records change β€” the cell cannot be typed
1072
  into.
@@ -1083,6 +1211,7 @@ export default function ColumnMenu({
1083
  fields,
1084
  linkTargets = [],
1085
  locked,
 
1086
  viewer,
1087
  sortedDir,
1088
  isFiltered,
@@ -1118,7 +1247,7 @@ export default function ColumnMenu({
1118
  userOptions = [],
1119
  measures = [],
1120
  }: ColumnMenuProps) {
1121
- const [pane, setPane] = useState<MenuPane>(initialPane ?? "menu");
1122
  const [note, setNote] = useState(field.note ?? "");
1123
  const [position, setPosition] = useState<CreatePosition | null>(initialPosition ?? null);
1124
  /** Delete is DESTRUCTIVE (a custom field's stored values go with it) β€” first click arms,
@@ -1159,6 +1288,9 @@ export default function ColumnMenu({
1159
  /** 0 = every linked record (Airtable's only behaviour). Non-zero needs a sort β€” see below. */
1160
  const [rollupLimit, setRollupLimit] = useState(0);
1161
  const [rollupSortBy, setRollupSortBy] = useState("");
 
 
 
1162
  const [swapTo, setSwapTo] = useState("");
1163
  /** The "New field" half of Change-field keeps its own name/options β€” a half-typed insert
1164
  * form must not leak into a swap and vice versa. */
@@ -1290,7 +1422,12 @@ export default function ColumnMenu({
1290
  // absent on the next read. The same "permanently blank column" argument as the two rules
1291
  // above, one step worse β€” the column does not exist at all.
1292
  (kind !== "link" || linkTable !== "") &&
1293
- (kind !== "rollup" || (rollupLink !== "" && (rollupFn === "countall" || rollupField !== "")));
 
 
 
 
 
1294
 
1295
  const extraFor = (
1296
  t: CreateKind,
@@ -1323,7 +1460,11 @@ export default function ColumnMenu({
1323
  // with no declared order, so sending one would turn a valid-looking form into a 400.
1324
  ...(rollupSortBy
1325
  ? { sortBy: rollupSortBy, sortDir: "desc" as const,
1326
- ...(rollupLimit ? { limit: rollupLimit } : {}) }
 
 
 
 
1327
  : {}),
1328
  },
1329
  };
@@ -1671,7 +1812,13 @@ export default function ColumnMenu({
1671
  linkSingle={linkSingle}
1672
  onLinkSingle={setLinkSingle}
1673
  rollupLink={rollupLink}
1674
- onRollupLink={setRollupLink}
 
 
 
 
 
 
1675
  rollupField={rollupField}
1676
  onRollupField={setRollupField}
1677
  rollupFn={rollupFn}
@@ -1680,6 +1827,12 @@ export default function ColumnMenu({
1680
  onRollupLimit={setRollupLimit}
1681
  rollupSortBy={rollupSortBy}
1682
  onRollupSortBy={setRollupSortBy}
 
 
 
 
 
 
1683
  />
1684
  {needsOptions(kind) && (
1685
  <OptionsEditor
@@ -2307,7 +2460,7 @@ export default function ColumnMenu({
2307
  is one line that STATES the period and opens that pane, because the window is the
2308
  first thing you want to know about a metric column and the menu had become the only
2309
  place it was written down. A row, not a control: the menu pane edits nothing inline. */}
2310
- {canPeriod && (
2311
  <button
2312
  type="button"
2313
  className="cg-column-periodline"
@@ -2329,17 +2482,19 @@ export default function ColumnMenu({
2329
  the one window for the field's name, type (with per-type choices/stars/formula)
2330
  and the Change-field control. Offered on EVERY field β€” what a read-only field
2331
  cannot change, the pane says honestly instead of hiding the door. */}
2332
- <button type="button" data-overlay-autofocus onClick={() => {
2333
- setRenameDraft(field.label);
2334
- setPane("edit");
2335
- }}>
2336
- <MenuLabel icon="rename" text="Edit field" />
2337
- </button>
 
 
2338
 
2339
  <div className="cg-menu-sep" role="separator" aria-hidden />
2340
 
2341
  {/* Group 1 β€” Duplicate Β· Insert left Β· Insert right (+ Add at end, same family). */}
2342
- {onDuplicate && (
2343
  <button type="button" onClick={() => { onDuplicate(); onClose(); }}>
2344
  <MenuLabel icon="duplicate" text="Duplicate field" />
2345
  </button>
@@ -2359,15 +2514,17 @@ export default function ColumnMenu({
2359
  {/* Group 2 β€” the rest of the def-editing family: description Β· permissions Β· format.
2360
  Rename and Change field live INSIDE Edit field now (owner item 8) β€” their old
2361
  rows are gone, not duplicated. */}
2362
- <button type="button" onClick={() => setPane("note")}>
2363
- <MenuLabel icon="description" text="Edit field description" />
2364
- </button>
2365
- {canPermissions && (
 
 
2366
  <button type="button" onClick={() => setPane("permissions")}>
2367
  <MenuLabel icon="permissions" text="Edit field permissions" />
2368
  </button>
2369
  )}
2370
- {formatKind && (
2371
  <button type="button" onClick={() => setPane("format")}>
2372
  <MenuLabel icon="format" text="Field format" />
2373
  </button>
@@ -2447,7 +2604,7 @@ export default function ColumnMenu({
2447
  >
2448
  <MenuLabel icon="hide" text="Hide field" />
2449
  </button>
2450
- {onDelete && (
2451
  <button
2452
  type="button"
2453
  className="is-danger"
 
5
  import { FieldSelectButton } from "./FieldSelect";
6
  import { CREATABLE_TYPES, choiceOptions, choiceRenames, directionLabel, isMachineOwned,
7
  isProfileField, parseOptions, ratingMax, ROLLUP_FN_LABELS, ROLLUP_FNS } from "./types";
8
+ import type { Field, FieldFormat, FieldScope, FieldType, Measure, RollupCondition, RollupFn,
9
  Viewer } from "./types";
10
  import type { LinkTarget } from "./apiBridge";
11
  import type { WindowSpec } from "./windows";
 
63
  rollup?: {
64
  link: string; field?: string; fn: string;
65
  limit?: number; sortBy?: string; sortDir?: "asc" | "desc";
66
+ distinctBy?: string;
67
+ conditions?: RollupCondition[];
68
+ conditionConj?: "and" | "or";
69
  };
70
  }
71
 
 
86
  */
87
  linkTargets?: LinkTarget[];
88
  locked: boolean;
89
+ /** Product-owned preset fields may still drive view actions, but their schema is immutable. */
90
+ schemaLocked?: boolean;
91
  /** Wave-5 item 1 β€” who is looking. Gates the permissions entry (creator-or-admin). */
92
  viewer?: Viewer;
93
  /** Wave-5 item 3 β€” what the CURRENT VIEW does with this field, so the conditional
 
753
  onRollupLimit,
754
  rollupSortBy = "",
755
  onRollupSortBy,
756
+ rollupDistinctBy = "",
757
+ onRollupDistinctBy,
758
+ rollupConditions = [],
759
+ onRollupConditions,
760
+ rollupConditionConj = "and",
761
+ onRollupConditionConj,
762
  }: {
763
  kind: FieldType | "measure";
764
  fields: Field[];
 
793
  onRollupLimit?: (v: number) => void;
794
  rollupSortBy?: string;
795
  onRollupSortBy?: (v: string) => void;
796
+ rollupDistinctBy?: string;
797
+ onRollupDistinctBy?: (v: string) => void;
798
+ rollupConditions?: RollupCondition[];
799
+ onRollupConditions?: (v: RollupCondition[]) => void;
800
+ rollupConditionConj?: "and" | "or";
801
+ onRollupConditionConj?: (v: "and" | "or") => void;
802
  }) {
803
  if (kind === "formula") {
804
  // 2026-07-31 (owner item 2): other FORMULA fields are referencable now β€” evaluation is
 
1045
  </select>
1046
  </label>
1047
  ) : null}
1048
+ <div className="cg-rollup-conditions">
1049
+ <div className="cg-rollup-conditions__head">
1050
+ <span>Conditions</span>
1051
+ <button
1052
+ type="button"
1053
+ className="cg-btn"
1054
+ disabled={targetFields.length === 0 || rollupConditions.length >= 20}
1055
+ onClick={() => onRollupConditions?.([
1056
+ ...rollupConditions,
1057
+ { field: targetFields[0]?.key ?? "", op: "eq", value: "" },
1058
+ ])}
1059
+ >
1060
+ Add condition
1061
+ </button>
1062
+ </div>
1063
+ {rollupConditions.length > 1 ? (
1064
+ <select
1065
+ className="cg-input"
1066
+ aria-label="Match all or any rollup conditions"
1067
+ value={rollupConditionConj}
1068
+ onChange={(event) =>
1069
+ onRollupConditionConj?.(event.target.value as "and" | "or")
1070
+ }
1071
+ >
1072
+ <option value="and">All conditions must match</option>
1073
+ <option value="or">Any condition may match</option>
1074
+ </select>
1075
+ ) : null}
1076
+ {rollupConditions.map((condition, index) => (
1077
+ <div className="cg-rollup-condition" key={`${index}-${condition.field}`}>
1078
+ <select
1079
+ className="cg-input"
1080
+ aria-label={`Condition ${index + 1} column`}
1081
+ value={condition.field}
1082
+ onChange={(event) => onRollupConditions?.(rollupConditions.map((item, i) =>
1083
+ i === index ? { ...item, field: event.target.value } : item
1084
+ ))}
1085
+ >
1086
+ {targetFields.map((targetField) => (
1087
+ <option key={targetField.key} value={targetField.key}>{targetField.label}</option>
1088
+ ))}
1089
+ </select>
1090
+ <select
1091
+ className="cg-input"
1092
+ aria-label={`Condition ${index + 1} operator`}
1093
+ value={condition.op}
1094
+ onChange={(event) => onRollupConditions?.(rollupConditions.map((item, i) =>
1095
+ i === index
1096
+ ? { ...item, op: event.target.value as RollupCondition["op"] }
1097
+ : item
1098
+ ))}
1099
+ >
1100
+ <option value="eq">is</option>
1101
+ <option value="neq">is not</option>
1102
+ <option value="contains">contains</option>
1103
+ <option value="not_contains">does not contain</option>
1104
+ <option value="is_empty">is empty</option>
1105
+ <option value="is_not_empty">is not empty</option>
1106
+ <option value="gt">is greater than</option>
1107
+ <option value="gte">is at least</option>
1108
+ <option value="lt">is less than</option>
1109
+ <option value="lte">is at most</option>
1110
+ </select>
1111
+ {!(["is_empty", "is_not_empty"] as string[]).includes(condition.op) ? (
1112
+ <input
1113
+ className="cg-input"
1114
+ aria-label={`Condition ${index + 1} value`}
1115
+ value={condition.value ?? ""}
1116
+ onChange={(event) => onRollupConditions?.(rollupConditions.map((item, i) =>
1117
+ i === index ? { ...item, value: event.target.value } : item
1118
+ ))}
1119
+ />
1120
+ ) : null}
1121
+ <button
1122
+ type="button"
1123
+ className="cg-icon-btn"
1124
+ aria-label={`Remove condition ${index + 1}`}
1125
+ onClick={() => onRollupConditions?.(
1126
+ rollupConditions.filter((_item, i) => i !== index)
1127
+ )}
1128
+ >
1129
+ Γ—
1130
+ </button>
1131
+ </div>
1132
+ ))}
1133
+ <div className="cg-field-hint">
1134
+ Like Airtable, these conditions select linked records for this Rollup; view filters
1135
+ do not change its result.
1136
+ </div>
1137
+ </div>
1138
  <label>
1139
  <span>Across</span>
1140
  <select
 
1165
  />
1166
  </label>
1167
  ) : null}
1168
+ <label>
1169
+ <span>Deduplicate by</span>
1170
+ <select
1171
+ id={`${idPrefix}-rollup-distinct`}
1172
+ className="cg-input"
1173
+ value={rollupDistinctBy}
1174
+ aria-label="Deduplicate linked records by"
1175
+ onChange={(event) => onRollupDistinctBy?.(event.target.value)}
1176
+ >
1177
+ <option value="">Keep every linked row</option>
1178
+ {targetFields.map((f) => (
1179
+ <option key={f.key} value={f.key}>One row per {f.label}</option>
1180
+ ))}
1181
+ </select>
1182
+ </label>
1183
  {/* β›” THE RULE STATED AT THE CONTROL, because the server refuses the pair and a refusal
1184
  the user meets after pressing Create is a refusal they had no way to avoid. "The
1185
  last N" with no declared order is not a measurement β€” it is whichever N rows happen
 
1189
  ? `The ${rollupLimit || 1} most recent linked records, newest first.`
1190
  : "Every linked record. Choose an order above to summarise just the most recent few."}
1191
  </div>
1192
+ {rollupDistinctBy ? (
1193
+ <div className="cg-field-hint">
1194
+ Duplicate linked rows with the same identity count once; the newest row wins when an
1195
+ order is selected.
1196
+ </div>
1197
+ ) : null}
1198
  <div className="cg-field-hint">
1199
  Computed for you and refreshed as the linked records change β€” the cell cannot be typed
1200
  into.
 
1211
  fields,
1212
  linkTargets = [],
1213
  locked,
1214
+ schemaLocked = false,
1215
  viewer,
1216
  sortedDir,
1217
  isFiltered,
 
1247
  userOptions = [],
1248
  measures = [],
1249
  }: ColumnMenuProps) {
1250
+ const [pane, setPane] = useState<MenuPane>(schemaLocked ? "menu" : initialPane ?? "menu");
1251
  const [note, setNote] = useState(field.note ?? "");
1252
  const [position, setPosition] = useState<CreatePosition | null>(initialPosition ?? null);
1253
  /** Delete is DESTRUCTIVE (a custom field's stored values go with it) β€” first click arms,
 
1288
  /** 0 = every linked record (Airtable's only behaviour). Non-zero needs a sort β€” see below. */
1289
  const [rollupLimit, setRollupLimit] = useState(0);
1290
  const [rollupSortBy, setRollupSortBy] = useState("");
1291
+ const [rollupDistinctBy, setRollupDistinctBy] = useState("");
1292
+ const [rollupConditions, setRollupConditions] = useState<RollupCondition[]>([]);
1293
+ const [rollupConditionConj, setRollupConditionConj] = useState<"and" | "or">("and");
1294
  const [swapTo, setSwapTo] = useState("");
1295
  /** The "New field" half of Change-field keeps its own name/options β€” a half-typed insert
1296
  * form must not leak into a swap and vice versa. */
 
1422
  // absent on the next read. The same "permanently blank column" argument as the two rules
1423
  // above, one step worse β€” the column does not exist at all.
1424
  (kind !== "link" || linkTable !== "") &&
1425
+ (kind !== "rollup" || (
1426
+ rollupLink !== "" &&
1427
+ (rollupFn === "countall" || rollupField !== "") &&
1428
+ // `latest` without an order is store order wearing a deterministic name.
1429
+ (rollupFn !== "latest" || rollupSortBy !== "")
1430
+ ));
1431
 
1432
  const extraFor = (
1433
  t: CreateKind,
 
1460
  // with no declared order, so sending one would turn a valid-looking form into a 400.
1461
  ...(rollupSortBy
1462
  ? { sortBy: rollupSortBy, sortDir: "desc" as const,
1463
+ limit: rollupLimit || 1 }
1464
+ : {}),
1465
+ ...(rollupDistinctBy ? { distinctBy: rollupDistinctBy } : {}),
1466
+ ...(rollupConditions.length
1467
+ ? { conditions: rollupConditions, conditionConj: rollupConditionConj }
1468
  : {}),
1469
  },
1470
  };
 
1812
  linkSingle={linkSingle}
1813
  onLinkSingle={setLinkSingle}
1814
  rollupLink={rollupLink}
1815
+ onRollupLink={(value) => {
1816
+ setRollupLink(value);
1817
+ setRollupField("");
1818
+ setRollupSortBy("");
1819
+ setRollupDistinctBy("");
1820
+ setRollupConditions([]);
1821
+ }}
1822
  rollupField={rollupField}
1823
  onRollupField={setRollupField}
1824
  rollupFn={rollupFn}
 
1827
  onRollupLimit={setRollupLimit}
1828
  rollupSortBy={rollupSortBy}
1829
  onRollupSortBy={setRollupSortBy}
1830
+ rollupDistinctBy={rollupDistinctBy}
1831
+ onRollupDistinctBy={setRollupDistinctBy}
1832
+ rollupConditions={rollupConditions}
1833
+ onRollupConditions={setRollupConditions}
1834
+ rollupConditionConj={rollupConditionConj}
1835
+ onRollupConditionConj={setRollupConditionConj}
1836
  />
1837
  {needsOptions(kind) && (
1838
  <OptionsEditor
 
2460
  is one line that STATES the period and opens that pane, because the window is the
2461
  first thing you want to know about a metric column and the menu had become the only
2462
  place it was written down. A row, not a control: the menu pane edits nothing inline. */}
2463
+ {canPeriod && !schemaLocked && (
2464
  <button
2465
  type="button"
2466
  className="cg-column-periodline"
 
2482
  the one window for the field's name, type (with per-type choices/stars/formula)
2483
  and the Change-field control. Offered on EVERY field β€” what a read-only field
2484
  cannot change, the pane says honestly instead of hiding the door. */}
2485
+ {!schemaLocked && (
2486
+ <button type="button" data-overlay-autofocus onClick={() => {
2487
+ setRenameDraft(field.label);
2488
+ setPane("edit");
2489
+ }}>
2490
+ <MenuLabel icon="rename" text="Edit field" />
2491
+ </button>
2492
+ )}
2493
 
2494
  <div className="cg-menu-sep" role="separator" aria-hidden />
2495
 
2496
  {/* Group 1 β€” Duplicate Β· Insert left Β· Insert right (+ Add at end, same family). */}
2497
+ {!schemaLocked && onDuplicate && (
2498
  <button type="button" onClick={() => { onDuplicate(); onClose(); }}>
2499
  <MenuLabel icon="duplicate" text="Duplicate field" />
2500
  </button>
 
2514
  {/* Group 2 β€” the rest of the def-editing family: description Β· permissions Β· format.
2515
  Rename and Change field live INSIDE Edit field now (owner item 8) β€” their old
2516
  rows are gone, not duplicated. */}
2517
+ {!schemaLocked && (
2518
+ <button type="button" onClick={() => setPane("note")}>
2519
+ <MenuLabel icon="description" text="Edit field description" />
2520
+ </button>
2521
+ )}
2522
+ {!schemaLocked && canPermissions && (
2523
  <button type="button" onClick={() => setPane("permissions")}>
2524
  <MenuLabel icon="permissions" text="Edit field permissions" />
2525
  </button>
2526
  )}
2527
+ {!schemaLocked && formatKind && (
2528
  <button type="button" onClick={() => setPane("format")}>
2529
  <MenuLabel icon="format" text="Field format" />
2530
  </button>
 
2604
  >
2605
  <MenuLabel icon="hide" text="Hide field" />
2606
  </button>
2607
+ {!schemaLocked && onDelete && (
2608
  <button
2609
  type="button"
2610
  className="is-danger"
web/src/customer-grid/CustomerGrid.tsx CHANGED
@@ -93,13 +93,14 @@ import {
93
  } from "./clipboard";
94
  import { cellTipText, expandButtonRect, GROUP_HEADER_FONT, GROUP_LABEL_PAD, headerMarkLayout,
95
  headerMarkSizes, tipLeft } from "./overlayPlacement";
96
- import { AnchoredOverlay } from "./OverlaySurface";
97
  import type { AnchorRect } from "./OverlaySurface";
98
  import { StarIcon } from "./Stars";
99
  import { ALL_VIEW_ID, MAX_CALENDAR_METRICS, allViewName,
100
  MAX_FROZEN, choiceOptions, choiceVocabulary, clampFrozenCount, cleanDisplay, formulaOf,
101
  topicForScope,
102
  isDateFamilyType, isFilterGroup, isGroupableField, isMachineOwned, isMachineWritten,
 
103
  isNumericFieldType,
104
  isPickType, mayEditField,
105
  isModeFrozen, isUndeletableView, mayEditView, mayToggleViewLock,
@@ -495,18 +496,136 @@ function buildOverlayField(
495
  * transition whole (the fold froze, then SNAPPED β€” the exact "static" the owner named). The
496
  * props surface is one stable string, so memo makes chrome state changes free; the grid still
497
  * re-renders for its own state (edits, resize) and remounts on route change via `key`. */
498
- function CustomerGrid({ scope = "customer" }: { scope?: SurfaceScope } = {}) {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
499
  // Wave 16 C-TOPIC: which TABLE this tree is drawing, derived from the one scope prop.
500
  const topic = topicForScope(scope);
501
  const {
502
  fields: payloadFields,
503
- rawRows,
504
  payload,
505
  loading,
506
  overlayEdits,
507
  setOverlayEdits,
508
  patchOverlay,
509
- } = useCustomerData(scope);
 
 
 
 
 
 
 
 
 
 
 
 
510
 
511
  const [fields, setFields] = useState<Field[]>([]);
512
  const [views, setViews] = useState<SavedView[]>([]);
@@ -551,6 +670,14 @@ function CustomerGrid({ scope = "customer" }: { scope?: SurfaceScope } = {}) {
551
  setJsonAt(null);
552
  requestAnimationFrame(() => gridRef.current?.focus());
553
  }, []);
 
 
 
 
 
 
 
 
554
  /** WAVE 21 item 11 (R10) β€” is "Select records from a list" open? Opened from the view
555
  * rail's "…" and closed by the dialog; the SELECTION it produces outlives it. */
556
  const [selectFromFile, setSelectFromFile] = useState(false);
@@ -672,7 +799,7 @@ function CustomerGrid({ scope = "customer" }: { scope?: SurfaceScope } = {}) {
672
  // fills only missing objects and keeps the standalone path useful.
673
  useEffect(() => {
674
  if (!payload || payloadFields.length === 0 || initializedKey.current === storageKey) return;
675
- const local = readLocal(storageKey);
676
  // Item 3c: the def half of the no-blip layer. A RECENT local stamp beats a
677
  // lagged host echo (rename survives, retype holds, a delete stays deleted);
678
  // a caught-up echo returns host objects byte-identical (see optimism.ts).
@@ -688,7 +815,7 @@ function CustomerGrid({ scope = "customer" }: { scope?: SurfaceScope } = {}) {
688
  Date.now(),
689
  payload.workspace != null
690
  );
691
- const hostViews = payload.workspace?.views ?? [];
692
  const byId = new Map<string, SavedView>();
693
  byId.set(ALL_VIEW_ID, allRecordsView(initialFields, scope));
694
  // D-19 β€” the HOST'S LIST DECIDES WHICH VIEWS EXIST. A local copy the host no longer
@@ -765,7 +892,7 @@ function CustomerGrid({ scope = "customer" }: { scope?: SurfaceScope } = {}) {
765
  * the config the client is actually filtering with, and sending anything else would ask the
766
  * host to resolve a question nobody on screen is asking.
767
  */
768
- if (reemit.size > 0) {
769
  const stamped = { ...(local?.reemitted ?? {}) };
770
  for (const view of initialViews) {
771
  const key = reemit.get(view.id);
@@ -780,10 +907,10 @@ function CustomerGrid({ scope = "customer" }: { scope?: SurfaceScope } = {}) {
780
  } else {
781
  reemittedRef.current = { ...(local?.reemitted ?? {}) };
782
  }
783
- }, [payload, payloadFields, storageKey]);
784
 
785
  useEffect(() => {
786
- if (!workspaceReady) return;
787
  writeLocal(storageKey, {
788
  fields,
789
  views,
@@ -800,7 +927,7 @@ function CustomerGrid({ scope = "customer" }: { scope?: SurfaceScope } = {}) {
800
  // drop a view this browser created seconds ago.
801
  viewWrites: pruneTombstones(viewWritesRef.current, Date.now()),
802
  });
803
- }, [workspaceReady, storageKey, fields, views, activeViewId]);
804
 
805
  /**
806
  * ⭐ THE LIVE WORKSPACE (owner report, 2026-08-04) β€” what has APPEARED since we mounted.
@@ -825,7 +952,7 @@ function CustomerGrid({ scope = "customer" }: { scope?: SurfaceScope } = {}) {
825
  */
826
  const hostWorkspaceViews = payload?.workspace?.views;
827
  useEffect(() => {
828
- if (!workspaceReady) return;
829
  const now = Date.now();
830
  const nextFields = adoptNewFields(
831
  fields, payloadFields, fieldStampsRef.current.deleted, now
@@ -835,7 +962,7 @@ function CustomerGrid({ scope = "customer" }: { scope?: SurfaceScope } = {}) {
835
  adoptNewViews(current, hostWorkspaceViews, viewTombstonesRef.current, now,
836
  (config) => normalizeConfig(config, nextFields))
837
  );
838
- }, [workspaceReady, hostWorkspaceViews, payloadFields, fields]);
839
 
840
  /** Item 3c β€” stamp a def write / a delete. Pruned at every touch so the persisted blob
841
  * stays a recent window, never an archive. */
@@ -896,7 +1023,9 @@ function CustomerGrid({ scope = "customer" }: { scope?: SurfaceScope } = {}) {
896
  * this resolves, the appended row does not exist yet, so "bottom" would land on the last OLD
897
  * row.
898
  */
899
- const isUserTable = scope.startsWith("ut_");
 
 
900
  /**
901
  * ⭐ 2026-08-07 β€” the databases a `link` column may point at, fetched when the column menu
902
  * OPENS rather than on every render of the grid.
@@ -921,7 +1050,7 @@ function CustomerGrid({ scope = "customer" }: { scope?: SurfaceScope } = {}) {
921
  };
922
  }, [columnMenu, linkTargets.length]);
923
  const appendRow = useCallback(async (): Promise<undefined> => {
924
- if (!isUserTable) return undefined;
925
  const made = await addTableRow(scope);
926
  // `undefined` either way, never glide's "bottom": glide would move the selection to the
927
  // last row it currently knows about, which is the row BEFORE the one just created. The
@@ -936,7 +1065,7 @@ function CustomerGrid({ scope = "customer" }: { scope?: SurfaceScope } = {}) {
936
  });
937
  signal(ROWS_STALE_EVENT);
938
  return undefined;
939
- }, [isUserTable, scope]);
940
 
941
  /* wave20 item 2 β€” measure key + window -> the columns that display it. Built ONCE per field
942
  list and handed to every consumer of "which column is this rule about", so the tint
@@ -946,7 +1075,7 @@ function CustomerGrid({ scope = "customer" }: { scope?: SurfaceScope } = {}) {
946
 
947
  // Airtable behavior: configuration changes to the active view autosave.
948
  useEffect(() => {
949
- if (!workspaceReady) return;
950
  const active = views.find((view) => view.id === activeViewId);
951
  if (!active || sameConfig(active.config, config)) {
952
  setSaveState("saved");
@@ -968,7 +1097,7 @@ function CustomerGrid({ scope = "customer" }: { scope?: SurfaceScope } = {}) {
968
  return () => {
969
  if (saveTimer.current !== null) window.clearTimeout(saveTimer.current);
970
  };
971
- }, [workspaceReady, activeViewId, config, views]);
972
 
973
  // Numeric dimensions force a glide relayout when either the component frame
974
  // or Streamlit's main column changes width (notably sidebar collapse).
@@ -999,7 +1128,7 @@ function CustomerGrid({ scope = "customer" }: { scope?: SurfaceScope } = {}) {
999
  const mode = tableMode(payload?.counts);
1000
  const serverWindowed = mode === "server-windowed";
1001
  // Owner items 4+6 β€” the Cohort page's grid renders without the Views sidebar.
1002
- const hideViews = payload?.workspace?.hideViews === true;
1003
  // Wave-6 item 10 β€” how this view displays. A WINDOWED table is always the grid: list/
1004
  // calendar/kanban compute over the whole matched set, and one page is not it (CG-3's rule,
1005
  // the same reason grouping is off there).
@@ -1066,8 +1195,8 @@ function CustomerGrid({ scope = "customer" }: { scope?: SurfaceScope } = {}) {
1066
  // permissions vs the viewer, fail-closed on restricted fields when the viewer is unknown.
1067
  const viewer = payload?.viewer;
1068
  const canEditField = useCallback(
1069
- (f: Field): boolean => mayEditField(f, viewer),
1070
- [viewer]
1071
  );
1072
  const unresolvedCount = useMemo(
1073
  () => unresolvedConditions(config.filters, { cohortSets, today }),
@@ -1301,8 +1430,20 @@ function CustomerGrid({ scope = "customer" }: { scope?: SurfaceScope } = {}) {
1301
  useGridSelection(
1302
  displayRows,
1303
  displayPidToIndex,
1304
- visibleCols.length
 
1305
  );
 
 
 
 
 
 
 
 
 
 
 
1306
  /* ════════════════════════ owner item 16 / R4 / C-UNDO ════════════════════════
1307
  THE RECORDING LAYER. `undoStack.ts` owns the stack and every inverse; this owns the one
1308
  thing it cannot: reading the value a cell held BEFORE the write, which only exists at the
@@ -1519,7 +1660,7 @@ function CustomerGrid({ scope = "customer" }: { scope?: SurfaceScope } = {}) {
1519
  */
1520
  const deleteRecords = useCallback(
1521
  async (pids: number[]): Promise<boolean> => {
1522
- if (!isUserTable || !pids.length) return false;
1523
  // The row as it stands NOW, straight off the rendered records β€” the same values the
1524
  // reader can see, so a restore puts back what they watched disappear.
1525
  const byPid = new Map<number, Row>();
@@ -1639,7 +1780,7 @@ function CustomerGrid({ scope = "customer" }: { scope?: SurfaceScope } = {}) {
1639
  );
1640
  return true;
1641
  },
1642
- [isUserTable, displayRows, fields, scope, clearSelection, delArmed]
1643
  );
1644
 
1645
  const onGridDelete = useCallback(
@@ -1665,7 +1806,7 @@ function CustomerGrid({ scope = "customer" }: { scope?: SurfaceScope } = {}) {
1665
  */
1666
  const dragged = !!sel.current
1667
  && (sel.current.range.width > 1 || sel.current.range.height > 1);
1668
- if (isUserTable && sel.rows.length > 0 && !dragged) {
1669
  const pids: number[] = [];
1670
  for (const rowIndex of sel.rows) {
1671
  const vr = displayRows[rowIndex];
@@ -1708,7 +1849,8 @@ function CustomerGrid({ scope = "customer" }: { scope?: SurfaceScope } = {}) {
1708
  // fields (`onCellEdited`'s own `canEditField`), it just does not arrive as one entry.
1709
  return false;
1710
  },
1711
- [displayRows, visibleCols, fieldByKey, canEditField, patchManyAndRecord]
 
1712
  );
1713
 
1714
  /**
@@ -1940,7 +2082,7 @@ function CustomerGrid({ scope = "customer" }: { scope?: SurfaceScope } = {}) {
1940
  // (`.cg-grid-void` is `pointer-events: none`), which is the worst version of this bug:
1941
  // the gate went green on an affordance nobody could see. Found by READING THE SCREENSHOT
1942
  // ([[ui-invisible-to-assertions]], [[finalize-visual-review-sop]]).
1943
- if (isUserTable)
1944
  rowsPx += typeof rowHeight === "number" ? rowHeight : rowHeight(displayRows.length);
1945
 
1946
  /* Fit is tested against the client box the OTHER axis's scrollbar leaves behind β€” the same
@@ -1963,7 +2105,7 @@ function CustomerGrid({ scope = "customer" }: { scope?: SurfaceScope } = {}) {
1963
  below: rowsPx < clientH ? rowsPx : null,
1964
  right: colsPx < clientW ? colsPx : null,
1965
  };
1966
- }, [visibleCols, displayRows, rowHeight, gridSize, isUserTable]);
1967
  /* ═══ end W18-B VOID (geometry) ═══ */
1968
 
1969
  // The record drawer resolves positions against what the MODE paints: the display slice for
@@ -2163,6 +2305,17 @@ function CustomerGrid({ scope = "customer" }: { scope?: SurfaceScope } = {}) {
2163
  });
2164
  return;
2165
  }
 
 
 
 
 
 
 
 
 
 
 
2166
  // ⭐ Wave-23 C7 β€” a JSON cell opens the big viewer. It is the ONLY door: the cell carries
2167
  // `allowOverlay:false`, because glide's overlay is a one-line box and one keystroke in the
2168
  // wrong place inside a 32 KB document turns a well-formed payload into an unparseable one,
@@ -2318,6 +2471,7 @@ function CustomerGrid({ scope = "customer" }: { scope?: SurfaceScope } = {}) {
2318
 
2319
  const openHeaderMenu = useCallback(
2320
  (column: number, bounds: Rectangle) => {
 
2321
  const definition = visibleCols[column];
2322
  if (!definition?.id) return;
2323
  setColumnMenu({
@@ -2332,7 +2486,7 @@ function CustomerGrid({ scope = "customer" }: { scope?: SurfaceScope } = {}) {
2332
  },
2333
  });
2334
  },
2335
- [visibleCols]
2336
  );
2337
  const onHeaderClicked = useCallback(
2338
  (column: number, event: HeaderClickedEventArgs) => {
@@ -3956,7 +4110,7 @@ function CustomerGrid({ scope = "customer" }: { scope?: SurfaceScope } = {}) {
3956
  const menuVisIndex = menuField ? visibleKeys.indexOf(menuField.key) : -1;
3957
  const menuPinnedTo = menuVisIndex >= 0 && frozenN > 1 && menuVisIndex + 1 === frozenN;
3958
  // Item 10 β€” the toolbar's mode switcher (grid-only on windowed tables, see displayMode).
3959
- const modeControl = serverWindowed ? undefined : (
3960
  <ModeSwitch
3961
  mode={displayMode}
3962
  onMode={setDisplayMode}
@@ -3998,6 +4152,18 @@ function CustomerGrid({ scope = "customer" }: { scope?: SurfaceScope } = {}) {
3998
  ?? (jsonField && jsonRow ? jsonRow[jsonField.key] : "")
3999
  ?? ""
4000
  );
 
 
 
 
 
 
 
 
 
 
 
 
4001
  const pickerField = picker ? fieldByKey.get(picker.fieldKey) : undefined;
4002
  // A `user` field's choices come from the HOST's real user list, a `select`'s from its own
4003
  // definition β€” so an assignee is always someone who can log in, and a status is always one of
@@ -4313,7 +4479,7 @@ function CustomerGrid({ scope = "customer" }: { scope?: SurfaceScope } = {}) {
4313
  height={gridSize.height}
4314
  customRenderers={[ratingCellRenderer, userCellRenderer, imageCellRenderer]}
4315
  headerIcons={HEADER_ICONS}
4316
- rightElement={
4317
  // Wave-6 item 4 β€” the "+" of the header row: the create-field form,
4318
  // insert-at-end.
4319
  //
@@ -4350,14 +4516,14 @@ function CustomerGrid({ scope = "customer" }: { scope?: SurfaceScope } = {}) {
4350
  >
4351
  +
4352
  </button>
4353
- }
4354
  rightElementProps={{ sticky: false, fill: false }}
4355
  // Owner item 4 (R8) β€” the trailing "+" row, USER DATABASES ONLY. Passing
4356
  // `onRowAppended` is what makes glide paint the ghost row at all, so the
4357
  // affordance exists exactly where a POST can succeed. See `appendRow`.
4358
- onRowAppended={isUserTable ? appendRow : undefined}
4359
  trailingRowOptions={
4360
- isUserTable
4361
  ? {
4362
  // The hint sits in the identity column (targetColumn 0), which is the
4363
  // one a new record is named in β€” the same cell the cursor lands on.
@@ -4555,7 +4721,7 @@ function CustomerGrid({ scope = "customer" }: { scope?: SurfaceScope } = {}) {
4555
 
4556
  Rendered only in `grid` mode: list / calendar / kanban / map have their own
4557
  open affordances and glide is not mounted to report bounds for them. */}
4558
- {displayMode === "grid" && expandAt && (
4559
  <button
4560
  type="button"
4561
  className="cg-row-expand"
@@ -4615,7 +4781,7 @@ function CustomerGrid({ scope = "customer" }: { scope?: SurfaceScope } = {}) {
4615
  broke this gate's own coordinate math before it broke a user). It states the
4616
  count and offers "Add to cohort" over exactly the checked pids β€” the same guarded
4617
  add_to_list event the view menu uses, so the host treats both alike. */}
4618
- {selectedPids.size > 0 && !serverWindowed &&
4619
  (displayMode === "grid" || displayMode === "map") && (
4620
  <div className="cg-selbar">
4621
  <span className="cg-selbar-count">
@@ -4627,7 +4793,7 @@ function CustomerGrid({ scope = "customer" }: { scope?: SurfaceScope } = {}) {
4627
  ⚠ ONLY on a `ut_` database: the customer and product grids are projections of
4628
  Odoo, where a "delete" would be a write to somebody else's system of record β€”
4629
  the server refuses it, and a button that must be refused is worse than none. */}
4630
- {isUserTable && (
4631
  <button
4632
  type="button"
4633
  /* ARMED is a VISIBLE state, not just a changed handler: the button that is
@@ -4744,6 +4910,7 @@ function CustomerGrid({ scope = "customer" }: { scope?: SurfaceScope } = {}) {
4744
  // one-line picker, so it is left out until somebody asks for it.
4745
  linkTargets={linkTargets.filter((t) => t.key !== scope)}
4746
  locked={menuField.key === lockedKey}
 
4747
  viewer={viewer}
4748
  sortedDir={menuSortedDir}
4749
  isFiltered={menuIsFiltered}
@@ -5102,6 +5269,21 @@ function CustomerGrid({ scope = "customer" }: { scope?: SurfaceScope } = {}) {
5102
  onClose={closeJson}
5103
  />
5104
  )}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5105
  {pickerField && picker && (
5106
  <AnchoredOverlay
5107
  anchor={picker.anchor}
@@ -5299,7 +5481,7 @@ function CustomerGrid({ scope = "customer" }: { scope?: SurfaceScope } = {}) {
5299
  </AnchoredOverlay>
5300
  )}
5301
 
5302
- {detailRecord && detailPid !== null && (
5303
  <RecordDetail
5304
  fields={fields}
5305
  record={detailRecord}
 
93
  } from "./clipboard";
94
  import { cellTipText, expandButtonRect, GROUP_HEADER_FONT, GROUP_LABEL_PAD, headerMarkLayout,
95
  headerMarkSizes, tipLeft } from "./overlayPlacement";
96
+ import { AnchoredOverlay, BodyPortal, useOverlayLayer } from "./OverlaySurface";
97
  import type { AnchorRect } from "./OverlaySurface";
98
  import { StarIcon } from "./Stars";
99
  import { ALL_VIEW_ID, MAX_CALENDAR_METRICS, allViewName,
100
  MAX_FROZEN, choiceOptions, choiceVocabulary, clampFrozenCount, cleanDisplay, formulaOf,
101
  topicForScope,
102
  isDateFamilyType, isFilterGroup, isGroupableField, isMachineOwned, isMachineWritten,
103
+ isDerivedLink,
104
  isNumericFieldType,
105
  isPickType, mayEditField,
106
  isModeFrozen, isUndeletableView, mayEditView, mayToggleViewLock,
 
496
  * transition whole (the fold froze, then SNAPPED β€” the exact "static" the owner named). The
497
  * props surface is one stable string, so memo makes chrome state changes free; the grid still
498
  * re-renders for its own state (edits, resize) and remounts on route change via `key`. */
499
+ interface CustomerGridProps {
500
+ scope?: SurfaceScope;
501
+ /** Read-only Grid view embedded in a linked-record modal. It keeps the standard
502
+ * filter/sort/search toolbar while withdrawing schema and row mutations. */
503
+ embedded?: boolean;
504
+ /** Exact linked pids to project from the target database. */
505
+ embeddedRecordIds?: readonly number[];
506
+ /** Selection mode used by an editable ordinary-Link modal. */
507
+ embeddedSelectable?: boolean;
508
+ embeddedSelectedIds?: readonly number[];
509
+ onEmbeddedSelectionChange?: (recordIds: number[]) => void;
510
+ }
511
+
512
+ interface LinkGridModalProps {
513
+ label: string;
514
+ table: SurfaceScope;
515
+ recordIds: readonly number[];
516
+ editable?: boolean;
517
+ single?: boolean;
518
+ onSave?: (recordIds: number[]) => void;
519
+ onClose: () => void;
520
+ }
521
+
522
+ function LinkGridModal({ label, table, recordIds, editable = false, single = false,
523
+ onSave, onClose }: LinkGridModalProps) {
524
+ const panelRef = useRef<HTMLDivElement>(null);
525
+ const [selectedIds, setSelectedIds] = useState<number[]>(() => [...recordIds]);
526
+ const changeSelectedIds = useCallback((ids: number[]) => {
527
+ const next = single ? ids.slice(-1) : ids;
528
+ setSelectedIds((current) =>
529
+ current.join(",") === next.join(",") ? current : next
530
+ );
531
+ }, [single]);
532
+ useOverlayLayer({
533
+ panelRef,
534
+ onDismiss: onClose,
535
+ dismissOnOutside: true,
536
+ initialFocus: "[data-overlay-autofocus]",
537
+ trapFocus: true,
538
+ });
539
+ return (
540
+ <BodyPortal>
541
+ <div className="cg-record-backdrop">
542
+ <section
543
+ className="cg-link-grid-modal"
544
+ ref={panelRef}
545
+ role="dialog"
546
+ aria-modal="true"
547
+ aria-label={label}
548
+ data-overlay-kind="linked-record-grid"
549
+ tabIndex={-1}
550
+ >
551
+ <header className="cg-link-grid-head">
552
+ <div>
553
+ <div className="cg-link-grid-title">{label}</div>
554
+ <div className="cg-link-grid-sub">
555
+ {(editable ? selectedIds.length : recordIds.length).toLocaleString()} linked{
556
+ single ? " (one allowed)" : ""
557
+ } {(editable ? selectedIds.length : recordIds.length) === 1 ? "record" : "records"}
558
+ </div>
559
+ </div>
560
+ <button
561
+ type="button"
562
+ className="cg-icon-btn"
563
+ aria-label="Close linked records"
564
+ data-overlay-autofocus
565
+ onClick={onClose}
566
+ >
567
+ Γ—
568
+ </button>
569
+ </header>
570
+ <div className="cg-link-grid-body">
571
+ <CustomerGrid
572
+ scope={table}
573
+ embedded
574
+ embeddedRecordIds={editable ? undefined : recordIds}
575
+ embeddedSelectable={editable}
576
+ embeddedSelectedIds={selectedIds}
577
+ onEmbeddedSelectionChange={changeSelectedIds}
578
+ />
579
+ </div>
580
+ {editable ? (
581
+ <footer className="cg-link-grid-foot">
582
+ <button type="button" className="cg-btn" onClick={onClose}>Cancel</button>
583
+ <button
584
+ type="button"
585
+ className="cg-btn cg-btn--primary"
586
+ onClick={() => { onSave?.(selectedIds); onClose(); }}
587
+ >
588
+ Save links
589
+ </button>
590
+ </footer>
591
+ ) : null}
592
+ </section>
593
+ </div>
594
+ </BodyPortal>
595
+ );
596
+ }
597
+
598
+ function CustomerGrid({
599
+ scope = "customer",
600
+ embedded = false,
601
+ embeddedRecordIds,
602
+ embeddedSelectable = false,
603
+ embeddedSelectedIds = [],
604
+ onEmbeddedSelectionChange,
605
+ }: CustomerGridProps = {}) {
606
  // Wave 16 C-TOPIC: which TABLE this tree is drawing, derived from the one scope prop.
607
  const topic = topicForScope(scope);
608
  const {
609
  fields: payloadFields,
610
+ rawRows: fetchedRows,
611
  payload,
612
  loading,
613
  overlayEdits,
614
  setOverlayEdits,
615
  patchOverlay,
616
+ } = useCustomerData(scope, {
617
+ bindSurface: !embedded,
618
+ includeWorkspace: !embedded,
619
+ writable: !embedded,
620
+ });
621
+ const embeddedIdsKey = embeddedRecordIds?.join(",") ?? "";
622
+ const rawRows = useMemo(() => {
623
+ if (!embeddedRecordIds) return fetchedRows;
624
+ const wanted = new Set(embeddedRecordIds);
625
+ return fetchedRows.filter((row) => wanted.has(row.pid));
626
+ // The scalar key keeps this stable when a caller reconstructs the same id list.
627
+ // eslint-disable-next-line react-hooks/exhaustive-deps
628
+ }, [fetchedRows, embeddedIdsKey]);
629
 
630
  const [fields, setFields] = useState<Field[]>([]);
631
  const [views, setViews] = useState<SavedView[]>([]);
 
670
  setJsonAt(null);
671
  requestAnimationFrame(() => gridRef.current?.focus());
672
  }, []);
673
+ /** A relation opens as the target database's real Grid view, projected to the linked ids.
674
+ * This is intentionally distinct from JsonViewer: filters, sorts, search, column visibility,
675
+ * and the standard cell renderers all remain available inside the large modal. */
676
+ const [linkAt, setLinkAt] = useState<{ pid: number; fieldKey: string } | null>(null);
677
+ const closeLink = useCallback(() => {
678
+ setLinkAt(null);
679
+ requestAnimationFrame(() => gridRef.current?.focus());
680
+ }, []);
681
  /** WAVE 21 item 11 (R10) β€” is "Select records from a list" open? Opened from the view
682
  * rail's "…" and closed by the dialog; the SELECTION it produces outlives it. */
683
  const [selectFromFile, setSelectFromFile] = useState(false);
 
799
  // fills only missing objects and keeps the standalone path useful.
800
  useEffect(() => {
801
  if (!payload || payloadFields.length === 0 || initializedKey.current === storageKey) return;
802
+ const local = embedded ? null : readLocal(storageKey);
803
  // Item 3c: the def half of the no-blip layer. A RECENT local stamp beats a
804
  // lagged host echo (rename survives, retype holds, a delete stays deleted);
805
  // a caught-up echo returns host objects byte-identical (see optimism.ts).
 
815
  Date.now(),
816
  payload.workspace != null
817
  );
818
+ const hostViews = embedded ? [] : payload.workspace?.views ?? [];
819
  const byId = new Map<string, SavedView>();
820
  byId.set(ALL_VIEW_ID, allRecordsView(initialFields, scope));
821
  // D-19 β€” the HOST'S LIST DECIDES WHICH VIEWS EXIST. A local copy the host no longer
 
892
  * the config the client is actually filtering with, and sending anything else would ask the
893
  * host to resolve a question nobody on screen is asking.
894
  */
895
+ if (!embedded && reemit.size > 0) {
896
  const stamped = { ...(local?.reemitted ?? {}) };
897
  for (const view of initialViews) {
898
  const key = reemit.get(view.id);
 
907
  } else {
908
  reemittedRef.current = { ...(local?.reemitted ?? {}) };
909
  }
910
+ }, [payload, payloadFields, storageKey, embedded, scope]);
911
 
912
  useEffect(() => {
913
+ if (!workspaceReady || embedded) return;
914
  writeLocal(storageKey, {
915
  fields,
916
  views,
 
927
  // drop a view this browser created seconds ago.
928
  viewWrites: pruneTombstones(viewWritesRef.current, Date.now()),
929
  });
930
+ }, [workspaceReady, storageKey, fields, views, activeViewId, embedded]);
931
 
932
  /**
933
  * ⭐ THE LIVE WORKSPACE (owner report, 2026-08-04) β€” what has APPEARED since we mounted.
 
952
  */
953
  const hostWorkspaceViews = payload?.workspace?.views;
954
  useEffect(() => {
955
+ if (!workspaceReady || embedded) return;
956
  const now = Date.now();
957
  const nextFields = adoptNewFields(
958
  fields, payloadFields, fieldStampsRef.current.deleted, now
 
962
  adoptNewViews(current, hostWorkspaceViews, viewTombstonesRef.current, now,
963
  (config) => normalizeConfig(config, nextFields))
964
  );
965
+ }, [workspaceReady, hostWorkspaceViews, payloadFields, fields, embedded]);
966
 
967
  /** Item 3c β€” stamp a def write / a delete. Pruned at every touch so the persisted blob
968
  * stays a recent window, never an archive. */
 
1023
  * this resolves, the appended row does not exist yet, so "bottom" would land on the last OLD
1024
  * row.
1025
  */
1026
+ const isUserTable = !embedded && scope.startsWith("ut_");
1027
+ const recordsMutable = payload?.recordsMutable !== false;
1028
+ const canMutateRecords = isUserTable && recordsMutable;
1029
  /**
1030
  * ⭐ 2026-08-07 β€” the databases a `link` column may point at, fetched when the column menu
1031
  * OPENS rather than on every render of the grid.
 
1050
  };
1051
  }, [columnMenu, linkTargets.length]);
1052
  const appendRow = useCallback(async (): Promise<undefined> => {
1053
+ if (!canMutateRecords) return undefined;
1054
  const made = await addTableRow(scope);
1055
  // `undefined` either way, never glide's "bottom": glide would move the selection to the
1056
  // last row it currently knows about, which is the row BEFORE the one just created. The
 
1065
  });
1066
  signal(ROWS_STALE_EVENT);
1067
  return undefined;
1068
+ }, [canMutateRecords, scope]);
1069
 
1070
  /* wave20 item 2 β€” measure key + window -> the columns that display it. Built ONCE per field
1071
  list and handed to every consumer of "which column is this rule about", so the tint
 
1075
 
1076
  // Airtable behavior: configuration changes to the active view autosave.
1077
  useEffect(() => {
1078
+ if (!workspaceReady || embedded) return;
1079
  const active = views.find((view) => view.id === activeViewId);
1080
  if (!active || sameConfig(active.config, config)) {
1081
  setSaveState("saved");
 
1097
  return () => {
1098
  if (saveTimer.current !== null) window.clearTimeout(saveTimer.current);
1099
  };
1100
+ }, [workspaceReady, activeViewId, config, views, embedded]);
1101
 
1102
  // Numeric dimensions force a glide relayout when either the component frame
1103
  // or Streamlit's main column changes width (notably sidebar collapse).
 
1128
  const mode = tableMode(payload?.counts);
1129
  const serverWindowed = mode === "server-windowed";
1130
  // Owner items 4+6 β€” the Cohort page's grid renders without the Views sidebar.
1131
+ const hideViews = embedded || payload?.workspace?.hideViews === true;
1132
  // Wave-6 item 10 β€” how this view displays. A WINDOWED table is always the grid: list/
1133
  // calendar/kanban compute over the whole matched set, and one page is not it (CG-3's rule,
1134
  // the same reason grouping is off there).
 
1195
  // permissions vs the viewer, fail-closed on restricted fields when the viewer is unknown.
1196
  const viewer = payload?.viewer;
1197
  const canEditField = useCallback(
1198
+ (f: Field): boolean => !embedded && recordsMutable && mayEditField(f, viewer),
1199
+ [embedded, recordsMutable, viewer]
1200
  );
1201
  const unresolvedCount = useMemo(
1202
  () => unresolvedConditions(config.filters, { cohortSets, today }),
 
1430
  useGridSelection(
1431
  displayRows,
1432
  displayPidToIndex,
1433
+ visibleCols.length,
1434
+ embeddedSelectable ? embeddedSelectedIds : []
1435
  );
1436
+ const embeddedSelectedKey = embeddedSelectedIds.join(",");
1437
+ const selectedPidKey = [...selectedPids].join(",");
1438
+ useEffect(() => {
1439
+ if (!embeddedSelectable) return;
1440
+ if (selectedPidKey !== embeddedSelectedKey)
1441
+ selectPids(embeddedSelectedIds as number[], "replace");
1442
+ }, [embeddedSelectable, embeddedSelectedKey, embeddedSelectedIds, selectedPidKey, selectPids]);
1443
+ useEffect(() => {
1444
+ if (!embeddedSelectable) return;
1445
+ onEmbeddedSelectionChange?.([...selectedPids]);
1446
+ }, [embeddedSelectable, onEmbeddedSelectionChange, selectedPids]);
1447
  /* ════════════════════════ owner item 16 / R4 / C-UNDO ════════════════════════
1448
  THE RECORDING LAYER. `undoStack.ts` owns the stack and every inverse; this owns the one
1449
  thing it cannot: reading the value a cell held BEFORE the write, which only exists at the
 
1660
  */
1661
  const deleteRecords = useCallback(
1662
  async (pids: number[]): Promise<boolean> => {
1663
+ if (!canMutateRecords || !pids.length) return false;
1664
  // The row as it stands NOW, straight off the rendered records β€” the same values the
1665
  // reader can see, so a restore puts back what they watched disappear.
1666
  const byPid = new Map<number, Row>();
 
1780
  );
1781
  return true;
1782
  },
1783
+ [canMutateRecords, displayRows, fields, scope, clearSelection, delArmed]
1784
  );
1785
 
1786
  const onGridDelete = useCallback(
 
1806
  */
1807
  const dragged = !!sel.current
1808
  && (sel.current.range.width > 1 || sel.current.range.height > 1);
1809
+ if (canMutateRecords && sel.rows.length > 0 && !dragged) {
1810
  const pids: number[] = [];
1811
  for (const rowIndex of sel.rows) {
1812
  const vr = displayRows[rowIndex];
 
1849
  // fields (`onCellEdited`'s own `canEditField`), it just does not arrive as one entry.
1850
  return false;
1851
  },
1852
+ [canMutateRecords, deleteRecords, displayRows, visibleCols, fieldByKey, canEditField,
1853
+ patchManyAndRecord]
1854
  );
1855
 
1856
  /**
 
2082
  // (`.cg-grid-void` is `pointer-events: none`), which is the worst version of this bug:
2083
  // the gate went green on an affordance nobody could see. Found by READING THE SCREENSHOT
2084
  // ([[ui-invisible-to-assertions]], [[finalize-visual-review-sop]]).
2085
+ if (canMutateRecords)
2086
  rowsPx += typeof rowHeight === "number" ? rowHeight : rowHeight(displayRows.length);
2087
 
2088
  /* Fit is tested against the client box the OTHER axis's scrollbar leaves behind β€” the same
 
2105
  below: rowsPx < clientH ? rowsPx : null,
2106
  right: colsPx < clientW ? colsPx : null,
2107
  };
2108
+ }, [visibleCols, displayRows, rowHeight, gridSize, canMutateRecords]);
2109
  /* ═══ end W18-B VOID (geometry) ═══ */
2110
 
2111
  // The record drawer resolves positions against what the MODE paints: the display slice for
 
2305
  });
2306
  return;
2307
  }
2308
+ // A linked-record cell is a doorway to the target database, not an opaque id list.
2309
+ // The large modal mounts the same Grid surface over exactly these pids, so its standard
2310
+ // search, Filters, Sort, Fields, and column menus keep working for every database kind.
2311
+ if (row.kind === "data" && field?.type === "link" && field.link?.table) {
2312
+ if (field.link.table.startsWith("ut_")) {
2313
+ event.preventDefault();
2314
+ setActiveCell(cell[0], cell[1]);
2315
+ setLinkAt({ pid: row.record.pid, fieldKey: field.key });
2316
+ return;
2317
+ }
2318
+ }
2319
  // ⭐ Wave-23 C7 β€” a JSON cell opens the big viewer. It is the ONLY door: the cell carries
2320
  // `allowOverlay:false`, because glide's overlay is a one-line box and one keystroke in the
2321
  // wrong place inside a 32 KB document turns a well-formed payload into an unparseable one,
 
2471
 
2472
  const openHeaderMenu = useCallback(
2473
  (column: number, bounds: Rectangle) => {
2474
+ if (embedded) return;
2475
  const definition = visibleCols[column];
2476
  if (!definition?.id) return;
2477
  setColumnMenu({
 
2486
  },
2487
  });
2488
  },
2489
+ [embedded, visibleCols]
2490
  );
2491
  const onHeaderClicked = useCallback(
2492
  (column: number, event: HeaderClickedEventArgs) => {
 
4110
  const menuVisIndex = menuField ? visibleKeys.indexOf(menuField.key) : -1;
4111
  const menuPinnedTo = menuVisIndex >= 0 && frozenN > 1 && menuVisIndex + 1 === frozenN;
4112
  // Item 10 β€” the toolbar's mode switcher (grid-only on windowed tables, see displayMode).
4113
+ const modeControl = serverWindowed || embedded ? undefined : (
4114
  <ModeSwitch
4115
  mode={displayMode}
4116
  onMode={setDisplayMode}
 
4152
  ?? (jsonField && jsonRow ? jsonRow[jsonField.key] : "")
4153
  ?? ""
4154
  );
4155
+ const linkField = linkAt ? fieldByKey.get(linkAt.fieldKey) : undefined;
4156
+ const linkRow = linkAt ? rawRows.find((row) => row.pid === linkAt.pid) : undefined;
4157
+ const linkValue = String(
4158
+ (linkAt && overlayEdits[linkAt.pid]?.[linkAt.fieldKey])
4159
+ ?? (linkField && linkRow ? linkRow[linkField.key] : "")
4160
+ ?? ""
4161
+ );
4162
+ const linkedRecordIds = [...new Set(
4163
+ linkValue.split(",").map((part) => Number(part.trim())).filter(
4164
+ (pid) => Number.isInteger(pid) && pid > 0
4165
+ )
4166
+ )];
4167
  const pickerField = picker ? fieldByKey.get(picker.fieldKey) : undefined;
4168
  // A `user` field's choices come from the HOST's real user list, a `select`'s from its own
4169
  // definition β€” so an assignee is always someone who can log in, and a status is always one of
 
4479
  height={gridSize.height}
4480
  customRenderers={[ratingCellRenderer, userCellRenderer, imageCellRenderer]}
4481
  headerIcons={HEADER_ICONS}
4482
+ rightElement={embedded ? undefined : (
4483
  // Wave-6 item 4 β€” the "+" of the header row: the create-field form,
4484
  // insert-at-end.
4485
  //
 
4516
  >
4517
  +
4518
  </button>
4519
+ )}
4520
  rightElementProps={{ sticky: false, fill: false }}
4521
  // Owner item 4 (R8) β€” the trailing "+" row, USER DATABASES ONLY. Passing
4522
  // `onRowAppended` is what makes glide paint the ghost row at all, so the
4523
  // affordance exists exactly where a POST can succeed. See `appendRow`.
4524
+ onRowAppended={canMutateRecords ? appendRow : undefined}
4525
  trailingRowOptions={
4526
+ canMutateRecords
4527
  ? {
4528
  // The hint sits in the identity column (targetColumn 0), which is the
4529
  // one a new record is named in β€” the same cell the cursor lands on.
 
4721
 
4722
  Rendered only in `grid` mode: list / calendar / kanban / map have their own
4723
  open affordances and glide is not mounted to report bounds for them. */}
4724
+ {!embedded && displayMode === "grid" && expandAt && (
4725
  <button
4726
  type="button"
4727
  className="cg-row-expand"
 
4781
  broke this gate's own coordinate math before it broke a user). It states the
4782
  count and offers "Add to cohort" over exactly the checked pids β€” the same guarded
4783
  add_to_list event the view menu uses, so the host treats both alike. */}
4784
+ {!embedded && selectedPids.size > 0 && !serverWindowed &&
4785
  (displayMode === "grid" || displayMode === "map") && (
4786
  <div className="cg-selbar">
4787
  <span className="cg-selbar-count">
 
4793
  ⚠ ONLY on a `ut_` database: the customer and product grids are projections of
4794
  Odoo, where a "delete" would be a write to somebody else's system of record β€”
4795
  the server refuses it, and a button that must be refused is worse than none. */}
4796
+ {canMutateRecords && (
4797
  <button
4798
  type="button"
4799
  /* ARMED is a VISIBLE state, not just a changed handler: the button that is
 
4910
  // one-line picker, so it is left out until somebody asks for it.
4911
  linkTargets={linkTargets.filter((t) => t.key !== scope)}
4912
  locked={menuField.key === lockedKey}
4913
+ schemaLocked={menuField.automation?.preset === true}
4914
  viewer={viewer}
4915
  sortedDir={menuSortedDir}
4916
  isFiltered={menuIsFiltered}
 
5269
  onClose={closeJson}
5270
  />
5271
  )}
5272
+ {linkField?.type === "link" && linkField.link?.table.startsWith("ut_") && linkAt && (
5273
+ <LinkGridModal
5274
+ label={linkField.label}
5275
+ table={linkField.link.table as SurfaceScope}
5276
+ recordIds={linkedRecordIds}
5277
+ editable={canEditField(linkField) && !isDerivedLink(linkField)}
5278
+ single={linkField.link.single === true}
5279
+ onSave={(ids) => patchAndRecord(
5280
+ linkAt.pid,
5281
+ { [linkField.key]: ids.join(",") },
5282
+ "linked records"
5283
+ )}
5284
+ onClose={closeLink}
5285
+ />
5286
+ )}
5287
  {pickerField && picker && (
5288
  <AnchoredOverlay
5289
  anchor={picker.anchor}
 
5481
  </AnchoredOverlay>
5482
  )}
5483
 
5484
+ {!embedded && detailRecord && detailPid !== null && (
5485
  <RecordDetail
5486
  fields={fields}
5487
  record={detailRecord}
web/src/customer-grid/RecordDetail.css CHANGED
@@ -32,6 +32,72 @@
32
  outline: none;
33
  }
34
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
35
  @keyframes cg-record-open {
36
  from {
37
  opacity: 0;
 
32
  outline: none;
33
  }
34
 
35
+ /* Universal linked-record viewer: the target is the real Grid surface, not a JSON-shaped
36
+ approximation. It is deliberately wider than the record detail so filtering and multi-column
37
+ sorting have the same working room they do on the database page. */
38
+ .cg-link-grid-modal {
39
+ width: min(1320px, calc(100vw - 48px));
40
+ height: min(840px, calc(100vh - 48px));
41
+ min-height: 460px;
42
+ display: flex;
43
+ flex-direction: column;
44
+ overflow: hidden;
45
+ border: 1px solid color-mix(in srgb, var(--lp-line) 82%, var(--lp-muted));
46
+ border-radius: 12px;
47
+ background: var(--lp-surface);
48
+ box-shadow:
49
+ 0 30px 80px rgba(20, 30, 43, 0.24),
50
+ 0 4px 18px rgba(20, 30, 43, 0.12);
51
+ color: var(--lp-ink);
52
+ font-family: Inter, ui-sans-serif, system-ui, -apple-system, "Segoe UI", sans-serif;
53
+ font-size: var(--lp-fs-xs);
54
+ animation: cg-record-open 150ms cubic-bezier(0.2, 0, 0, 1);
55
+ }
56
+
57
+ .cg-link-grid-modal:focus { outline: none; }
58
+
59
+ .cg-link-grid-head {
60
+ flex: 0 0 54px;
61
+ min-height: 54px;
62
+ display: flex;
63
+ align-items: center;
64
+ justify-content: space-between;
65
+ gap: 16px;
66
+ padding: 0 14px 0 18px;
67
+ border-bottom: 1px solid var(--lp-line);
68
+ background: var(--lp-surface);
69
+ }
70
+
71
+ .cg-link-grid-title {
72
+ font-size: var(--lp-fs-sm);
73
+ font-weight: 650;
74
+ line-height: 1.2;
75
+ }
76
+
77
+ .cg-link-grid-sub {
78
+ margin-top: 2px;
79
+ color: var(--lp-muted);
80
+ font-size: var(--lp-fs-3xs);
81
+ }
82
+
83
+ .cg-link-grid-body {
84
+ flex: 1 1 auto;
85
+ min-width: 0;
86
+ min-height: 0;
87
+ background: var(--lp-surface);
88
+ }
89
+
90
+ .cg-link-grid-foot {
91
+ flex: 0 0 52px;
92
+ display: flex;
93
+ align-items: center;
94
+ justify-content: flex-end;
95
+ gap: 8px;
96
+ padding: 8px 14px;
97
+ border-top: 1px solid var(--lp-line);
98
+ background: var(--lp-surface);
99
+ }
100
+
101
  @keyframes cg-record-open {
102
  from {
103
  opacity: 0;
web/src/customer-grid/apiBridge.ts CHANGED
@@ -44,8 +44,9 @@ import {
44
  API_V1,
45
  CREDENTIALS,
46
  DATA_ERROR_EVENT,
47
- DERIVED_CELLS_EVENT,
48
- TOAST_EVENT,
 
49
  WORKSPACE_STALE_EVENT,
50
  UNAUTHORIZED_EVENT,
51
  signal,
@@ -257,12 +258,20 @@ export async function patchTopicRow(
257
  : [];
258
  const blanks: Partial<Row> = {};
259
  for (const k of cleared) blanks[k] = "";
260
- if (accepted && typeof accepted === "object") {
261
- const row = cached?.payload.rows.find((r) => r.pid === pid);
262
- if (row) Object.assign(row, accepted, blanks);
263
- onAccepted?.({ ...(accepted as Partial<Row>), ...blanks }, cleared);
264
- }
265
- }
 
 
 
 
 
 
 
 
266
  return res.ok;
267
  } catch {
268
  return false;
 
44
  API_V1,
45
  CREDENTIALS,
46
  DATA_ERROR_EVENT,
47
+ DERIVED_CELLS_EVENT,
48
+ ROWS_STALE_EVENT,
49
+ TOAST_EVENT,
50
  WORKSPACE_STALE_EVENT,
51
  UNAUTHORIZED_EVENT,
52
  signal,
 
258
  : [];
259
  const blanks: Partial<Row> = {};
260
  for (const k of cleared) blanks[k] = "";
261
+ if (accepted && typeof accepted === "object") {
262
+ const row = cached?.payload.rows.find((r) => r.pid === pid);
263
+ if (row) Object.assign(row, accepted, blanks);
264
+ onAccepted?.({ ...(accepted as Partial<Row>), ...blanks }, cleared);
265
+ }
266
+ // A user-table edit may change a reciprocal Link or a Rollup in another database. Their
267
+ // row payloads share no cache key with this table, so invalidate the user-table family and
268
+ // let the active grid re-read the server's materialised relationship cells.
269
+ if (topic.rowsPath.startsWith("tables/")) {
270
+ for (const key of [...rowsCache.keys()])
271
+ if (key.startsWith("tables/")) rowsCache.delete(key);
272
+ signal(ROWS_STALE_EVENT);
273
+ }
274
+ }
275
  return res.ok;
276
  } catch {
277
  return false;
web/src/customer-grid/types.ts CHANGED
@@ -1309,8 +1309,10 @@ export interface Field {
1309
  urlField?: string;
1310
  /** kind-specific knobs. Scalars only β€” the host clamps keys and value lengths. */
1311
  settings?: Record<string, string | number | boolean>;
1312
- /** the automation definition that owns this column (engine-stamped; the machine marker). */
1313
- flowId?: string;
 
 
1314
  /** true on the `stage_<autoId>` column an automation board walks its records through. */
1315
  stageField?: boolean;
1316
  };
@@ -1432,8 +1434,12 @@ export interface Field {
1432
  table: string;
1433
  /** the column IN THE LINKED TABLE that holds this row's key β€” presence = derived */
1434
  on?: string;
1435
- /** the column on THIS table supplying the join value; defaults to the profile/pinned one */
1436
- from?: string;
 
 
 
 
1437
  /** Airtable's `prefersSingleRecordLink` */
1438
  single?: boolean;
1439
  };
@@ -1453,11 +1459,27 @@ export interface Field {
1453
  /** the column in the linked table to aggregate; absent only for `countall` */
1454
  field?: string;
1455
  fn: RollupFn;
1456
- limit?: number;
1457
- sortBy?: string;
1458
- sortDir?: "asc" | "desc";
1459
- };
1460
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1461
 
1462
  /**
1463
  * Airtable's 17 rollup functions, minus the two that are meaningless over a scalar cell.
@@ -1469,7 +1491,7 @@ export interface Field {
1469
  * the same discipline every other cross-language vocabulary here uses.
1470
  */
1471
  export const ROLLUP_FNS = [
1472
- "sum", "average", "min", "max", "count", "counta", "countall",
1473
  "and", "or", "xor", "concatenate", "arrayjoin", "arraycompact", "arrayunique",
1474
  ] as const;
1475
  export type RollupFn = (typeof ROLLUP_FNS)[number];
@@ -1480,7 +1502,8 @@ export const ROLLUP_NUMERIC_FNS: ReadonlySet<string> =
1480
 
1481
  /** Human wording for each function, so a column menu never shows a bare identifier. */
1482
  export const ROLLUP_FN_LABELS: Record<RollupFn, string> = {
1483
- sum: "Sum", average: "Average", min: "Minimum", max: "Maximum",
 
1484
  count: "Count of numbers", counta: "Count of filled cells", countall: "Count of records",
1485
  and: "All are true", or: "Any is true", xor: "Exactly one is true",
1486
  concatenate: "Join (no separator)", arrayjoin: "Join with commas",
@@ -1494,9 +1517,9 @@ export const ROLLUP_FN_LABELS: Record<RollupFn, string> = {
1494
  * is rewritten on every refresh pass, so accepting a typed value would not merely be wrong β€” it
1495
  * would appear to work and then silently revert, which is worse than refusing it.
1496
  */
1497
- export function isDerivedLink(field: Field): boolean {
1498
- return field.type === "link" && !!field.link?.on;
1499
- }
1500
 
1501
  /**
1502
  * Is this the table's PROFILE column (C3/R7)?
@@ -2832,9 +2855,11 @@ export interface ScopeCounts {
2832
  windowed: boolean;
2833
  }
2834
 
2835
- export interface CustomersPayload {
2836
- fields: Field[];
2837
- rows: Row[];
 
 
2838
  workspace?: GridWorkspace;
2839
  /** Present only for server-windowed tables. See ScopeCounts. */
2840
  counts?: ScopeCounts;
 
1309
  urlField?: string;
1310
  /** kind-specific knobs. Scalars only β€” the host clamps keys and value lengths. */
1311
  settings?: Record<string, string | number | boolean>;
1312
+ /** the automation definition that owns this column (engine-stamped; the machine marker). */
1313
+ flowId?: string;
1314
+ /** true when the field is part of a product-owned preset schema and cannot be redefined */
1315
+ preset?: boolean;
1316
  /** true on the `stage_<autoId>` column an automation board walks its records through. */
1317
  stageField?: boolean;
1318
  };
 
1434
  table: string;
1435
  /** the column IN THE LINKED TABLE that holds this row's key β€” presence = derived */
1436
  on?: string;
1437
+ /** the column on THIS table supplying the join value; defaults to the profile/pinned one */
1438
+ from?: string;
1439
+ /** source link field key when this is Airtable's computed reciprocal side */
1440
+ inverse?: string;
1441
+ /** reciprocal field key on the target table when this is the user-picked source side */
1442
+ reciprocal?: string;
1443
  /** Airtable's `prefersSingleRecordLink` */
1444
  single?: boolean;
1445
  };
 
1459
  /** the column in the linked table to aggregate; absent only for `countall` */
1460
  field?: string;
1461
  fn: RollupFn;
1462
+ limit?: number;
1463
+ sortBy?: string;
1464
+ sortDir?: "asc" | "desc";
1465
+ /** collapse duplicate logical records after ranking, before applying the window */
1466
+ distinctBy?: string;
1467
+ /** optional Airtable-style conditions applied to linked records before aggregation */
1468
+ conditions?: RollupCondition[];
1469
+ conditionConj?: "and" | "or";
1470
+ };
1471
+ }
1472
+
1473
+ export const ROLLUP_CONDITION_OPS = [
1474
+ "eq", "neq", "contains", "not_contains", "is_empty", "is_not_empty",
1475
+ "gt", "gte", "lt", "lte",
1476
+ ] as const;
1477
+ export type RollupConditionOp = (typeof ROLLUP_CONDITION_OPS)[number];
1478
+ export interface RollupCondition {
1479
+ field: string;
1480
+ op: RollupConditionOp;
1481
+ value?: string;
1482
+ }
1483
 
1484
  /**
1485
  * Airtable's 17 rollup functions, minus the two that are meaningless over a scalar cell.
 
1491
  * the same discipline every other cross-language vocabulary here uses.
1492
  */
1493
  export const ROLLUP_FNS = [
1494
+ "sum", "average", "min", "max", "latest", "count", "counta", "countall",
1495
  "and", "or", "xor", "concatenate", "arrayjoin", "arraycompact", "arrayunique",
1496
  ] as const;
1497
  export type RollupFn = (typeof ROLLUP_FNS)[number];
 
1502
 
1503
  /** Human wording for each function, so a column menu never shows a bare identifier. */
1504
  export const ROLLUP_FN_LABELS: Record<RollupFn, string> = {
1505
+ sum: "Sum", average: "Average", min: "Minimum", max: "Maximum",
1506
+ latest: "Latest value",
1507
  count: "Count of numbers", counta: "Count of filled cells", countall: "Count of records",
1508
  and: "All are true", or: "Any is true", xor: "Exactly one is true",
1509
  concatenate: "Join (no separator)", arrayjoin: "Join with commas",
 
1517
  * is rewritten on every refresh pass, so accepting a typed value would not merely be wrong β€” it
1518
  * would appear to work and then silently revert, which is worse than refusing it.
1519
  */
1520
+ export function isDerivedLink(field: Field): boolean {
1521
+ return field.type === "link" && !!(field.link?.on || field.link?.inverse);
1522
+ }
1523
 
1524
  /**
1525
  * Is this the table's PROFILE column (C3/R7)?
 
2855
  windowed: boolean;
2856
  }
2857
 
2858
+ export interface CustomersPayload {
2859
+ fields: Field[];
2860
+ rows: Row[];
2861
+ /** false on automation-owned child databases whose records are populated only by the engine */
2862
+ recordsMutable?: boolean;
2863
  workspace?: GridWorkspace;
2864
  /** Present only for server-windowed tables. See ScopeCounts. */
2865
  counts?: ScopeCounts;
web/src/customer-grid/useCustomerData.ts CHANGED
@@ -158,12 +158,29 @@ export interface CustomerData {
158
  patchOverlay: (pid: number, updates: Partial<Row>) => void;
159
  }
160
 
161
- export function useCustomerData(scope: SurfaceScope = "customer"): CustomerData {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
162
  // β›” SET BEFORE THE FIRST FETCH, not in an effect. Every event the grid emits carries this
163
  // scope, and an effect would run AFTER the first render β€” so a write triggered by that render
164
  // would be posted as `customer` from the Cohort surface. Module scope + a synchronous set is
165
  // what keeps the read and the write talking about the same surface.
166
- setSurfaceScope(scope);
167
  // Wave 16 C-TOPIC: the topic is DERIVED from the scope (cohort = the customer topic), so
168
  // the rows fetch, the patch route and the workspace scope can never disagree about which
169
  // table this is.
@@ -257,12 +274,14 @@ export function useCustomerData(scope: SurfaceScope = "customer"): CustomerData
257
  loudScope.current = scope;
258
 
259
  async function load() {
260
- const [payload, workspace] = await Promise.all([
261
- fetchTopicRows(topic),
262
  // A workspace that cannot be read at first paint is not "a table with no saved
263
  // views" β€” it is a table whose schema we do not have, and the grid used to fill
264
  // that hole from a localStorage bucket shared with every other surface.
265
- fetchWorkspace(scope, { announceFailure: firstPaint }),
 
 
266
  ]);
267
  if (cancelled || !payload) return;
268
  setData(workspace ? withWorkspace(payload, workspace) : payload);
@@ -277,8 +296,9 @@ export function useCustomerData(scope: SurfaceScope = "customer"): CustomerData
277
  // nothing and cost the expensive call. The measure channel rides the workspace
278
  // (see `withWorkspace`), so a NEW measure column's values arrive on this cheap
279
  // re-read too β€” no pool refetch.
280
- async function reread() {
281
- const ws = await fetchWorkspace(scope);
 
282
  if (cancelled || !ws) return; // absent stays absent β€” never blank a live panel
283
  setData((prev) => (prev ? withWorkspace(prev, ws) : prev));
284
  }
@@ -332,13 +352,14 @@ export function useCustomerData(scope: SurfaceScope = "customer"): CustomerData
332
  // `topic` is a stable module constant per scope (topicForScope β€” ut topics are memoized
333
  // per key for exactly this dependency), so this cannot loop. `rowsBump` re-runs it on the
334
  // rows-stale signal only.
335
- }, [scope, topic, rowsBump]);
336
 
337
  // Optimistic overlay write: apply the edit immediately (so the grid cell and
338
  // the detail panel reflect it at once), snapshot the pre-edit entry, then
339
  // PATCH. On a non-ok response, restore the snapshot so the UI never shows a
340
  // value the server rejected.
341
- const patchOverlay = useCallback((pid: number, updates: Partial<Row>) => {
 
342
  let prevEntry: Partial<Row> | undefined;
343
  setOverlayEdits((prev) => {
344
  prevEntry = prev[pid];
@@ -403,7 +424,7 @@ export function useCustomerData(scope: SurfaceScope = "customer"): CustomerData
403
  return next;
404
  });
405
  });
406
- }, [topic]);
407
 
408
  return {
409
  fields: data?.fields ?? [],
 
158
  patchOverlay: (pid: number, updates: Partial<Row>) => void;
159
  }
160
 
161
+ interface CustomerDataOptions {
162
+ /** A linked-record grid is nested inside its parent grid. It must not replace the
163
+ * module-level write scope that the parent still owns. */
164
+ bindSurface?: boolean;
165
+ /** Linked-record grids deliberately open on a clean Grid view, not on the target
166
+ * database's last saved view. The rows payload already carries its field contract. */
167
+ includeWorkspace?: boolean;
168
+ /** Defence in depth for read-only nested grids. */
169
+ writable?: boolean;
170
+ }
171
+
172
+ export function useCustomerData(
173
+ scope: SurfaceScope = "customer",
174
+ options: CustomerDataOptions = {},
175
+ ): CustomerData {
176
+ const bindSurface = options.bindSurface !== false;
177
+ const includeWorkspace = options.includeWorkspace !== false;
178
+ const writable = options.writable !== false;
179
  // β›” SET BEFORE THE FIRST FETCH, not in an effect. Every event the grid emits carries this
180
  // scope, and an effect would run AFTER the first render β€” so a write triggered by that render
181
  // would be posted as `customer` from the Cohort surface. Module scope + a synchronous set is
182
  // what keeps the read and the write talking about the same surface.
183
+ if (bindSurface) setSurfaceScope(scope);
184
  // Wave 16 C-TOPIC: the topic is DERIVED from the scope (cohort = the customer topic), so
185
  // the rows fetch, the patch route and the workspace scope can never disagree about which
186
  // table this is.
 
274
  loudScope.current = scope;
275
 
276
  async function load() {
277
+ const [payload, workspace] = await Promise.all([
278
+ fetchTopicRows(topic),
279
  // A workspace that cannot be read at first paint is not "a table with no saved
280
  // views" β€” it is a table whose schema we do not have, and the grid used to fill
281
  // that hole from a localStorage bucket shared with every other surface.
282
+ includeWorkspace
283
+ ? fetchWorkspace(scope, { announceFailure: firstPaint })
284
+ : Promise.resolve(null),
285
  ]);
286
  if (cancelled || !payload) return;
287
  setData(workspace ? withWorkspace(payload, workspace) : payload);
 
296
  // nothing and cost the expensive call. The measure channel rides the workspace
297
  // (see `withWorkspace`), so a NEW measure column's values arrive on this cheap
298
  // re-read too β€” no pool refetch.
299
+ async function reread() {
300
+ if (!includeWorkspace) return;
301
+ const ws = await fetchWorkspace(scope);
302
  if (cancelled || !ws) return; // absent stays absent β€” never blank a live panel
303
  setData((prev) => (prev ? withWorkspace(prev, ws) : prev));
304
  }
 
352
  // `topic` is a stable module constant per scope (topicForScope β€” ut topics are memoized
353
  // per key for exactly this dependency), so this cannot loop. `rowsBump` re-runs it on the
354
  // rows-stale signal only.
355
+ }, [scope, topic, rowsBump, includeWorkspace]);
356
 
357
  // Optimistic overlay write: apply the edit immediately (so the grid cell and
358
  // the detail panel reflect it at once), snapshot the pre-edit entry, then
359
  // PATCH. On a non-ok response, restore the snapshot so the UI never shows a
360
  // value the server rejected.
361
+ const patchOverlay = useCallback((pid: number, updates: Partial<Row>) => {
362
+ if (!writable) return;
363
  let prevEntry: Partial<Row> | undefined;
364
  setOverlayEdits((prev) => {
365
  prevEntry = prev[pid];
 
424
  return next;
425
  });
426
  });
427
+ }, [topic, writable]);
428
 
429
  return {
430
  fields: data?.fields ?? [],
web/src/customer-grid/useGridSelection.ts CHANGED
@@ -42,9 +42,12 @@ export interface GridSelectionApi {
42
  export function useGridSelection(
43
  visibleRows: VisibleRow[],
44
  pidToIndex: Map<number, number>,
45
- colCount: number
 
46
  ): GridSelectionApi {
47
- const [selectedPids, setSelectedPids] = useState<Set<number>>(new Set());
 
 
48
  const [current, setCurrent] = useState<GridSelection["current"] | undefined>(
49
  undefined
50
  );
 
42
  export function useGridSelection(
43
  visibleRows: VisibleRow[],
44
  pidToIndex: Map<number, number>,
45
+ colCount: number,
46
+ initialSelectedPids: readonly number[] = []
47
  ): GridSelectionApi {
48
+ const [selectedPids, setSelectedPids] = useState<Set<number>>(
49
+ () => new Set(initialSelectedPids)
50
+ );
51
  const [current, setCurrent] = useState<GridSelection["current"] | undefined>(
52
  undefined
53
  );
web/src/index.css CHANGED
@@ -1435,13 +1435,49 @@ body {
1435
  cursor: pointer;
1436
  }
1437
  .cg-option-add:hover { text-decoration: underline; }
1438
- .cg-field-hint {
1439
  display: block;
1440
  font-size: var(--lp-fs-3xs);
1441
  line-height: 1.45;
1442
  color: #8a8a96;
1443
  padding: 2px 0 4px;
1444
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1445
  /* why a control is unavailable, in the place the control would have been. An empty popover
1446
  reads as broken; a sentence reads as a decision (CG-3). */
1447
  .cg-pop-note {
 
1435
  cursor: pointer;
1436
  }
1437
  .cg-option-add:hover { text-decoration: underline; }
1438
+ .cg-field-hint {
1439
  display: block;
1440
  font-size: var(--lp-fs-3xs);
1441
  line-height: 1.45;
1442
  color: #8a8a96;
1443
  padding: 2px 0 4px;
1444
+ }
1445
+
1446
+ .cg-rollup-conditions {
1447
+ display: grid;
1448
+ gap: 8px;
1449
+ }
1450
+
1451
+ .cg-rollup-conditions__head {
1452
+ display: flex;
1453
+ align-items: center;
1454
+ justify-content: space-between;
1455
+ gap: 10px;
1456
+ color: var(--cg-text);
1457
+ font-size: var(--lp-fs-2xs);
1458
+ font-weight: 650;
1459
+ }
1460
+
1461
+ .cg-rollup-conditions__head .cg-btn {
1462
+ min-height: 28px;
1463
+ padding: 4px 9px;
1464
+ }
1465
+
1466
+ .cg-rollup-condition {
1467
+ display: grid;
1468
+ grid-template-columns: minmax(0, 1fr) minmax(0, 1fr) 30px;
1469
+ align-items: center;
1470
+ gap: 6px;
1471
+ }
1472
+
1473
+ .cg-rollup-condition > input {
1474
+ grid-column: 1 / span 2;
1475
+ }
1476
+
1477
+ .cg-rollup-condition .cg-icon-btn {
1478
+ width: 30px;
1479
+ height: 30px;
1480
+ }
1481
  /* why a control is unavailable, in the place the control would have been. An empty popover
1482
  reads as broken; a sentence reads as a decision (CG-3). */
1483
  .cg-pop-note {