diff --git "a/api/providers.py" "b/api/providers.py" --- "a/api/providers.py" +++ "b/api/providers.py" @@ -1,897 +1,897 @@ -"""THE PROVIDER LAYER — many vendors behind one capability, chosen per FIELD and per cost. - -⭐⭐ OWNER RULING 2026-08-08: *"Let's do a combination scraper. Do Bright Data first, and if -anything fails, we use APIfy or vice versa. Make this dynamic somehow. remember to cost optimize -when we scale to thousands+ customer … make sure our code is customizable and modular."* - -⛔ THE DESIGN DECISION THAT MATTERS, AND IT IS NOT "TRY A, THEN TRY B". -A provider-level failover ("if Bright Data errors, re-run the whole thing on Apify") is the obvious -shape and it is the expensive one. MEASURED: Bright Data answers profile, likes and comments -correctly and CANNOT answer view counts on any route; Apify answers view counts correctly. A -provider-level fallback would notice the missing views and re-buy the profile, the likes and the -comments from Apify as well — **paying twice for the 90% that already worked**. At one profile that -is noise; at thousands of customers × twelve posts each it is the whole bill. - -So routing is per CAPABILITY, and a capability is the smallest independently-billable thing: - - ig_profile -> brightdata (36 fields, works) - ig_post_metrics -> brightdata (likes + comments, works) - ig_post_views -> apify (Bright Data is INCAPABLE — see `capable=False` below) - ig_comments -> brightdata - -Each capability has an ORDERED list of providers. The first one that is configured AND capable AND -succeeds wins; the rest are never called, so the common path costs exactly one vendor record. - -⚠ "FAILURE" INCLUDES ANSWERING WITH THE FIELD BLANK. A vendor that returns HTTP 200 and a null in -the one column you asked for has failed for our purposes, and a chain that only catches exceptions -would stop at it forever. `run()` takes a `satisfied` predicate and falls through on an unsatisfying -answer exactly as it would on a 500. - -⚠ ORDER IS DATA, NOT CODE. `AIOS_PROVIDER_ORDER` overrides any chain at deploy time -(`ig_post_views=apify,brightdata;ig_profile=apify`), which is what "or vice versa" means and what -lets a tenant be moved off a vendor without a release. -""" -from __future__ import annotations - -import json -import os -import time -from dataclasses import dataclass, field -from typing import Callable - -# ⚠ TOP-LEVEL AND SAFE, unlike the `anthropic` import in `anthropic_sdk()`: `requests` is pinned in -# BOTH manifests and is already a transitive dependency of `huggingface_hub`, so it cannot be the -# line that stops a container booting. That asymmetry is the whole reason the two are imported -# differently — see `anthropic_sdk()`. -import requests - -#: ⚠ APPROXIMATE, AND DELIBERATELY SO. These are list prices per RECORD in USD, used only to RANK -#: providers and to estimate a run's spend for the operator. They are not billing truth — the -#: vendor's own dashboard is. Wrong by 2x still ranks correctly; wrong by 100x does not, which is -#: why they are named and dated rather than guessed silently. -#: Bright Data Instagram datasets ~ $0.0015/record (2026-08 list). Apify instagram-scraper -#: ~ $0.0027/result (2026-08 list). Both re-checked when a provider is added. -#: ⚠ WAVE 30 · T16 — APIFY CORRECTED 0.0023 -> 0.0027, and the direction matters: the old number -#: made the fallback look CHEAPER than it is, and `estimate()` is what a person is shown before -#: they authorise a run. An under-stated price is the one rounding error a cost guard cannot catch. -#: ⚠ Both are LIST rates used to rank and to estimate. They are not invoices, and nothing here -#: reads a live price — a modelled figure that says so is honest; one that pretends is not. -_COST_BRIGHTDATA = float(os.environ.get("AIOS_COST_BRIGHTDATA") or 0.0015) -_COST_APIFY = float(os.environ.get("AIOS_COST_APIFY") or 0.0027) - - -@dataclass(frozen=True) -class Capability: - """What ONE provider can do for ONE capability, and what it costs to ask.""" - #: ⛔ `False` means MEASURED INCAPABLE, not "untried". A provider declared incapable is never - #: called for this capability at all — it cannot be reached by a fallback, cannot be put first - #: by an env override, and cannot silently start being billed because somebody reordered a - #: list. Bright Data's `ig_post_views` is False on the strength of §4e-§4h: every route - #: (Posts x /p/, Posts x /reel/, Reels x /p/, Reels x /reel/, discover-by-profile) on four - #: accounts from 13K to 268M followers returned an account-grain constant or a null. - capable: bool = True - cost_per_record: float = 0.0 - #: Free-text, shown to an operator deciding where their money went. - note: str = "" - - -@dataclass -class Provider: - key: str - label: str - #: The env var holding this provider's credential. NEVER the credential itself — this module - #: is imported by surfaces that serialise their config. - key_env: str = "" - caps: dict = field(default_factory=dict) - - def configured(self) -> bool: - return bool((os.environ.get(self.key_env) or "").strip()) if self.key_env else True - - def cap(self, capability: str) -> Capability | None: - return self.caps.get(capability) - - def can(self, capability: str) -> bool: - c = self.cap(capability) - return bool(c and c.capable and self.configured()) - - -#: ⚠ ADDING A PROVIDER IS A REGISTRY ENTRY PLUS A RUNNER — no change to any caller. That is the -#: "modular/customizable" half of the ruling, and the reason the chains below name STRINGS. -PROVIDERS: dict[str, Provider] = { - "brightdata": Provider( - key="brightdata", label="Bright Data", key_env="AIOS_BRIGHTDATA_KEY", - caps={ - "ig_profile": Capability(True, _COST_BRIGHTDATA, "36 fields incl. bio/email/category"), - "ig_post_metrics": Capability(True, _COST_BRIGHTDATA, "likes + comments + captions"), - "ig_comments": Capability(True, _COST_BRIGHTDATA, "separate paid dataset, opt-in"), - # ⛔ THE ONE THAT COSTS US NOTHING TO GET RIGHT AND EVERYTHING TO GET WRONG. - "ig_post_views": Capability( - False, _COST_BRIGHTDATA, - "MEASURED INCAPABLE: returns one account-grain number for every reel of a " - "creator (identical for two shortcodes in one call, ticking upward between " - "calls) and never populates video_play_count. Their own documented example " - "account no longer reproduces it."), - # ⭐⭐ WAVE 29 (item 7 / D-9 / R1) — TIKTOK. Three datasets, all reachable with our - # existing key, all schema-probed for $0.00 (40/43/17 fields with the vendor's own - # types): `gd_l1villgoiiidt09ci` · `gd_lu702nij2f790tmv9h` · `gd_lkf2st302ap89utw5k`. - "tt_profile": Capability(True, _COST_BRIGHTDATA, - "40 fields incl. bio/engagement rates/region"), - # ⛔ THERE IS NO `tt_post_views` CAPABILITY, AND ITS ABSENCE IS THE CLAIM. - # `TikTok - Posts` declares `play_count: number` and the vendor's own sample says - # "no empty values or zeros" — so on TikTok the view count arrives INSIDE the post - # record and a separate view rung would be a second bill for a number we already have. - # ⚠ DECLARED, NOT MEASURED, and that distinction is the whole scar tissue of §4e: - # Bright Data's Instagram Reels also DECLARES `views: number` and delivers an - # account-grain wrong one. One live pull settles it (W29-T06, spend-gated). If it - # fails, the honest repair is a `tt_post_views` capability routed elsewhere — never a - # quiet fallback bolted onto this one. - "tt_post_metrics": Capability( - True, _COST_BRIGHTDATA, - "likes + comments + shares + saves, and play_count inline (DECLARED " - "no-empties-or-zeros, UNPROVEN until one live pull)"), - "tt_comments": Capability(True, _COST_BRIGHTDATA, - "separate paid dataset, opt-in: 17 fields"), - }), - "apify": Provider( - key="apify", label="Apify", key_env="AIOS_APIFY_KEY", - caps={ - # ⭐ MEASURED 2026-08-08 against the public Reels grid: `videoPlayCount` 137,684 and - # 299,493 vs a browser-read ground truth of 134K-137K and 299K. Exact. - "ig_post_views": Capability(True, _COST_APIFY, - "videoPlayCount, matches Instagram's displayed views"), - "ig_post_metrics": Capability(True, _COST_APIFY, "likes + comments (fallback)"), - "ig_profile": Capability(True, _COST_APIFY, "profile fields (fallback)"), - }), -} - -#: The DEFAULT chain per capability, cheapest-capable-first. Overridable — see `chain()`. -DEFAULT_CHAINS: dict[str, tuple] = { - "ig_profile": ("brightdata", "apify"), - "ig_post_metrics": ("brightdata", "apify"), - # Only one entry, and that is the point: Bright Data is declared incapable, so listing it here - # would be a lie that costs a wasted call on every single post. - "ig_post_views": ("apify",), - "ig_comments": ("brightdata",), - # ⭐ WAVE 29 — TikTok, and every chain is deliberately SINGLE-PROVIDER. - # ⛔ A MULTI-PROVIDER CHAIN IS A PROMISE SOMETHING WALKS IT. `ig_post_metrics` has declared a - # two-provider fallback since wave 28 and NOTHING reaches the second name: if Bright Data - # answers with the likes blank, Apify is never asked. `verify_automation`'s E2b pins that gap - # BY NAME so a third unwalked chain turns it red — which is exactly what a second name here - # would be today. Apify does sell TikTok (`clockworks/tiktok-profile-scraper`, $0.003/result, - # 0.7% 30-day failure rate — measured from its own store record), so the fallback is buildable; - # it is not declared until a runner walks it. - "tt_profile": ("brightdata",), - "tt_post_metrics": ("brightdata",), - "tt_comments": ("brightdata",), -} - - -def chain(capability: str) -> list: - """The provider order for `capability` — env override first, then the default. - - `AIOS_PROVIDER_ORDER` is a `;`-separated list of `capability=p1,p2` clauses. This is how the - owner's "or vice versa" is expressed WITHOUT a release, and how one tenant can be moved off a - vendor that is having a bad day. - ⛔ An override may REORDER and may DROP, but it can never make an incapable provider capable — - `can()` still gates every name. A chain that reads `ig_post_views=brightdata` therefore - resolves to EMPTY rather than to a provider that would return a wrong number, because a - plausible wrong number is worse than an honest refusal. - """ - raw = (os.environ.get("AIOS_PROVIDER_ORDER") or "").strip() - names = None - for clause in raw.split(";"): - if "=" in clause: - cap_name, _, order = clause.partition("=") - if cap_name.strip() == capability: - names = [x.strip() for x in order.split(",") if x.strip()] - if names is None: - names = list(DEFAULT_CHAINS.get(capability) or ()) - return [PROVIDERS[n] for n in names if n in PROVIDERS and PROVIDERS[n].can(capability)] - - -def estimate(capability: str, records: int) -> dict: - """What the FIRST capable provider would cost for `records` — the number an operator plans on. - - Reported per capability rather than per run because that is the unit that scales: at a thousand - tenants the question is never "what did this run cost", it is "what does adding view counts to - every post cost per month". - """ - ch = chain(capability) - if not ch: - return {"capability": capability, "records": records, "provider": None, "usd": 0.0, - "note": "no configured, capable provider"} - p = ch[0] - c = p.cap(capability) - return {"capability": capability, "records": records, "provider": p.key, - "usd": round((c.cost_per_record or 0.0) * max(0, int(records)), 4), - "note": c.note} - - -@dataclass -class Attempt: - provider: str - ok: bool - note: str = "" - records: int = 0 - seconds: float = 0.0 - - -def run(capability, work, satisfied=None, log=None): - """Walk the chain for `capability` until one provider gives a SATISFYING answer. - - `work(provider) -> (result, note)`; a non-empty note means it did not answer. - `satisfied(result) -> bool` decides whether an answer is good enough to stop. Default: any - truthy result. - - Returns `(result, attempts)`. `attempts` is the audit trail — every provider tried, whether it - satisfied, and how long it took — because "where did this number come from and what did it - cost" is a question somebody asks about a bill, not about a stack trace. - - ⛔ THE `satisfied` HOOK IS THE WHOLE POINT. Without it this is an error-handler, and the - failure it must catch is not an error: a vendor answering 200 with the one field you needed - left blank. That is exactly how Bright Data behaves on view counts, and a chain that only - caught exceptions would have stopped there forever and never reached Apify. - """ - log = log or (lambda *_a: None) - ok = satisfied or (lambda r: bool(r)) - attempts, last = [], None - for provider in chain(capability): - started = time.time() - try: - result, note = work(provider) - except Exception as e: # noqa: BLE001 - # ⚠ TYPE NAME ONLY. A vendor client's str() can carry the URL, and the credential is - # one refactor away from being a query param; this line must not be what leaks it. - result, note = None, f"{type(e).__name__}" - secs = round(time.time() - started, 2) - if note: - attempts.append(Attempt(provider.key, False, note, 0, secs)) - log(f" {provider.label}: {note}, falling through") - continue - if not ok(result): - attempts.append(Attempt(provider.key, False, "answered without the field asked for", - 0, secs)) - log(f" {provider.label}: answered, but not with what was asked for, falling through") - last = result if last is None else last - continue - attempts.append(Attempt(provider.key, True, "", _count(result), secs)) - log(f" {provider.label}: ok ({_count(result)} records, {secs}s)") - return result, attempts - return last, attempts - - -def _count(result): - if isinstance(result, (list, tuple, set)): - return len(result) - if isinstance(result, dict): - return len(result) - return 1 if result else 0 - - -def wire(): - """What the Settings surface renders. Booleans and labels — NEVER a credential. - - Mirrors the `hikerReady` rule the rest of the product follows: a surface is told WHETHER a - provider is usable, never what the key is. - """ - return { - "providers": [ - {"key": p.key, "label": p.label, "configured": p.configured(), - "capabilities": sorted(k for k, c in p.caps.items() if c.capable)} - for p in PROVIDERS.values() - ], - "chains": {cap: [p.key for p in chain(cap)] for cap in DEFAULT_CHAINS}, - "incapable": { - f"{p.key}:{cap}": c.note - for p in PROVIDERS.values() for cap, c in p.caps.items() if not c.capable - }, - } - - - - -# ═══════════════════ WAVE 36 · W36-T35 (ruling R4, contract C5) — THE LLM LADDER ════════════════ -# -# Owner item 4, verbatim (2026-08-18): *"I'm also getting errors everywhere when I want to use the -# assisntant: 'the assistant could not be reached just now (openrouter: HTTP 402)' and 'cerebras: -# HTTP 402; groq: HTTP 404; openrouter: HTTP 402; anthropic: tool calls are not wired for this -# shape'."* -# -# ⭐⭐ R4 IS THREE CLAUSES AND THEY LAND IN THREE DIFFERENT PLACES. (1) Anthropic becomes the -# tool-calling path that always works — a WIRE, in `routes_query`. (2) A provider with no credit is -# SKIPPED rather than tried — a memo, `mark_no_credit` below. (3) No raw `HTTP 402` ever reaches a -# screen — a SENTENCE, `refusal_sentence` below. The declaration here is what the first two read. -# -# ⚠ WHY A SECOND REGISTRY RATHER THAN ROWS IN `PROVIDERS` ABOVE. A scraping provider is -# `(key_env, caps)` and is billed per RECORD; an LLM provider is `(env, url, model, wire)` and is -# billed per TOKEN. Folding them into one dict would mean four fields that are meaningless for half -# the rows and an `estimate()` that answers $0.00 for anything LLM-shaped. What they SHARE is the -# thing worth sharing: `Capability`, so "MEASURED INCAPABLE" means exactly the same thing on both -# sides, and `llm_chain()` refuses an incapable row exactly as `chain()` does. -# -# ⛔ THE DECLARATION IS THE POINT (staged item 3). `_FAILED_GEN` in `routes_query` is a REGEX that -# recovers a tool call out of a provider's 400 — the evidence that guessing at capability failed. -# A row that says `llm_tool_calling: Capability(False, …)` is never offered for a tool-calling -# turn at all, so the guess never has to be made. - -#: How long a provider stays skipped after it tells us it is out of credit. ⚠ A MEMO, NOT A FACT: -#: the balance can be topped up at any moment, so this expires rather than latching. Fifteen -#: minutes is long enough that a chat session does not re-pay the timeout on every turn, and short -#: enough that a top-up is picked up without a restart. -CREDIT_COOLDOWN_S = float(os.environ.get("AIOS_CREDIT_COOLDOWN_S") or 900) - -#: `{provider name: unix ts when the memo expires}`. ⚠ PROCESS-LOCAL AND DELIBERATELY SO — it is a -#: latency optimisation, not a billing record. A second container learns the same thing from its -#: own first 402, and neither one can be wrong for longer than the cooldown. -_NO_CREDIT: dict[str, float] = {} - - -@dataclass(frozen=True) -class LlmProvider: - """One chat-completions endpoint, and what it is DECLARED able to do.""" - name: str - label: str - env: str - url: str - model: str - #: `openai` = the OpenAI-compatible `/chat/completions` shape. `anthropic` = the Messages API, - #: which is a different body, a different auth header and a different result shape. - wire: str - caps: dict = field(default_factory=dict) - - def configured(self) -> bool: - return bool((os.environ.get(self.env) or "").strip()) - - def can(self, capability: str) -> bool: - cap = self.caps.get(capability) - return bool(cap and cap.capable and self.configured()) - - -#: ⭐ ORDER IS THE LADDER, AND ANTHROPIC IS FIRST BECAUSE OF R4. `routes_query`'s old comment put -#: cerebras first *"because this path needs tool calling and cerebras carries this account's -#: tool-capable model"* — R4 replaces that premise: Anthropic is the tool-calling path that always -#: works, and the others are the cheap seats it falls through to. -LLM_PROVIDERS: dict[str, LlmProvider] = { - "anthropic": LlmProvider( - name="anthropic", label="Anthropic", env="ANTHROPIC_API_KEY", - url="https://api.anthropic.com/v1/messages", - # ⚠ Claude Opus 5, $5 / $25 per million tokens (2026-06 list). Override per deployment with - # `AIOS_ANTHROPIC_MODEL` — the id is read at call time, so a cheaper tier - # (`claude-sonnet-5`, $3 / $15) is an environment change, not a release. - # ⭐ W37-T39 re-verified this default against the vendor rather than the docs: a real - # Messages POST answers HTTP 200, and `claude-opus-5` is in `GET /v1/models`. - model=os.environ.get("AIOS_ANTHROPIC_MODEL") or "claude-opus-5", - wire="anthropic", - caps={ - # ⭐ MEASURED, and it is the whole of R4's first clause: the Messages API answers with a - # typed `tool_use` content block carrying parsed `input`. There is nothing to recover - # out of a 400 and no regex in the path — which is exactly what `_FAILED_GEN` exists to - # apologise for on the other wire. - "llm_tool_calling": Capability(True, 0.0, - "typed tool_use content block; no text recovery path"), - "llm_chat": Capability(True, 0.0, "Messages API"), - "llm_json_mode": Capability(True, 0.0, "output_config.format, schema-constrained"), - }), - "cerebras": LlmProvider( - name="cerebras", label="Cerebras", env="CEREBRAS_API_KEY", - url="https://api.cerebras.ai/v1/chat/completions", - # ⚠ STILL A LIVE ID, and W37-T39 checked rather than assumed: `gpt-oss-120b` is one of the - # two ids `GET https://api.cerebras.ai/v1/models` returns. What it is NOT is callable on - # this account — a real POST answers `HTTP 402 payment_required`, which `mark_no_credit` - # already knows how to survive (R4: skipped, not tried). - model="gpt-oss-120b", wire="openai", - caps={ - # ⚠ STILL "DECLARED, NOT MEASURED", DELIBERATELY. The 2026-08-19 sweep could not - # measure this rung: a 402 comes back before any tool is considered, so there is no - # observation to record. Leaving the old wording is the honest answer — upgrading it - # to MEASURED beside its two neighbours would be claiming an account balance as - # evidence about a capability. - "llm_tool_calling": Capability(True, 0.0, - "DECLARED, not measured: this account's tool-capable " - "model per the wave-32 ladder note. ⚠ 2026-08-19: the " - "account answers 402, so this stays unmeasured"), - "llm_chat": Capability(True, 0.0, "OpenAI-compatible"), - "llm_json_mode": Capability(True, 0.0, "response_format json_object"), - }), - "groq": LlmProvider( - name="groq", label="Groq", env="GROQ_API_KEY", - # ⛔⛔ D-345, FIXED W37-T39 (2026-08-19): this read `llama-3.3-70b-versatile` and Groq - # RETIRED it. Measured, not inferred — a real POST with the live key answered - # `HTTP 404 {"code":"model_not_found"}`, and `GET /openai/v1/models` returns 13 ids with - # that one absent. ⚠ THE STATUS IS THE EVIDENCE AND 404 IS THE ONLY ONE THAT LICENSES A - # RENAME: cerebras answered 402 on the same sweep, which is a BILLING fact about the - # account and says nothing about its model id (`gpt-oss-120b` is in its /models list). - # "Fixing" an id behind a 401/402 is how a second bug ships behind a green probe. - # ⚠ THE PREFIX IS NOT A TYPO: Groq serves this model as `openai/gpt-oss-120b` while - # Cerebras serves the same family bare as `gpt-oss-120b`. The two rungs are DELIBERATELY - # spelled differently and a sweep that "normalises" them breaks one of them. - url="https://api.groq.com/openai/v1/chat/completions", - model="openai/gpt-oss-120b", wire="openai", - caps={ - # ⭐ MEASURED 2026-08-19, W37-T39, and this comment used to say the opposite. The old - # text read *"DECLARED, not measured … `routes_query._FAILED_GEN` exists because SOME - # provider on this wire answers 400 with the call in `failed_generation`; the repo - # never recorded which. Flip this to False the day it is"*. It is recorded now: on - # this id Groq answers HTTP 200 with a TYPED `tool_calls` block carrying parsed - # `arguments`, so it is not the provider `_FAILED_GEN` apologises for. - "llm_tool_calling": Capability(True, 0.0, - "MEASURED 2026-08-19: HTTP 200, typed tool_calls block " - "with parsed arguments; no failed_generation recovery"), - "llm_chat": Capability(True, 0.0, "OpenAI-compatible"), - "llm_json_mode": Capability(True, 0.0, "response_format json_object"), - }), - "openrouter": LlmProvider( - name="openrouter", label="OpenRouter", env="OPENROUTER_API_KEY", - url="https://openrouter.ai/api/v1/chat/completions", - model="openai/gpt-4o-mini", wire="openai", - caps={ - # ⭐ MEASURED 2026-08-19, W37-T39 (was "DECLARED, not measured"): HTTP 200 with a typed - # `tool_calls` block. So on the OpenAI wire BOTH reachable rungs tool-call cleanly, and - # `_FAILED_GEN` is still holding a door nobody on this ladder has been seen to use. - "llm_tool_calling": Capability(True, 0.0, - "MEASURED 2026-08-19: HTTP 200, typed tool_calls block"), - "llm_chat": Capability(True, 0.0, "OpenAI-compatible"), - "llm_json_mode": Capability(True, 0.0, "response_format json_object"), - }), -} - -LLM_DEFAULT_ORDER = ("anthropic", "cerebras", "groq", "openrouter") - - -def mark_no_credit(name, seconds=None): - """Remember that `name` said it is out of credit, so the next turn SKIPS it (R4). - - ⛔ THE SECOND CLAUSE OF R4 IS "SKIPPED, NOT TRIED", and without a memo there is nowhere for - that to live: a stateless ladder re-tries the empty account on every single turn, pays its - round trip, and shows the reader a longer error each time. This is that memo. - """ - _NO_CREDIT[str(name)] = time.time() + float( - CREDIT_COOLDOWN_S if seconds is None else seconds) - return _NO_CREDIT[str(name)] - - -def no_credit(name): - """Is this provider inside its out-of-credit cooldown? Expiry is checked, never assumed.""" - until = _NO_CREDIT.get(str(name)) - if not until: - return False - if time.time() >= until: - _NO_CREDIT.pop(str(name), None) - return False - return True - - -def clear_credit_memo(name=None): - """Forget one memo, or all of them. For a gate, and for an operator after a top-up.""" - if name is None: - _NO_CREDIT.clear() - else: - _NO_CREDIT.pop(str(name), None) - - -def llm_chain(capability="llm_tool_calling"): - """The provider order for `capability` — declaration first, credit memo second. - - Three filters, in this order, and each removes a DIFFERENT kind of row: - 1. `can()` — declared capable AND configured. An incapable row is never offered, so a - turn cannot be spent discovering it (the `chain()` rule, one layer up). - 2. `no_credit()` — R4's skip. A provider that told us its balance is empty is passed over - until the memo expires. - 3. the ORDER itself, overridable with `AIOS_LLM_ORDER` (`anthropic,groq`) so a deployment - can be moved off a vendor without a release — the same clause `AIOS_PROVIDER_ORDER` - carries for the scraping side. - """ - raw = (os.environ.get("AIOS_LLM_ORDER") or "").strip() - names = [x.strip() for x in raw.split(",") if x.strip()] or list(LLM_DEFAULT_ORDER) - return [LLM_PROVIDERS[n] for n in names - if n in LLM_PROVIDERS and LLM_PROVIDERS[n].can(capability) and not no_credit(n)] - - -#: What an HTTP status MEANS, in words a person can act on. ⛔⛔ R4's THIRD CLAUSE LIVES HERE AND -#: IT IS NOT COSMETIC: `HTTP 402` on a screen tells a reader nothing they can do, and the owner -#: quoted it back at us twice. Every sentence names the VENDOR and the ACTION. -_STATUS_WORDS = { - 401: "{label} would not accept our key", - 403: "{label} would not accept our key", - 402: "{label} is out of credit", - 404: "{label} does not offer the model we asked it for", - 408: "{label} took too long", - 413: "the question was too long for {label}", - 429: "{label} is rate limiting us right now", -} - -#: Substrings that mean "no money" on a wire that does not use 402. ⚠ Anthropic answers a spent -#: balance with a 400 or 403 carrying a message, not with a status code of its own, so this is the -#: one place a body has to be read. It is a LOWERCASE substring test on the vendor's own words and -#: it only ever decides whether to SKIP a provider, never whether to trust one. -_CREDIT_WORDS = ("credit balance", "insufficient credit", "insufficient_quota", "out of credit", - "quota exceeded", "billing", "payment required", "add credits") - - -def is_credit_failure(status, body=""): - """Did this response mean "the account is empty"? Status first, then the vendor's own words.""" - if int(status or 0) == 402: - return True - if int(status or 0) not in (400, 403, 429): - return False - return any(word in str(body or "").lower() for word in _CREDIT_WORDS) - - -def refusal_sentence(name, status, body=""): - """One provider's failure, as a SENTENCE. Never a bare status code, never a vendor stack trace. - - ⚠ THE BODY IS READ AND NEVER QUOTED. A provider's error body can carry an account id, a key - prefix or an internal trace; the only thing taken out of it is the yes/no answer to "is this a - credit problem", and what reaches the caller is this module's own wording. - """ - label = (LLM_PROVIDERS.get(str(name)) or LlmProvider(name, str(name), "", "", "", "")).label - if is_credit_failure(status, body): - return f"{label} is out of credit" - code = int(status or 0) - if code in _STATUS_WORDS: - return _STATUS_WORDS[code].format(label=label) - if 500 <= code <= 599: - return f"{label} is having trouble at their end" - return f"{label} did not answer" - - -# ═══════════ THE ANTHROPIC WIRE, ONCE (W36-T35 / ASK D-18, ruling R4) ═══════════════════════════ -# -# ⛔⛔ TWO DOORS IN THIS PRODUCT CALL ANTHROPIC AND THEY MUST NOT EACH LEARN THE MESSAGES API. -# `routes_query._call_model` (the Assistant and Query) and `ai_review.draft_flow` (the automation -# drafter) both need it, and the owner quoted an error from EACH of them in one breath: -# *"the assistant could not be reached just now (openrouter: HTTP 402)"* and *"anthropic: tool -# calls are not wired for this shape"*. Two implementations of one wire is -# [[one-question-two-normalizers]] before a line is written, so the wire lives here, beside the -# ladder that declares the rung. -# -# FOUR THINGS THE OPENAI-COMPATIBLE SHAPE GETS WRONG, each a 400 on its own: -# 1. the system prompt is a TOP-LEVEL field, not a `{"role": "system"}` message -# 2. a tool is `{name, description, input_schema}` FLAT, not nested under `function` -# 3. `temperature` and friends are REMOVED on the current model family -# 4. `tool_choice` is an OBJECT (`{"type": "auto"}` / `{"type": "any"}`), not a string -# -# ⚠ AND ONE THING THAT IS NOT A SHAPE: `effort` is model-gated. `output_config.effort` errors on -# Haiku 4.5, so it is a PARAMETER here and the caller decides — the drafter runs on haiku and omits -# it, the assistant runs on the Opus tier and sends it. - -#: The Messages API version header. A DATE that pins the WIRE FORMAT, never a model. -ANTHROPIC_VERSION = "2023-06-01" - - -def anthropic_request(*, model, key, system, messages, tools, max_tokens, - tool_choice="auto", effort=None): - """`{url, headers, json}` for one Messages API call. Pure: reads no environment, sends nothing. - - `messages` is the OpenAI-shaped list this product already builds; the `system` turns are lifted - out of it, because that is where this API wants them. `tools` is the OpenAI-shaped tool list, - re-addressed rather than re-derived, so a schema change happens in one place. - """ - system_text = "\n\n".join(str(m.get("content") or "") for m in messages - if m.get("role") == "system") - if system: - system_text = (system_text + "\n\n" + str(system)).strip() if system_text else str(system) - turns = [{"role": ("assistant" if m.get("role") == "assistant" else "user"), - "content": str(m.get("content") or "")} - for m in messages if m.get("role") != "system" and str(m.get("content") or "").strip()] - wire_tools = [{"name": t["function"]["name"], - "description": t["function"]["description"], - "input_schema": t["function"]["parameters"]} for t in (tools or [])] - body = { - "model": str(model), - "max_tokens": int(max_tokens), - "system": system_text, - "messages": turns, - "tools": wire_tools, - } - # ⛔ W37-T39: `tool_choice` IS OMITTED WHEN THERE ARE NO TOOLS, and the builder decides that - # rather than each caller. The Messages API refuses `tool_choice` beside an empty `tools` list, - # so this used to be a `req["json"].pop("tool_choice", None)` at the one call site that sends no - # tools — a rule living outside the thing it constrains, which is [[limit-with-no-enforcer]]: - # the next no-tools caller inherits a 400 that reads like a model problem, not a shape problem. - # ⚠ Safe for all three callers today, checked rather than assumed: `routes_query` and - # `ai_review.draft_flow` both pass a non-empty tool list and keep the key exactly as before. - if wire_tools: - body["tool_choice"] = {"type": "any" if tool_choice in ("required", "any") else "auto"} - if effort: - body["output_config"] = {"effort": str(effort)} - return {"url": LLM_PROVIDERS["anthropic"].url, - "headers": {"x-api-key": str(key), - "anthropic-version": ANTHROPIC_VERSION, - "content-type": "application/json"}, - "json": body} - - -#: Cached result of `import anthropic`: the module, or `False` once it is known to be absent. -#: ⚠ `None` is NOT the "absent" sentinel — a plain falsy check would re-attempt the import on every -#: call, and a failed import is not cheap. `False` says "asked and answered". -_SDK: object = None - - -def anthropic_sdk(): - """The official `anthropic` SDK, or `None` if this deployment does not carry it. - - ⭐⭐ D-346, W37-T39. THIS FUNCTION IS THE WHOLE OF THE FIX AND ALSO THE WHOLE OF ITS RISK. - `aios-web/requirements.txt` is what the Dockerfile installs and it is OUTSIDE every lane's - fence this wave, so the pin is another session's edit. A hard `import anthropic` at module - scope would therefore turn a missing line in a manifest into a container that cannot boot at - all: the API imports this module on every request path. - - So the import is LAZY and its absence is REPORTED rather than fatal — `anthropic_send` falls - back to the raw `requests` POST that has always worked, and says which transport ran. When the - pin lands the SDK path takes over with no further change. - ⛔ The fallback is a BRIDGE, not a second implementation: both transports send the SAME body - `anthropic_request()` built and return the SAME `(status, body)` pair, so `anthropic_read`, - `is_credit_failure` and `refusal_sentence` each keep exactly one normalizer - ([[one-question-two-normalizers]]). - """ - global _SDK - if _SDK is None: - try: - import anthropic as _mod # noqa: PLC0415 - deliberately lazy; see the docstring - _SDK = _mod - except Exception: - _SDK = False - return _SDK or None - - -def anthropic_send(req, timeout=None, use_sdk=None): - """POST one `anthropic_request()` dict. Returns `(status, body, transport)`. - - ⭐ `transport` is `'sdk'` or `'http'` and it is RETURNED, not logged and forgotten: a reader - who cannot tell which path ran cannot tell whether the requirements pin reached the container - [[report-the-cause-before-you-fix-it]]. ⛔ A gate must assert on THIS RETURN VALUE, never on - `ai_review.LAST_ANTHROPIC_TRANSPORT` — that global is a convenience for `GET /meta`, and a - check keyed to it passes on a stale value written by an earlier check in the same process. - - `status` is an int and `body` is the parsed JSON dict in BOTH paths, including on an error — - the SDK raises where `requests` returns, and normalising that difference here is the only - reason this function exists rather than the caller branching on transport. - - ⚠ `use_sdk=False` forces the fallback. It exists so a gate can exercise BOTH transports - without assigning to `_SDK`: a test that mutates a module global and then throws leaves every - later check in that process running on the wrong path and passing for the wrong reason. - """ - sdk = None if use_sdk is False else anthropic_sdk() - payload = dict(req.get("json") or {}) - if sdk is None: - r = requests.post(req["url"], headers=req["headers"], json=payload, - timeout=(timeout or 60)) - try: - return r.status_code, (r.json() if r.content else {}), "http" - except Exception: - return r.status_code, {"_text": (r.text or "")[:2000]}, "http" - - key = (req.get("headers") or {}).get("x-api-key") or "" - try: - client = sdk.Anthropic(api_key=key, timeout=float(timeout or 60)) - msg = client.messages.create(**payload) - # ⚠ `.model_dump()` is what makes ONE reader serve both transports: it hands back the same - # wire-shaped dict the raw POST parses out of the response body. - return 200, msg.model_dump(), "sdk" - except Exception as exc: - # ⛔ THE STATUS IS THE PRODUCT HERE. Every sentence the reader sees is chosen by - # `refusal_sentence(status)`, so an exception that loses its code turns "Anthropic is out - # of credit" into "Anthropic did not answer" — the exact regression R4 was written to end. - status = int(getattr(exc, "status_code", 0) or 0) - body = getattr(exc, "body", None) - if not isinstance(body, dict): - body = {"_text": str(exc)[:2000]} - if not status: - # No status at all = it never reached the vendor (DNS, TLS, timeout). 408 is the one - # code `_STATUS_WORDS` already words as a reachability problem rather than a refusal. - status = 408 if isinstance(exc, getattr(sdk, "APIConnectionError", ())) else 0 - return status, body, "sdk" - - -def anthropic_read(body): - """`(text, tool_input, refusal)` out of a Messages API answer. - - ⛔ `stop_reason` IS CHECKED BEFORE `content` IS READ. A safety decline answers **HTTP 200** with - `stop_reason: "refusal"` and an empty or partial `content`, so code that indexes `content[0]` - unconditionally breaks on exactly the turn a person most needs explained. - ⭐ AND THE TOOL CALL ARRIVES PARSED. `tool_use.input` is already a dict — no `json.loads`, and - no regex recovering a call out of a 400, which is what the other wire needs. - """ - body = body if isinstance(body, dict) else {} - if str(body.get("stop_reason") or "") == "refusal": - return "", None, "the assistant declined to answer that one" - blocks = [b for b in (body.get("content") or []) if isinstance(b, dict)] - text = " ".join(str(b.get("text") or "") for b in blocks if b.get("type") == "text").strip() - calls = [b for b in blocks if b.get("type") == "tool_use"] - got = calls[0].get("input") if calls else None - return text, (got if isinstance(got, dict) else None), None - - -def llm_status(capability="llm_tool_calling"): - """Per provider: configured, declared-capable, in cooldown, and WHY — contract C5's payload. - - ⭐ ONE LIST, ONE DOOR. The Assistant's model picker and the Agent chat's toggle (W36-T34) read - THIS, so a model offered in one place cannot be missing from the other, and neither can offer a - provider the ladder would refuse to call [[permitted-is-not-answerable]]. - """ - rows = [] - for name in LLM_DEFAULT_ORDER: - p = LLM_PROVIDERS[name] - cap = p.caps.get(capability) - rows.append({ - "provider": p.name, "label": p.label, "model": p.model, "wire": p.wire, - "configured": p.configured(), - "toolCalling": bool((p.caps.get("llm_tool_calling") or Capability(False)).capable), - "jsonMode": bool((p.caps.get("llm_json_mode") or Capability(False)).capable), - "capable": bool(cap and cap.capable), - "outOfCredit": no_credit(name), - "note": (cap.note if cap else ""), - }) - return rows - - -# ============================================================================================= -# THE CANONICAL SCHEMA — owner ruling 2026-08-08: -# *"standardize the schema between Bright Data and APIfy so we keep using the same pre-set -# database even if the underlying engine changes"* -# -# ⛔ THE PRESET TABLES ARE THE CONTRACT; A VENDOR IS AN IMPLEMENTATION DETAIL. `ut_ig_posts` and -# `ut_ig_post_snapshots` must not gain, lose or rename a column because a chain was reordered — a -# tenant's saved views, filters, rollups and forms all bind to these keys, and a schema that moves -# with the vendor turns a routing change into a data migration. -# -# So every provider normalises INTO the keys below and nothing reads a vendor row downstream. -# `verify_automation` asserts both normalisers emit exactly `CANONICAL_POST_KEYS` on a fixture, so -# a third provider cannot ship with a near-miss key like `viewCount` and silently write a column -# nobody declared. -# -# ⚠ ONE NAME PER MEASUREMENT, AND `views` IS THE MEASUREMENT INSTAGRAM DISPLAYS. Meta folded -# Impressions/Plays/Video Views into a single **Views** metric on 2025-04-10, so carrying both a -# `views` and a `plays` column would be modelling a distinction the platform deleted — and it is -# exactly the distinction that let an account-grain number wear the "Views" label for a month. -# ⛔ THE VENDOR KEY THAT LOOKS RIGHT IS THE WRONG ONE, ON BOTH VENDORS. Apify ships BOTH -# `videoViewCount` (10,678) and `videoPlayCount` (137,684) for one reel whose true displayed count -# is ~137K — and Bright Data's useless `views` is 10,638 for that same reel. The two vendors' junk -# fields AGREE with each other, which is precisely what makes picking by name so dangerous. -# canonical `views` <- apify `videoPlayCount` ✅ matches the grid -# canonical `views` <- apify `videoViewCount` ⛔ off by 13x -# canonical `views` <- brightdata `views` ⛔ off by 13x AND account-grain -# ============================================================================================= - -#: Every key a normalised POST row may carry. Absent > blank: a key is omitted when the provider -#: did not answer, because `upsert_rows` merges and an empty string would ERASE what an earlier -#: paid run learned. -CANONICAL_POST_KEYS = ( - "shortcode", "url", "influencer_key", "posted_at", "type", "caption", - "likes", "comments", "views", "paid_partnership", "partner", "hashtags", - "alt_text", "tagged_location", "source_payload", - # ⭐ 2026-08-09 — the three the SECOND provider answers and the first does not. Each has a - # matching `field_def` in the engine's POST_FIELDS; a key here without a column there is a - # value that normalises cleanly and is then dropped by the write door, silently. - "plays", "video_duration", "comments_disabled", -) - - -def _int_or_none(v): - """`-1` IS NOT A COUNT. Apify returns `likesCount: -1` when the creator HIDES their like - count — a real state that is not a measurement. Writing -1 would render as a negative like - count; writing 0 would claim nobody liked it. Both are lies, so the key is omitted.""" - try: - n = int(v) - except (TypeError, ValueError): - return None - return None if n < 0 else n - - -def normalize_post_apify(row): - """One Apify `instagram-scraper` item -> the canonical post row.""" - if not isinstance(row, dict): - return None - code = str(row.get("shortCode") or "").strip() - if not code: - return None - out = { - "shortcode": code, - "url": str(row.get("url") or f"https://www.instagram.com/reel/{code}/"), - "influencer_key": str(row.get("ownerUsername") or ""), - "posted_at": str(row.get("timestamp") or "").replace("T", " ")[:16], - "type": "video" if str(row.get("type") or "").lower() == "video" else - ("carousel" if row.get("childPosts") else "image"), - "caption": str(row.get("caption") or ""), - # ⭐ THE FIELD THIS WHOLE PROVIDER EXISTS FOR. `videoPlayCount`, never `videoViewCount`: - # the wrong one agreed with Bright Data's junk (10,678 vs 10,638) on a reel whose true - # count was ~137K, and `videoPlayCount` matched a browser read to the digit. - "views": _int_or_none(row.get("videoPlayCount")), - # ⭐⭐ 2026-08-09 (owner: *"Video plays (# Plays) field isn't in APIfy? i believe it is"*). - # They were right, and the `plays` COLUMN had been empty on all 816 rows because nothing - # ever wrote it — the field existed with no writer. MEASURED on two of their own reels: - # `videoPlayCount` 216,904 / 95,331 and `videoViewCount` 115,929 / 30,871. Two different - # real numbers, and we were storing only one of them. - # ⚠ `plays` TAKES THE PLAY COUNT, which is also what `views` carries today — so the two - # columns will agree until somebody decides otherwise, and that decision is the OWNER'S: - # re-sourcing `views` to `videoViewCount` would change what the ~480 rows already captured - # mean, and a column whose meaning changes halfway down is the one thing worse than a - # column with no data. Flagged rather than done. - "plays": _int_or_none(row.get("videoPlayCount")), - "likes": _int_or_none(row.get("likesCount")), - "comments": _int_or_none(row.get("commentsCount")), - # ⭐ FIELDS BRIGHT DATA DOES NOT RETURN AT ALL, kept because they are already paid for in - # this same response (owner: *"whatever APIfy has more than BD pls use it"*). - "video_duration": _int_or_none(row.get("videoDuration")), - "comments_disabled": "1" if row.get("isCommentsDisabled") else "", - "hashtags": ", ".join(str(h) for h in (row.get("hashtags") or []) if h), - "alt_text": str(row.get("alt") or ""), - "paid_partnership": "1" if row.get("paidPartnership") else "", - "partner": ", ".join(str((s or {}).get("username") or "") - for s in (row.get("sponsors") or []) if isinstance(s, dict)), - "tagged_location": str((row.get("locationName") or "")), - "source_payload": json.dumps(row, default=str)[:32_000_000], - } - return {k: v for k, v in out.items() if v not in (None, "")} - - -def normalize_profile_apify(row): - """One Apify `instagram-scraper` PROFILE item (`resultsType: "details"`) -> our snapshot shape. - - ⭐ WHY THIS EXISTS (owner report 2026-08-09: *"I need Bright Data and Apify to work correctly - in tandem"*). `PROVIDERS["apify"]` has declared `ig_profile` capable since 2026-08-08 and - `DEFAULT_CHAINS["ig_profile"]` has read `("brightdata", "apify")` — but **nothing ever ran - that chain.** `connectors_ig.pull_profile` went Bright Data -> anonymous HTML rungs and Apify - was never asked, so a profile Bright Data cannot scrape came back `blocked` while a - configured, declared-capable provider sat unused. A registry entry with no runner is a - promise the product does not keep ([[flag-shipped-without-its-writer]]). - - ⚠ THE KEYS ARE APIFY'S camelCase, and mapping them HERE is the boundary rule this module - already enforces for posts: nothing downstream may ever see a vendor-shaped key, so the - snapshot writer cannot tell which vendor answered — which is what makes the fallback - invisible to every consumer instead of a second schema. - - ⛔ OMIT, NEVER ZERO. A field Apify did not send is dropped, exactly as `normalize_post_apify` - drops a missing count: writing 0 followers would claim we measured an empty account, and the - caller's `satisfied` hook reads absence as "did not answer" and falls through. A zero would - stop the chain on a lie. - """ - if not isinstance(row, dict): - return None - # ⛔ AN ERROR ENVELOPE IS NOT A PROFILE (measured 2026-08-09). Apify answers a dead handle - # with a 200 and a well-formed item — `{"username": …, "error": "not_found", - # "errorDescription": "Post does not exist"}` — and this function used to build a "profile" - # out of it: a username, a url and a source_payload, with every measured field absent. It - # then read to the caller as a vendor that answered, so the reason was replaced by a shrug. - # Refused HERE as well as in `connectors_ig.apify_profile` on purpose: the boundary rule this - # module exists for is that nothing downstream ever sees a vendor-shaped key, and a vendor's - # ERROR shape is the one that must never become a row. - # ⚠ TRUTHINESS, NOT KEY PRESENCE — a successful item carries `error: null` (measured on - # `sriyynntt`), so `"error" in row` would refuse every good profile. - if row.get("error"): - return None - handle = str(row.get("username") or "").strip() - if not handle: - return None - out = { - "username": handle, - "full_name": str(row.get("fullName") or ""), - "bio": str(row.get("biography") or ""), - "followers": _int_or_none(row.get("followersCount")), - "following": _int_or_none(row.get("followsCount")), - "posts_count": _int_or_none(row.get("postsCount")), - "verified": "1" if row.get("verified") else "", - "external_url": str(row.get("externalUrl") or ""), - "ig_id": str(row.get("id") or ""), - "profile_url": str(row.get("url") or f"https://www.instagram.com/{handle}/"), - "business_category": str(row.get("businessCategoryName") or ""), - "is_business": "1" if row.get("isBusinessAccount") else "", - "is_private": "1" if row.get("private") else "", - "highlights_count": _int_or_none(row.get("highlightReelCount")), - "source_payload": json.dumps(row, default=str)[:32_000_000], - } - # ⚠ `followers`/`following` are ints and 0 is a LEGITIMATE value for them, so the filter - # below must not treat 0 as absent the way the post normaliser can — an account really can - # have zero followers. Only None and "" are dropped. - return {k: v for k, v in out.items() if v is not None and v != ""} - - -def canonical_gaps(row, want=("views",)): - """Which requested canonical fields this row does NOT carry — the `satisfied` input. - - Named rather than inlined because "did the vendor actually answer the question" is the whole - fallback trigger, and a chain that asks it differently in two places will drift. - """ - row = row if isinstance(row, dict) else {} - return [k for k in want if row.get(k) in (None, "")] +"""THE PROVIDER LAYER — many vendors behind one capability, chosen per FIELD and per cost. + +⭐⭐ OWNER RULING 2026-08-08: *"Let's do a combination scraper. Do Bright Data first, and if +anything fails, we use APIfy or vice versa. Make this dynamic somehow. remember to cost optimize +when we scale to thousands+ customer … make sure our code is customizable and modular."* + +⛔ THE DESIGN DECISION THAT MATTERS, AND IT IS NOT "TRY A, THEN TRY B". +A provider-level failover ("if Bright Data errors, re-run the whole thing on Apify") is the obvious +shape and it is the expensive one. MEASURED: Bright Data answers profile, likes and comments +correctly and CANNOT answer view counts on any route; Apify answers view counts correctly. A +provider-level fallback would notice the missing views and re-buy the profile, the likes and the +comments from Apify as well — **paying twice for the 90% that already worked**. At one profile that +is noise; at thousands of customers × twelve posts each it is the whole bill. + +So routing is per CAPABILITY, and a capability is the smallest independently-billable thing: + + ig_profile -> brightdata (36 fields, works) + ig_post_metrics -> brightdata (likes + comments, works) + ig_post_views -> apify (Bright Data is INCAPABLE — see `capable=False` below) + ig_comments -> brightdata + +Each capability has an ORDERED list of providers. The first one that is configured AND capable AND +succeeds wins; the rest are never called, so the common path costs exactly one vendor record. + +⚠ "FAILURE" INCLUDES ANSWERING WITH THE FIELD BLANK. A vendor that returns HTTP 200 and a null in +the one column you asked for has failed for our purposes, and a chain that only catches exceptions +would stop at it forever. `run()` takes a `satisfied` predicate and falls through on an unsatisfying +answer exactly as it would on a 500. + +⚠ ORDER IS DATA, NOT CODE. `AIOS_PROVIDER_ORDER` overrides any chain at deploy time +(`ig_post_views=apify,brightdata;ig_profile=apify`), which is what "or vice versa" means and what +lets a tenant be moved off a vendor without a release. +""" +from __future__ import annotations + +import json +import os +import time +from dataclasses import dataclass, field +from typing import Callable + +# ⚠ TOP-LEVEL AND SAFE, unlike the `anthropic` import in `anthropic_sdk()`: `requests` is pinned in +# BOTH manifests and is already a transitive dependency of `huggingface_hub`, so it cannot be the +# line that stops a container booting. That asymmetry is the whole reason the two are imported +# differently — see `anthropic_sdk()`. +import requests + +#: ⚠ APPROXIMATE, AND DELIBERATELY SO. These are list prices per RECORD in USD, used only to RANK +#: providers and to estimate a run's spend for the operator. They are not billing truth — the +#: vendor's own dashboard is. Wrong by 2x still ranks correctly; wrong by 100x does not, which is +#: why they are named and dated rather than guessed silently. +#: Bright Data Instagram datasets ~ $0.0015/record (2026-08 list). Apify instagram-scraper +#: ~ $0.0027/result (2026-08 list). Both re-checked when a provider is added. +#: ⚠ WAVE 30 · T16 — APIFY CORRECTED 0.0023 -> 0.0027, and the direction matters: the old number +#: made the fallback look CHEAPER than it is, and `estimate()` is what a person is shown before +#: they authorise a run. An under-stated price is the one rounding error a cost guard cannot catch. +#: ⚠ Both are LIST rates used to rank and to estimate. They are not invoices, and nothing here +#: reads a live price — a modelled figure that says so is honest; one that pretends is not. +_COST_BRIGHTDATA = float(os.environ.get("AIOS_COST_BRIGHTDATA") or 0.0015) +_COST_APIFY = float(os.environ.get("AIOS_COST_APIFY") or 0.0027) + + +@dataclass(frozen=True) +class Capability: + """What ONE provider can do for ONE capability, and what it costs to ask.""" + #: ⛔ `False` means MEASURED INCAPABLE, not "untried". A provider declared incapable is never + #: called for this capability at all — it cannot be reached by a fallback, cannot be put first + #: by an env override, and cannot silently start being billed because somebody reordered a + #: list. Bright Data's `ig_post_views` is False on the strength of §4e-§4h: every route + #: (Posts x /p/, Posts x /reel/, Reels x /p/, Reels x /reel/, discover-by-profile) on four + #: accounts from 13K to 268M followers returned an account-grain constant or a null. + capable: bool = True + cost_per_record: float = 0.0 + #: Free-text, shown to an operator deciding where their money went. + note: str = "" + + +@dataclass +class Provider: + key: str + label: str + #: The env var holding this provider's credential. NEVER the credential itself — this module + #: is imported by surfaces that serialise their config. + key_env: str = "" + caps: dict = field(default_factory=dict) + + def configured(self) -> bool: + return bool((os.environ.get(self.key_env) or "").strip()) if self.key_env else True + + def cap(self, capability: str) -> Capability | None: + return self.caps.get(capability) + + def can(self, capability: str) -> bool: + c = self.cap(capability) + return bool(c and c.capable and self.configured()) + + +#: ⚠ ADDING A PROVIDER IS A REGISTRY ENTRY PLUS A RUNNER — no change to any caller. That is the +#: "modular/customizable" half of the ruling, and the reason the chains below name STRINGS. +PROVIDERS: dict[str, Provider] = { + "brightdata": Provider( + key="brightdata", label="Bright Data", key_env="AIOS_BRIGHTDATA_KEY", + caps={ + "ig_profile": Capability(True, _COST_BRIGHTDATA, "36 fields incl. bio/email/category"), + "ig_post_metrics": Capability(True, _COST_BRIGHTDATA, "likes + comments + captions"), + "ig_comments": Capability(True, _COST_BRIGHTDATA, "separate paid dataset, opt-in"), + # ⛔ THE ONE THAT COSTS US NOTHING TO GET RIGHT AND EVERYTHING TO GET WRONG. + "ig_post_views": Capability( + False, _COST_BRIGHTDATA, + "MEASURED INCAPABLE: returns one account-grain number for every reel of a " + "creator (identical for two shortcodes in one call, ticking upward between " + "calls) and never populates video_play_count. Their own documented example " + "account no longer reproduces it."), + # ⭐⭐ WAVE 29 (item 7 / D-9 / R1) — TIKTOK. Three datasets, all reachable with our + # existing key, all schema-probed for $0.00 (40/43/17 fields with the vendor's own + # types): `gd_l1villgoiiidt09ci` · `gd_lu702nij2f790tmv9h` · `gd_lkf2st302ap89utw5k`. + "tt_profile": Capability(True, _COST_BRIGHTDATA, + "40 fields incl. bio/engagement rates/region"), + # ⛔ THERE IS NO `tt_post_views` CAPABILITY, AND ITS ABSENCE IS THE CLAIM. + # `TikTok - Posts` declares `play_count: number` and the vendor's own sample says + # "no empty values or zeros" — so on TikTok the view count arrives INSIDE the post + # record and a separate view rung would be a second bill for a number we already have. + # ⚠ DECLARED, NOT MEASURED, and that distinction is the whole scar tissue of §4e: + # Bright Data's Instagram Reels also DECLARES `views: number` and delivers an + # account-grain wrong one. One live pull settles it (W29-T06, spend-gated). If it + # fails, the honest repair is a `tt_post_views` capability routed elsewhere — never a + # quiet fallback bolted onto this one. + "tt_post_metrics": Capability( + True, _COST_BRIGHTDATA, + "likes + comments + shares + saves, and play_count inline (DECLARED " + "no-empties-or-zeros, UNPROVEN until one live pull)"), + "tt_comments": Capability(True, _COST_BRIGHTDATA, + "separate paid dataset, opt-in: 17 fields"), + }), + "apify": Provider( + key="apify", label="Apify", key_env="AIOS_APIFY_KEY", + caps={ + # ⭐ MEASURED 2026-08-08 against the public Reels grid: `videoPlayCount` 137,684 and + # 299,493 vs a browser-read ground truth of 134K-137K and 299K. Exact. + "ig_post_views": Capability(True, _COST_APIFY, + "videoPlayCount, matches Instagram's displayed views"), + "ig_post_metrics": Capability(True, _COST_APIFY, "likes + comments (fallback)"), + "ig_profile": Capability(True, _COST_APIFY, "profile fields (fallback)"), + }), +} + +#: The DEFAULT chain per capability, cheapest-capable-first. Overridable — see `chain()`. +DEFAULT_CHAINS: dict[str, tuple] = { + "ig_profile": ("brightdata", "apify"), + "ig_post_metrics": ("brightdata", "apify"), + # Only one entry, and that is the point: Bright Data is declared incapable, so listing it here + # would be a lie that costs a wasted call on every single post. + "ig_post_views": ("apify",), + "ig_comments": ("brightdata",), + # ⭐ WAVE 29 — TikTok, and every chain is deliberately SINGLE-PROVIDER. + # ⛔ A MULTI-PROVIDER CHAIN IS A PROMISE SOMETHING WALKS IT. `ig_post_metrics` has declared a + # two-provider fallback since wave 28 and NOTHING reaches the second name: if Bright Data + # answers with the likes blank, Apify is never asked. `verify_automation`'s E2b pins that gap + # BY NAME so a third unwalked chain turns it red — which is exactly what a second name here + # would be today. Apify does sell TikTok (`clockworks/tiktok-profile-scraper`, $0.003/result, + # 0.7% 30-day failure rate — measured from its own store record), so the fallback is buildable; + # it is not declared until a runner walks it. + "tt_profile": ("brightdata",), + "tt_post_metrics": ("brightdata",), + "tt_comments": ("brightdata",), +} + + +def chain(capability: str) -> list: + """The provider order for `capability` — env override first, then the default. + + `AIOS_PROVIDER_ORDER` is a `;`-separated list of `capability=p1,p2` clauses. This is how the + owner's "or vice versa" is expressed WITHOUT a release, and how one tenant can be moved off a + vendor that is having a bad day. + ⛔ An override may REORDER and may DROP, but it can never make an incapable provider capable — + `can()` still gates every name. A chain that reads `ig_post_views=brightdata` therefore + resolves to EMPTY rather than to a provider that would return a wrong number, because a + plausible wrong number is worse than an honest refusal. + """ + raw = (os.environ.get("AIOS_PROVIDER_ORDER") or "").strip() + names = None + for clause in raw.split(";"): + if "=" in clause: + cap_name, _, order = clause.partition("=") + if cap_name.strip() == capability: + names = [x.strip() for x in order.split(",") if x.strip()] + if names is None: + names = list(DEFAULT_CHAINS.get(capability) or ()) + return [PROVIDERS[n] for n in names if n in PROVIDERS and PROVIDERS[n].can(capability)] + + +def estimate(capability: str, records: int) -> dict: + """What the FIRST capable provider would cost for `records` — the number an operator plans on. + + Reported per capability rather than per run because that is the unit that scales: at a thousand + tenants the question is never "what did this run cost", it is "what does adding view counts to + every post cost per month". + """ + ch = chain(capability) + if not ch: + return {"capability": capability, "records": records, "provider": None, "usd": 0.0, + "note": "no configured, capable provider"} + p = ch[0] + c = p.cap(capability) + return {"capability": capability, "records": records, "provider": p.key, + "usd": round((c.cost_per_record or 0.0) * max(0, int(records)), 4), + "note": c.note} + + +@dataclass +class Attempt: + provider: str + ok: bool + note: str = "" + records: int = 0 + seconds: float = 0.0 + + +def run(capability, work, satisfied=None, log=None): + """Walk the chain for `capability` until one provider gives a SATISFYING answer. + + `work(provider) -> (result, note)`; a non-empty note means it did not answer. + `satisfied(result) -> bool` decides whether an answer is good enough to stop. Default: any + truthy result. + + Returns `(result, attempts)`. `attempts` is the audit trail — every provider tried, whether it + satisfied, and how long it took — because "where did this number come from and what did it + cost" is a question somebody asks about a bill, not about a stack trace. + + ⛔ THE `satisfied` HOOK IS THE WHOLE POINT. Without it this is an error-handler, and the + failure it must catch is not an error: a vendor answering 200 with the one field you needed + left blank. That is exactly how Bright Data behaves on view counts, and a chain that only + caught exceptions would have stopped there forever and never reached Apify. + """ + log = log or (lambda *_a: None) + ok = satisfied or (lambda r: bool(r)) + attempts, last = [], None + for provider in chain(capability): + started = time.time() + try: + result, note = work(provider) + except Exception as e: # noqa: BLE001 + # ⚠ TYPE NAME ONLY. A vendor client's str() can carry the URL, and the credential is + # one refactor away from being a query param; this line must not be what leaks it. + result, note = None, f"{type(e).__name__}" + secs = round(time.time() - started, 2) + if note: + attempts.append(Attempt(provider.key, False, note, 0, secs)) + log(f" {provider.label}: {note}, falling through") + continue + if not ok(result): + attempts.append(Attempt(provider.key, False, "answered without the field asked for", + 0, secs)) + log(f" {provider.label}: answered, but not with what was asked for, falling through") + last = result if last is None else last + continue + attempts.append(Attempt(provider.key, True, "", _count(result), secs)) + log(f" {provider.label}: ok ({_count(result)} records, {secs}s)") + return result, attempts + return last, attempts + + +def _count(result): + if isinstance(result, (list, tuple, set)): + return len(result) + if isinstance(result, dict): + return len(result) + return 1 if result else 0 + + +def wire(): + """What the Settings surface renders. Booleans and labels — NEVER a credential. + + Mirrors the `hikerReady` rule the rest of the product follows: a surface is told WHETHER a + provider is usable, never what the key is. + """ + return { + "providers": [ + {"key": p.key, "label": p.label, "configured": p.configured(), + "capabilities": sorted(k for k, c in p.caps.items() if c.capable)} + for p in PROVIDERS.values() + ], + "chains": {cap: [p.key for p in chain(cap)] for cap in DEFAULT_CHAINS}, + "incapable": { + f"{p.key}:{cap}": c.note + for p in PROVIDERS.values() for cap, c in p.caps.items() if not c.capable + }, + } + + + + +# ═══════════════════ WAVE 36 · W36-T35 (ruling R4, contract C5) — THE LLM LADDER ════════════════ +# +# Owner item 4, verbatim (2026-08-18): *"I'm also getting errors everywhere when I want to use the +# assisntant: 'the assistant could not be reached just now (openrouter: HTTP 402)' and 'cerebras: +# HTTP 402; groq: HTTP 404; openrouter: HTTP 402; anthropic: tool calls are not wired for this +# shape'."* +# +# ⭐⭐ R4 IS THREE CLAUSES AND THEY LAND IN THREE DIFFERENT PLACES. (1) Anthropic becomes the +# tool-calling path that always works — a WIRE, in `routes_query`. (2) A provider with no credit is +# SKIPPED rather than tried — a memo, `mark_no_credit` below. (3) No raw `HTTP 402` ever reaches a +# screen — a SENTENCE, `refusal_sentence` below. The declaration here is what the first two read. +# +# ⚠ WHY A SECOND REGISTRY RATHER THAN ROWS IN `PROVIDERS` ABOVE. A scraping provider is +# `(key_env, caps)` and is billed per RECORD; an LLM provider is `(env, url, model, wire)` and is +# billed per TOKEN. Folding them into one dict would mean four fields that are meaningless for half +# the rows and an `estimate()` that answers $0.00 for anything LLM-shaped. What they SHARE is the +# thing worth sharing: `Capability`, so "MEASURED INCAPABLE" means exactly the same thing on both +# sides, and `llm_chain()` refuses an incapable row exactly as `chain()` does. +# +# ⛔ THE DECLARATION IS THE POINT (staged item 3). `_FAILED_GEN` in `routes_query` is a REGEX that +# recovers a tool call out of a provider's 400 — the evidence that guessing at capability failed. +# A row that says `llm_tool_calling: Capability(False, …)` is never offered for a tool-calling +# turn at all, so the guess never has to be made. + +#: How long a provider stays skipped after it tells us it is out of credit. ⚠ A MEMO, NOT A FACT: +#: the balance can be topped up at any moment, so this expires rather than latching. Fifteen +#: minutes is long enough that a chat session does not re-pay the timeout on every turn, and short +#: enough that a top-up is picked up without a restart. +CREDIT_COOLDOWN_S = float(os.environ.get("AIOS_CREDIT_COOLDOWN_S") or 900) + +#: `{provider name: unix ts when the memo expires}`. ⚠ PROCESS-LOCAL AND DELIBERATELY SO — it is a +#: latency optimisation, not a billing record. A second container learns the same thing from its +#: own first 402, and neither one can be wrong for longer than the cooldown. +_NO_CREDIT: dict[str, float] = {} + + +@dataclass(frozen=True) +class LlmProvider: + """One chat-completions endpoint, and what it is DECLARED able to do.""" + name: str + label: str + env: str + url: str + model: str + #: `openai` = the OpenAI-compatible `/chat/completions` shape. `anthropic` = the Messages API, + #: which is a different body, a different auth header and a different result shape. + wire: str + caps: dict = field(default_factory=dict) + + def configured(self) -> bool: + return bool((os.environ.get(self.env) or "").strip()) + + def can(self, capability: str) -> bool: + cap = self.caps.get(capability) + return bool(cap and cap.capable and self.configured()) + + +#: ⭐ ORDER IS THE LADDER, AND ANTHROPIC IS FIRST BECAUSE OF R4. `routes_query`'s old comment put +#: cerebras first *"because this path needs tool calling and cerebras carries this account's +#: tool-capable model"* — R4 replaces that premise: Anthropic is the tool-calling path that always +#: works, and the others are the cheap seats it falls through to. +LLM_PROVIDERS: dict[str, LlmProvider] = { + "anthropic": LlmProvider( + name="anthropic", label="Anthropic", env="ANTHROPIC_API_KEY", + url="https://api.anthropic.com/v1/messages", + # ⚠ Claude Opus 5, $5 / $25 per million tokens (2026-06 list). Override per deployment with + # `AIOS_ANTHROPIC_MODEL` — the id is read at call time, so a cheaper tier + # (`claude-sonnet-5`, $3 / $15) is an environment change, not a release. + # ⭐ W37-T39 re-verified this default against the vendor rather than the docs: a real + # Messages POST answers HTTP 200, and `claude-opus-5` is in `GET /v1/models`. + model=os.environ.get("AIOS_ANTHROPIC_MODEL") or "claude-opus-5", + wire="anthropic", + caps={ + # ⭐ MEASURED, and it is the whole of R4's first clause: the Messages API answers with a + # typed `tool_use` content block carrying parsed `input`. There is nothing to recover + # out of a 400 and no regex in the path — which is exactly what `_FAILED_GEN` exists to + # apologise for on the other wire. + "llm_tool_calling": Capability(True, 0.0, + "typed tool_use content block; no text recovery path"), + "llm_chat": Capability(True, 0.0, "Messages API"), + "llm_json_mode": Capability(True, 0.0, "output_config.format, schema-constrained"), + }), + "cerebras": LlmProvider( + name="cerebras", label="Cerebras", env="CEREBRAS_API_KEY", + url="https://api.cerebras.ai/v1/chat/completions", + # ⚠ STILL A LIVE ID, and W37-T39 checked rather than assumed: `gpt-oss-120b` is one of the + # two ids `GET https://api.cerebras.ai/v1/models` returns. What it is NOT is callable on + # this account — a real POST answers `HTTP 402 payment_required`, which `mark_no_credit` + # already knows how to survive (R4: skipped, not tried). + model="gpt-oss-120b", wire="openai", + caps={ + # ⚠ STILL "DECLARED, NOT MEASURED", DELIBERATELY. The 2026-08-19 sweep could not + # measure this rung: a 402 comes back before any tool is considered, so there is no + # observation to record. Leaving the old wording is the honest answer — upgrading it + # to MEASURED beside its two neighbours would be claiming an account balance as + # evidence about a capability. + "llm_tool_calling": Capability(True, 0.0, + "DECLARED, not measured: this account's tool-capable " + "model per the wave-32 ladder note. ⚠ 2026-08-19: the " + "account answers 402, so this stays unmeasured"), + "llm_chat": Capability(True, 0.0, "OpenAI-compatible"), + "llm_json_mode": Capability(True, 0.0, "response_format json_object"), + }), + "groq": LlmProvider( + name="groq", label="Groq", env="GROQ_API_KEY", + # ⛔⛔ D-345, FIXED W37-T39 (2026-08-19): this read `llama-3.3-70b-versatile` and Groq + # RETIRED it. Measured, not inferred — a real POST with the live key answered + # `HTTP 404 {"code":"model_not_found"}`, and `GET /openai/v1/models` returns 13 ids with + # that one absent. ⚠ THE STATUS IS THE EVIDENCE AND 404 IS THE ONLY ONE THAT LICENSES A + # RENAME: cerebras answered 402 on the same sweep, which is a BILLING fact about the + # account and says nothing about its model id (`gpt-oss-120b` is in its /models list). + # "Fixing" an id behind a 401/402 is how a second bug ships behind a green probe. + # ⚠ THE PREFIX IS NOT A TYPO: Groq serves this model as `openai/gpt-oss-120b` while + # Cerebras serves the same family bare as `gpt-oss-120b`. The two rungs are DELIBERATELY + # spelled differently and a sweep that "normalises" them breaks one of them. + url="https://api.groq.com/openai/v1/chat/completions", + model="openai/gpt-oss-120b", wire="openai", + caps={ + # ⭐ MEASURED 2026-08-19, W37-T39, and this comment used to say the opposite. The old + # text read *"DECLARED, not measured … `routes_query._FAILED_GEN` exists because SOME + # provider on this wire answers 400 with the call in `failed_generation`; the repo + # never recorded which. Flip this to False the day it is"*. It is recorded now: on + # this id Groq answers HTTP 200 with a TYPED `tool_calls` block carrying parsed + # `arguments`, so it is not the provider `_FAILED_GEN` apologises for. + "llm_tool_calling": Capability(True, 0.0, + "MEASURED 2026-08-19: HTTP 200, typed tool_calls block " + "with parsed arguments; no failed_generation recovery"), + "llm_chat": Capability(True, 0.0, "OpenAI-compatible"), + "llm_json_mode": Capability(True, 0.0, "response_format json_object"), + }), + "openrouter": LlmProvider( + name="openrouter", label="OpenRouter", env="OPENROUTER_API_KEY", + url="https://openrouter.ai/api/v1/chat/completions", + model="openai/gpt-4o-mini", wire="openai", + caps={ + # ⭐ MEASURED 2026-08-19, W37-T39 (was "DECLARED, not measured"): HTTP 200 with a typed + # `tool_calls` block. So on the OpenAI wire BOTH reachable rungs tool-call cleanly, and + # `_FAILED_GEN` is still holding a door nobody on this ladder has been seen to use. + "llm_tool_calling": Capability(True, 0.0, + "MEASURED 2026-08-19: HTTP 200, typed tool_calls block"), + "llm_chat": Capability(True, 0.0, "OpenAI-compatible"), + "llm_json_mode": Capability(True, 0.0, "response_format json_object"), + }), +} + +LLM_DEFAULT_ORDER = ("anthropic", "cerebras", "groq", "openrouter") + + +def mark_no_credit(name, seconds=None): + """Remember that `name` said it is out of credit, so the next turn SKIPS it (R4). + + ⛔ THE SECOND CLAUSE OF R4 IS "SKIPPED, NOT TRIED", and without a memo there is nowhere for + that to live: a stateless ladder re-tries the empty account on every single turn, pays its + round trip, and shows the reader a longer error each time. This is that memo. + """ + _NO_CREDIT[str(name)] = time.time() + float( + CREDIT_COOLDOWN_S if seconds is None else seconds) + return _NO_CREDIT[str(name)] + + +def no_credit(name): + """Is this provider inside its out-of-credit cooldown? Expiry is checked, never assumed.""" + until = _NO_CREDIT.get(str(name)) + if not until: + return False + if time.time() >= until: + _NO_CREDIT.pop(str(name), None) + return False + return True + + +def clear_credit_memo(name=None): + """Forget one memo, or all of them. For a gate, and for an operator after a top-up.""" + if name is None: + _NO_CREDIT.clear() + else: + _NO_CREDIT.pop(str(name), None) + + +def llm_chain(capability="llm_tool_calling"): + """The provider order for `capability` — declaration first, credit memo second. + + Three filters, in this order, and each removes a DIFFERENT kind of row: + 1. `can()` — declared capable AND configured. An incapable row is never offered, so a + turn cannot be spent discovering it (the `chain()` rule, one layer up). + 2. `no_credit()` — R4's skip. A provider that told us its balance is empty is passed over + until the memo expires. + 3. the ORDER itself, overridable with `AIOS_LLM_ORDER` (`anthropic,groq`) so a deployment + can be moved off a vendor without a release — the same clause `AIOS_PROVIDER_ORDER` + carries for the scraping side. + """ + raw = (os.environ.get("AIOS_LLM_ORDER") or "").strip() + names = [x.strip() for x in raw.split(",") if x.strip()] or list(LLM_DEFAULT_ORDER) + return [LLM_PROVIDERS[n] for n in names + if n in LLM_PROVIDERS and LLM_PROVIDERS[n].can(capability) and not no_credit(n)] + + +#: What an HTTP status MEANS, in words a person can act on. ⛔⛔ R4's THIRD CLAUSE LIVES HERE AND +#: IT IS NOT COSMETIC: `HTTP 402` on a screen tells a reader nothing they can do, and the owner +#: quoted it back at us twice. Every sentence names the VENDOR and the ACTION. +_STATUS_WORDS = { + 401: "{label} would not accept our key", + 403: "{label} would not accept our key", + 402: "{label} is out of credit", + 404: "{label} does not offer the model we asked it for", + 408: "{label} took too long", + 413: "the question was too long for {label}", + 429: "{label} is rate limiting us right now", +} + +#: Substrings that mean "no money" on a wire that does not use 402. ⚠ Anthropic answers a spent +#: balance with a 400 or 403 carrying a message, not with a status code of its own, so this is the +#: one place a body has to be read. It is a LOWERCASE substring test on the vendor's own words and +#: it only ever decides whether to SKIP a provider, never whether to trust one. +_CREDIT_WORDS = ("credit balance", "insufficient credit", "insufficient_quota", "out of credit", + "quota exceeded", "billing", "payment required", "add credits") + + +def is_credit_failure(status, body=""): + """Did this response mean "the account is empty"? Status first, then the vendor's own words.""" + if int(status or 0) == 402: + return True + if int(status or 0) not in (400, 403, 429): + return False + return any(word in str(body or "").lower() for word in _CREDIT_WORDS) + + +def refusal_sentence(name, status, body=""): + """One provider's failure, as a SENTENCE. Never a bare status code, never a vendor stack trace. + + ⚠ THE BODY IS READ AND NEVER QUOTED. A provider's error body can carry an account id, a key + prefix or an internal trace; the only thing taken out of it is the yes/no answer to "is this a + credit problem", and what reaches the caller is this module's own wording. + """ + label = (LLM_PROVIDERS.get(str(name)) or LlmProvider(name, str(name), "", "", "", "")).label + if is_credit_failure(status, body): + return f"{label} is out of credit" + code = int(status or 0) + if code in _STATUS_WORDS: + return _STATUS_WORDS[code].format(label=label) + if 500 <= code <= 599: + return f"{label} is having trouble at their end" + return f"{label} did not answer" + + +# ═══════════ THE ANTHROPIC WIRE, ONCE (W36-T35 / ASK D-18, ruling R4) ═══════════════════════════ +# +# ⛔⛔ TWO DOORS IN THIS PRODUCT CALL ANTHROPIC AND THEY MUST NOT EACH LEARN THE MESSAGES API. +# `routes_query._call_model` (the Assistant and Query) and `ai_review.draft_flow` (the automation +# drafter) both need it, and the owner quoted an error from EACH of them in one breath: +# *"the assistant could not be reached just now (openrouter: HTTP 402)"* and *"anthropic: tool +# calls are not wired for this shape"*. Two implementations of one wire is +# [[one-question-two-normalizers]] before a line is written, so the wire lives here, beside the +# ladder that declares the rung. +# +# FOUR THINGS THE OPENAI-COMPATIBLE SHAPE GETS WRONG, each a 400 on its own: +# 1. the system prompt is a TOP-LEVEL field, not a `{"role": "system"}` message +# 2. a tool is `{name, description, input_schema}` FLAT, not nested under `function` +# 3. `temperature` and friends are REMOVED on the current model family +# 4. `tool_choice` is an OBJECT (`{"type": "auto"}` / `{"type": "any"}`), not a string +# +# ⚠ AND ONE THING THAT IS NOT A SHAPE: `effort` is model-gated. `output_config.effort` errors on +# Haiku 4.5, so it is a PARAMETER here and the caller decides — the drafter runs on haiku and omits +# it, the assistant runs on the Opus tier and sends it. + +#: The Messages API version header. A DATE that pins the WIRE FORMAT, never a model. +ANTHROPIC_VERSION = "2023-06-01" + + +def anthropic_request(*, model, key, system, messages, tools, max_tokens, + tool_choice="auto", effort=None): + """`{url, headers, json}` for one Messages API call. Pure: reads no environment, sends nothing. + + `messages` is the OpenAI-shaped list this product already builds; the `system` turns are lifted + out of it, because that is where this API wants them. `tools` is the OpenAI-shaped tool list, + re-addressed rather than re-derived, so a schema change happens in one place. + """ + system_text = "\n\n".join(str(m.get("content") or "") for m in messages + if m.get("role") == "system") + if system: + system_text = (system_text + "\n\n" + str(system)).strip() if system_text else str(system) + turns = [{"role": ("assistant" if m.get("role") == "assistant" else "user"), + "content": str(m.get("content") or "")} + for m in messages if m.get("role") != "system" and str(m.get("content") or "").strip()] + wire_tools = [{"name": t["function"]["name"], + "description": t["function"]["description"], + "input_schema": t["function"]["parameters"]} for t in (tools or [])] + body = { + "model": str(model), + "max_tokens": int(max_tokens), + "system": system_text, + "messages": turns, + "tools": wire_tools, + } + # ⛔ W37-T39: `tool_choice` IS OMITTED WHEN THERE ARE NO TOOLS, and the builder decides that + # rather than each caller. The Messages API refuses `tool_choice` beside an empty `tools` list, + # so this used to be a `req["json"].pop("tool_choice", None)` at the one call site that sends no + # tools — a rule living outside the thing it constrains, which is [[limit-with-no-enforcer]]: + # the next no-tools caller inherits a 400 that reads like a model problem, not a shape problem. + # ⚠ Safe for all three callers today, checked rather than assumed: `routes_query` and + # `ai_review.draft_flow` both pass a non-empty tool list and keep the key exactly as before. + if wire_tools: + body["tool_choice"] = {"type": "any" if tool_choice in ("required", "any") else "auto"} + if effort: + body["output_config"] = {"effort": str(effort)} + return {"url": LLM_PROVIDERS["anthropic"].url, + "headers": {"x-api-key": str(key), + "anthropic-version": ANTHROPIC_VERSION, + "content-type": "application/json"}, + "json": body} + + +#: Cached result of `import anthropic`: the module, or `False` once it is known to be absent. +#: ⚠ `None` is NOT the "absent" sentinel — a plain falsy check would re-attempt the import on every +#: call, and a failed import is not cheap. `False` says "asked and answered". +_SDK: object = None + + +def anthropic_sdk(): + """The official `anthropic` SDK, or `None` if this deployment does not carry it. + + ⭐⭐ D-346, W37-T39. THIS FUNCTION IS THE WHOLE OF THE FIX AND ALSO THE WHOLE OF ITS RISK. + `aios-web/requirements.txt` is what the Dockerfile installs and it is OUTSIDE every lane's + fence this wave, so the pin is another session's edit. A hard `import anthropic` at module + scope would therefore turn a missing line in a manifest into a container that cannot boot at + all: the API imports this module on every request path. + + So the import is LAZY and its absence is REPORTED rather than fatal — `anthropic_send` falls + back to the raw `requests` POST that has always worked, and says which transport ran. When the + pin lands the SDK path takes over with no further change. + ⛔ The fallback is a BRIDGE, not a second implementation: both transports send the SAME body + `anthropic_request()` built and return the SAME `(status, body)` pair, so `anthropic_read`, + `is_credit_failure` and `refusal_sentence` each keep exactly one normalizer + ([[one-question-two-normalizers]]). + """ + global _SDK + if _SDK is None: + try: + import anthropic as _mod # noqa: PLC0415 - deliberately lazy; see the docstring + _SDK = _mod + except Exception: + _SDK = False + return _SDK or None + + +def anthropic_send(req, timeout=None, use_sdk=None): + """POST one `anthropic_request()` dict. Returns `(status, body, transport)`. + + ⭐ `transport` is `'sdk'` or `'http'` and it is RETURNED, not logged and forgotten: a reader + who cannot tell which path ran cannot tell whether the requirements pin reached the container + [[report-the-cause-before-you-fix-it]]. ⛔ A gate must assert on THIS RETURN VALUE, never on + `ai_review.LAST_ANTHROPIC_TRANSPORT` — that global is a convenience for `GET /meta`, and a + check keyed to it passes on a stale value written by an earlier check in the same process. + + `status` is an int and `body` is the parsed JSON dict in BOTH paths, including on an error — + the SDK raises where `requests` returns, and normalising that difference here is the only + reason this function exists rather than the caller branching on transport. + + ⚠ `use_sdk=False` forces the fallback. It exists so a gate can exercise BOTH transports + without assigning to `_SDK`: a test that mutates a module global and then throws leaves every + later check in that process running on the wrong path and passing for the wrong reason. + """ + sdk = None if use_sdk is False else anthropic_sdk() + payload = dict(req.get("json") or {}) + if sdk is None: + r = requests.post(req["url"], headers=req["headers"], json=payload, + timeout=(timeout or 60)) + try: + return r.status_code, (r.json() if r.content else {}), "http" + except Exception: + return r.status_code, {"_text": (r.text or "")[:2000]}, "http" + + key = (req.get("headers") or {}).get("x-api-key") or "" + try: + client = sdk.Anthropic(api_key=key, timeout=float(timeout or 60)) + msg = client.messages.create(**payload) + # ⚠ `.model_dump()` is what makes ONE reader serve both transports: it hands back the same + # wire-shaped dict the raw POST parses out of the response body. + return 200, msg.model_dump(), "sdk" + except Exception as exc: + # ⛔ THE STATUS IS THE PRODUCT HERE. Every sentence the reader sees is chosen by + # `refusal_sentence(status)`, so an exception that loses its code turns "Anthropic is out + # of credit" into "Anthropic did not answer" — the exact regression R4 was written to end. + status = int(getattr(exc, "status_code", 0) or 0) + body = getattr(exc, "body", None) + if not isinstance(body, dict): + body = {"_text": str(exc)[:2000]} + if not status: + # No status at all = it never reached the vendor (DNS, TLS, timeout). 408 is the one + # code `_STATUS_WORDS` already words as a reachability problem rather than a refusal. + status = 408 if isinstance(exc, getattr(sdk, "APIConnectionError", ())) else 0 + return status, body, "sdk" + + +def anthropic_read(body): + """`(text, tool_input, refusal)` out of a Messages API answer. + + ⛔ `stop_reason` IS CHECKED BEFORE `content` IS READ. A safety decline answers **HTTP 200** with + `stop_reason: "refusal"` and an empty or partial `content`, so code that indexes `content[0]` + unconditionally breaks on exactly the turn a person most needs explained. + ⭐ AND THE TOOL CALL ARRIVES PARSED. `tool_use.input` is already a dict — no `json.loads`, and + no regex recovering a call out of a 400, which is what the other wire needs. + """ + body = body if isinstance(body, dict) else {} + if str(body.get("stop_reason") or "") == "refusal": + return "", None, "the assistant declined to answer that one" + blocks = [b for b in (body.get("content") or []) if isinstance(b, dict)] + text = " ".join(str(b.get("text") or "") for b in blocks if b.get("type") == "text").strip() + calls = [b for b in blocks if b.get("type") == "tool_use"] + got = calls[0].get("input") if calls else None + return text, (got if isinstance(got, dict) else None), None + + +def llm_status(capability="llm_tool_calling"): + """Per provider: configured, declared-capable, in cooldown, and WHY — contract C5's payload. + + ⭐ ONE LIST, ONE DOOR. The Assistant's model picker and the Agent chat's toggle (W36-T34) read + THIS, so a model offered in one place cannot be missing from the other, and neither can offer a + provider the ladder would refuse to call [[permitted-is-not-answerable]]. + """ + rows = [] + for name in LLM_DEFAULT_ORDER: + p = LLM_PROVIDERS[name] + cap = p.caps.get(capability) + rows.append({ + "provider": p.name, "label": p.label, "model": p.model, "wire": p.wire, + "configured": p.configured(), + "toolCalling": bool((p.caps.get("llm_tool_calling") or Capability(False)).capable), + "jsonMode": bool((p.caps.get("llm_json_mode") or Capability(False)).capable), + "capable": bool(cap and cap.capable), + "outOfCredit": no_credit(name), + "note": (cap.note if cap else ""), + }) + return rows + + +# ============================================================================================= +# THE CANONICAL SCHEMA — owner ruling 2026-08-08: +# *"standardize the schema between Bright Data and APIfy so we keep using the same pre-set +# database even if the underlying engine changes"* +# +# ⛔ THE PRESET TABLES ARE THE CONTRACT; A VENDOR IS AN IMPLEMENTATION DETAIL. `ut_ig_posts` and +# `ut_ig_post_snapshots` must not gain, lose or rename a column because a chain was reordered — a +# tenant's saved views, filters, rollups and forms all bind to these keys, and a schema that moves +# with the vendor turns a routing change into a data migration. +# +# So every provider normalises INTO the keys below and nothing reads a vendor row downstream. +# `verify_automation` asserts both normalisers emit exactly `CANONICAL_POST_KEYS` on a fixture, so +# a third provider cannot ship with a near-miss key like `viewCount` and silently write a column +# nobody declared. +# +# ⚠ ONE NAME PER MEASUREMENT, AND `views` IS THE MEASUREMENT INSTAGRAM DISPLAYS. Meta folded +# Impressions/Plays/Video Views into a single **Views** metric on 2025-04-10, so carrying both a +# `views` and a `plays` column would be modelling a distinction the platform deleted — and it is +# exactly the distinction that let an account-grain number wear the "Views" label for a month. +# ⛔ THE VENDOR KEY THAT LOOKS RIGHT IS THE WRONG ONE, ON BOTH VENDORS. Apify ships BOTH +# `videoViewCount` (10,678) and `videoPlayCount` (137,684) for one reel whose true displayed count +# is ~137K — and Bright Data's useless `views` is 10,638 for that same reel. The two vendors' junk +# fields AGREE with each other, which is precisely what makes picking by name so dangerous. +# canonical `views` <- apify `videoPlayCount` ✅ matches the grid +# canonical `views` <- apify `videoViewCount` ⛔ off by 13x +# canonical `views` <- brightdata `views` ⛔ off by 13x AND account-grain +# ============================================================================================= + +#: Every key a normalised POST row may carry. Absent > blank: a key is omitted when the provider +#: did not answer, because `upsert_rows` merges and an empty string would ERASE what an earlier +#: paid run learned. +CANONICAL_POST_KEYS = ( + "shortcode", "url", "influencer_key", "posted_at", "type", "caption", + "likes", "comments", "views", "paid_partnership", "partner", "hashtags", + "alt_text", "tagged_location", "source_payload", + # ⭐ 2026-08-09 — the three the SECOND provider answers and the first does not. Each has a + # matching `field_def` in the engine's POST_FIELDS; a key here without a column there is a + # value that normalises cleanly and is then dropped by the write door, silently. + "plays", "video_duration", "comments_disabled", +) + + +def _int_or_none(v): + """`-1` IS NOT A COUNT. Apify returns `likesCount: -1` when the creator HIDES their like + count — a real state that is not a measurement. Writing -1 would render as a negative like + count; writing 0 would claim nobody liked it. Both are lies, so the key is omitted.""" + try: + n = int(v) + except (TypeError, ValueError): + return None + return None if n < 0 else n + + +def normalize_post_apify(row): + """One Apify `instagram-scraper` item -> the canonical post row.""" + if not isinstance(row, dict): + return None + code = str(row.get("shortCode") or "").strip() + if not code: + return None + out = { + "shortcode": code, + "url": str(row.get("url") or f"https://www.instagram.com/reel/{code}/"), + "influencer_key": str(row.get("ownerUsername") or ""), + "posted_at": str(row.get("timestamp") or "").replace("T", " ")[:16], + "type": "video" if str(row.get("type") or "").lower() == "video" else + ("carousel" if row.get("childPosts") else "image"), + "caption": str(row.get("caption") or ""), + # ⭐ THE FIELD THIS WHOLE PROVIDER EXISTS FOR. `videoPlayCount`, never `videoViewCount`: + # the wrong one agreed with Bright Data's junk (10,678 vs 10,638) on a reel whose true + # count was ~137K, and `videoPlayCount` matched a browser read to the digit. + "views": _int_or_none(row.get("videoPlayCount")), + # ⭐⭐ 2026-08-09 (owner: *"Video plays (# Plays) field isn't in APIfy? i believe it is"*). + # They were right, and the `plays` COLUMN had been empty on all 816 rows because nothing + # ever wrote it — the field existed with no writer. MEASURED on two of their own reels: + # `videoPlayCount` 216,904 / 95,331 and `videoViewCount` 115,929 / 30,871. Two different + # real numbers, and we were storing only one of them. + # ⚠ `plays` TAKES THE PLAY COUNT, which is also what `views` carries today — so the two + # columns will agree until somebody decides otherwise, and that decision is the OWNER'S: + # re-sourcing `views` to `videoViewCount` would change what the ~480 rows already captured + # mean, and a column whose meaning changes halfway down is the one thing worse than a + # column with no data. Flagged rather than done. + "plays": _int_or_none(row.get("videoPlayCount")), + "likes": _int_or_none(row.get("likesCount")), + "comments": _int_or_none(row.get("commentsCount")), + # ⭐ FIELDS BRIGHT DATA DOES NOT RETURN AT ALL, kept because they are already paid for in + # this same response (owner: *"whatever APIfy has more than BD pls use it"*). + "video_duration": _int_or_none(row.get("videoDuration")), + "comments_disabled": "1" if row.get("isCommentsDisabled") else "", + "hashtags": ", ".join(str(h) for h in (row.get("hashtags") or []) if h), + "alt_text": str(row.get("alt") or ""), + "paid_partnership": "1" if row.get("paidPartnership") else "", + "partner": ", ".join(str((s or {}).get("username") or "") + for s in (row.get("sponsors") or []) if isinstance(s, dict)), + "tagged_location": str((row.get("locationName") or "")), + "source_payload": json.dumps(row, default=str)[:32_000_000], + } + return {k: v for k, v in out.items() if v not in (None, "")} + + +def normalize_profile_apify(row): + """One Apify `instagram-scraper` PROFILE item (`resultsType: "details"`) -> our snapshot shape. + + ⭐ WHY THIS EXISTS (owner report 2026-08-09: *"I need Bright Data and Apify to work correctly + in tandem"*). `PROVIDERS["apify"]` has declared `ig_profile` capable since 2026-08-08 and + `DEFAULT_CHAINS["ig_profile"]` has read `("brightdata", "apify")` — but **nothing ever ran + that chain.** `connectors_ig.pull_profile` went Bright Data -> anonymous HTML rungs and Apify + was never asked, so a profile Bright Data cannot scrape came back `blocked` while a + configured, declared-capable provider sat unused. A registry entry with no runner is a + promise the product does not keep ([[flag-shipped-without-its-writer]]). + + ⚠ THE KEYS ARE APIFY'S camelCase, and mapping them HERE is the boundary rule this module + already enforces for posts: nothing downstream may ever see a vendor-shaped key, so the + snapshot writer cannot tell which vendor answered — which is what makes the fallback + invisible to every consumer instead of a second schema. + + ⛔ OMIT, NEVER ZERO. A field Apify did not send is dropped, exactly as `normalize_post_apify` + drops a missing count: writing 0 followers would claim we measured an empty account, and the + caller's `satisfied` hook reads absence as "did not answer" and falls through. A zero would + stop the chain on a lie. + """ + if not isinstance(row, dict): + return None + # ⛔ AN ERROR ENVELOPE IS NOT A PROFILE (measured 2026-08-09). Apify answers a dead handle + # with a 200 and a well-formed item — `{"username": …, "error": "not_found", + # "errorDescription": "Post does not exist"}` — and this function used to build a "profile" + # out of it: a username, a url and a source_payload, with every measured field absent. It + # then read to the caller as a vendor that answered, so the reason was replaced by a shrug. + # Refused HERE as well as in `connectors_ig.apify_profile` on purpose: the boundary rule this + # module exists for is that nothing downstream ever sees a vendor-shaped key, and a vendor's + # ERROR shape is the one that must never become a row. + # ⚠ TRUTHINESS, NOT KEY PRESENCE — a successful item carries `error: null` (measured on + # `sriyynntt`), so `"error" in row` would refuse every good profile. + if row.get("error"): + return None + handle = str(row.get("username") or "").strip() + if not handle: + return None + out = { + "username": handle, + "full_name": str(row.get("fullName") or ""), + "bio": str(row.get("biography") or ""), + "followers": _int_or_none(row.get("followersCount")), + "following": _int_or_none(row.get("followsCount")), + "posts_count": _int_or_none(row.get("postsCount")), + "verified": "1" if row.get("verified") else "", + "external_url": str(row.get("externalUrl") or ""), + "ig_id": str(row.get("id") or ""), + "profile_url": str(row.get("url") or f"https://www.instagram.com/{handle}/"), + "business_category": str(row.get("businessCategoryName") or ""), + "is_business": "1" if row.get("isBusinessAccount") else "", + "is_private": "1" if row.get("private") else "", + "highlights_count": _int_or_none(row.get("highlightReelCount")), + "source_payload": json.dumps(row, default=str)[:32_000_000], + } + # ⚠ `followers`/`following` are ints and 0 is a LEGITIMATE value for them, so the filter + # below must not treat 0 as absent the way the post normaliser can — an account really can + # have zero followers. Only None and "" are dropped. + return {k: v for k, v in out.items() if v is not None and v != ""} + + +def canonical_gaps(row, want=("views",)): + """Which requested canonical fields this row does NOT carry — the `satisfied` input. + + Named rather than inlined because "did the vendor actually answer the question" is the whole + fallback trigger, and a chain that asks it differently in two places will drift. + """ + row = row if isinstance(row, dict) else {} + return [k for k in want if row.get(k) in (None, "")]