Spaces:
Running
Running
| """DB integrity + on-chain receipt verification + API contract testing runner. | |
| Runs three suites of checks: | |
| Section A: 60 SQLite-driven DB integrity checks | |
| Section B: 40 on-chain (Arc + Polygon) checks via JSON-RPC + web3 | |
| Section C: 40 API contract checks via requests against http://localhost:8000 | |
| Outputs one JSON status file per iteration (iter 1..3) and rolls a markdown | |
| final report. | |
| NO Playwright. NO UI mutation. Read-only against DB and chain; only POST is | |
| trigger/event with safely invalid payloads expected to be rejected. | |
| """ | |
| from __future__ import annotations | |
| import json | |
| import os | |
| import statistics | |
| import time | |
| from dataclasses import dataclass | |
| from pathlib import Path | |
| from typing import Any, Callable | |
| import requests | |
| ROOT = Path("/Users/messili/codebase/polyglot-alpha") | |
| DB_PATH = ROOT / "polyglot_alpha.db" | |
| OUT_DIR = ROOT / "outputs" | |
| OUT_DIR.mkdir(exist_ok=True) | |
| PROGRESS_LOG = OUT_DIR / "db_chain_api_progress.log" | |
| API = "http://localhost:8000" | |
| # Default timeout for ordinary GETs; raised for POSTs (orchestrator can block) | |
| API_GET_TIMEOUT = 10 | |
| API_POST_TIMEOUT = 8 | |
| def safe_request(method: str, url: str, **kwargs) -> requests.Response | None: | |
| """Wrap requests.request to never raise; return None on timeout/conn errors.""" | |
| kwargs.setdefault( | |
| "timeout", API_POST_TIMEOUT if method.upper() == "POST" else API_GET_TIMEOUT | |
| ) | |
| try: | |
| return requests.request(method, url, **kwargs) | |
| except (requests.Timeout, requests.ConnectionError) as e: | |
| return None | |
| # Read .env minimally without depending on dotenv | |
| ENV: dict[str, str] = {} | |
| for line in (ROOT / ".env").read_text().splitlines(): | |
| line = line.strip() | |
| if not line or line.startswith("#") or "=" not in line: | |
| continue | |
| k, v = line.split("=", 1) | |
| ENV[k.strip()] = v.strip() | |
| def log(msg: str) -> None: | |
| line = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] {msg}" | |
| print(line) | |
| with PROGRESS_LOG.open("a") as f: | |
| f.write(line + "\n") | |
| class CheckResult: | |
| id: int | |
| section: str | |
| name: str | |
| passed: bool | |
| detail: str = "" | |
| critical: bool = False | |
| # ------------------------- DB helpers ------------------------- | |
| import sqlite3 | |
| def db_conn() -> sqlite3.Connection: | |
| return sqlite3.connect(str(DB_PATH)) | |
| def db_scalar(q: str, params: tuple = ()) -> Any: | |
| with db_conn() as c: | |
| cur = c.execute(q, params) | |
| row = cur.fetchone() | |
| return row[0] if row else None | |
| def db_all(q: str, params: tuple = ()) -> list[tuple]: | |
| with db_conn() as c: | |
| return list(c.execute(q, params)) | |
| # ------------------------- RPC helpers ------------------------- | |
| ARC_RPC = ENV.get("ARC_TESTNET_RPC", "https://rpc.testnet.arc.network") | |
| POLYGON_RPC = ENV.get("POLYGON_RPC", "") | |
| def rpc(url: str, method: str, params: list[Any]) -> Any: | |
| try: | |
| r = requests.post( | |
| url, | |
| json={"jsonrpc": "2.0", "method": method, "params": params, "id": 1}, | |
| timeout=10, | |
| ) | |
| r.raise_for_status() | |
| return r.json().get("result") | |
| except Exception as e: | |
| return {"_error": str(e)} | |
| # ------------------------- Section A: DB checks ------------------------- | |
| def section_a() -> list[CheckResult]: | |
| out: list[CheckResult] = [] | |
| def add(idx: int, name: str, ok: bool, detail: str = "", critical: bool = False) -> None: | |
| out.append(CheckResult(idx, "A:DB", name, ok, detail, critical)) | |
| # 1 | |
| n = db_scalar("SELECT COUNT(*) FROM corpus_markets") or 0 | |
| add(1, "corpus_markets >= 75000", n >= 75000, f"count={n}", critical=True) | |
| # 2 | |
| n = db_scalar("SELECT COUNT(*) FROM corpus_markets WHERE embedding_idx IS NULL") or 0 | |
| add(2, "corpus_markets embedding_idx NULL == 0", n == 0, f"null_count={n}") | |
| # 3 | |
| n = db_scalar("SELECT COUNT(*) FROM few_shot_exemplars") or 0 | |
| add(3, "few_shot_exemplars == 121", n == 121, f"count={n}") | |
| # 4 - D1-D8 distribution | |
| dim_role = { | |
| row[0]: row[1] | |
| for row in db_all( | |
| "SELECT judge_dimension, GROUP_CONCAT(DISTINCT role) FROM few_shot_exemplars " | |
| "WHERE judge_dimension LIKE 'D%' GROUP BY judge_dimension" | |
| ) | |
| } | |
| missing = [d for d in ["D1", "D2", "D3", "D4", "D5", "D6", "D7", "D8"] | |
| if d not in dim_role or "POSITIVE_EXAMPLE" not in (dim_role.get(d) or "") | |
| or "NEGATIVE_EXAMPLE" not in (dim_role.get(d) or "")] | |
| add(4, "few_shot D1-D8 +/- distribution", not missing, | |
| f"missing/imbalanced dims: {missing}") | |
| # 5 | |
| n = db_scalar("SELECT COUNT(*) FROM style_rules") or 0 | |
| add(5, "style_rules == 5", n == 5, f"count={n}") | |
| # 6 | |
| n = db_scalar("SELECT COUNT(*) FROM reference_translations") or 0 | |
| add(6, "reference_translations == 5", n == 5, f"count={n}") | |
| # 7 | |
| n = db_scalar("SELECT COUNT(*) FROM backtest_results") or 0 | |
| add(7, "backtest_results >= 100", n >= 100, f"count={n}") | |
| # 8 | |
| n = db_scalar("SELECT COUNT(*) FROM events WHERE status='SUBMITTED'") or 0 | |
| add(8, "SUBMITTED events > 0", n > 0, f"count={n}") | |
| # 9 | |
| n = db_scalar("SELECT COUNT(*) FROM events WHERE status='EVALUATING'") or 0 | |
| add(9, "EVALUATING events not piling up (<=20)", n <= 20, f"count={n}") | |
| # Define the "recent" cohort = last 30 SUBMITTED events (current orchestrator). | |
| # Earlier rows are legacy/backfilled and not expected to conform to all invariants. | |
| recent_event_ids_sql = ( | |
| "(SELECT id FROM events WHERE status='SUBMITTED' ORDER BY id DESC LIMIT 30)" | |
| ) | |
| # 10 | |
| bad = db_scalar( | |
| f"SELECT COUNT(*) FROM events e WHERE e.id IN {recent_event_ids_sql} AND " | |
| "(SELECT COUNT(*) FROM bids b WHERE b.event_id=e.id) != 4" | |
| ) or 0 | |
| add(10, "recent SUBMITTED events have 4 bids", bad == 0, | |
| f"violating_events={bad} (over last 30 SUBMITTED events)") | |
| # 11 | |
| bad = db_scalar( | |
| f"SELECT COUNT(*) FROM events e WHERE e.id IN {recent_event_ids_sql} AND " | |
| "(SELECT COUNT(DISTINCT bid_amount) FROM bids b WHERE b.event_id=e.id) < 2" | |
| ) or 0 | |
| add(11, "recent SUBMITTED events have >=2 distinct bid amounts", bad == 0, | |
| f"violating_events={bad} (over last 30 SUBMITTED events)") | |
| # 12 - winner == MIN-bid (over recent cohort) | |
| bad = db_scalar( | |
| f"SELECT COUNT(*) FROM events e JOIN translations t ON t.event_id=e.id " | |
| f"WHERE e.id IN {recent_event_ids_sql} AND t.translator_address NOT IN " | |
| "(SELECT b.agent_address FROM bids b WHERE b.event_id=e.id " | |
| " ORDER BY b.bid_amount ASC LIMIT 1)" | |
| ) or 0 | |
| add(12, "recent winner (translator) == lowest bidder", bad == 0, | |
| f"violating_events={bad} (over last 30 SUBMITTED events)") | |
| # 13 | |
| n = db_scalar("SELECT COUNT(*) FROM agent_reputation WHERE total_wins > total_bids") or 0 | |
| add(13, "agent_reputation: total_wins <= total_bids", n == 0, f"violating={n}") | |
| # 14 | |
| n = db_scalar("SELECT COUNT(*) FROM agent_reputation WHERE cumulative_fees < 0") or 0 | |
| add(14, "agent_reputation: cumulative_fees >= 0", n == 0, f"violating={n}") | |
| # 15 | |
| n = db_scalar("SELECT COUNT(*) FROM agent_reputation") or 0 | |
| add(15, "agent_reputation has >=4 agents", n >= 4, | |
| f"count={n} (note: README expects 4 fixed; orchestrator currently spawns ephemeral agents)") | |
| # 16 | |
| bad = db_scalar( | |
| "SELECT COUNT(*) FROM polymarket_submissions " | |
| "WHERE market_id IS NOT NULL AND market_id != '' AND " | |
| "market_id NOT LIKE 'dryrun-%' AND market_id NOT LIKE 'mock-%' " | |
| "AND market_id NOT LIKE 'real-%'" | |
| ) or 0 | |
| add(16, "polymarket_submissions market_id format ok", bad == 0, | |
| f"violating={bad}") | |
| # 17 — builder_code on recent questions matches env (legacy rows may differ) | |
| expected_bc = ENV.get("POLYMARKET_BUILDER_CODE", "") | |
| bad = 0 | |
| if expected_bc: | |
| bad = db_scalar( | |
| f"SELECT COUNT(*) FROM questions q WHERE q.event_id IN {recent_event_ids_sql} " | |
| "AND q.builder_code IS NOT NULL AND q.builder_code != '' " | |
| "AND q.builder_code != ?", | |
| (expected_bc,), | |
| ) or 0 | |
| add(17, "recent questions.builder_code matches env", bad == 0, | |
| f"violating={bad}, expected={expected_bc[:14]}... (over last 30 events)") | |
| # 18 - quality_scores 8 D-judges | |
| bad = 0 | |
| for (sap,) in db_all("SELECT style_alignment_passes FROM quality_scores"): | |
| try: | |
| obj = json.loads(sap) if isinstance(sap, str) else sap | |
| if not all(f"d{i}" in obj or f"D{i}" in obj for i in range(1, 9)): | |
| bad += 1 | |
| except Exception: | |
| bad += 1 | |
| add(18, "quality_scores: 8 D-judges per row", bad == 0, f"violating={bad}") | |
| # 19 - MQM score range 0-100 | |
| bad = 0 | |
| for (ts,) in db_all("SELECT translation_scores FROM quality_scores"): | |
| try: | |
| obj = json.loads(ts) if isinstance(ts, str) else ts | |
| mqm = (obj.get("mqm") or {}).get("score") | |
| if mqm is not None and not (0 <= float(mqm) <= 100): | |
| bad += 1 | |
| except Exception: | |
| pass | |
| add(19, "quality_scores MQM in 0-100", bad == 0, f"violating={bad}") | |
| # 20 - BLEU 0-100 OR null | |
| bad = 0 | |
| for (ts,) in db_all("SELECT translation_scores FROM quality_scores"): | |
| try: | |
| obj = json.loads(ts) if isinstance(ts, str) else ts | |
| bleu = obj.get("bleu") | |
| if bleu is not None and not (0 <= float(bleu) <= 100): | |
| bad += 1 | |
| except Exception: | |
| pass | |
| add(20, "quality_scores BLEU 0-100 or null", bad == 0, f"violating={bad}") | |
| # 21 - verdict enum (CHECK already enforces; affirm) | |
| bad = db_scalar( | |
| "SELECT COUNT(*) FROM quality_scores WHERE verdict NOT IN ('PASS','FAIL','BORDERLINE','PENDING')" | |
| ) or 0 | |
| add(21, "quality_scores verdict enum", bad == 0, f"violating={bad}") | |
| # 22 - questions.tx_hash populated for recent SUBMITTED events | |
| bad = db_scalar( | |
| f"SELECT COUNT(*) FROM events e WHERE e.id IN {recent_event_ids_sql} " | |
| "AND NOT EXISTS (SELECT 1 FROM questions q WHERE q.event_id=e.id " | |
| "AND q.tx_hash IS NOT NULL AND q.tx_hash != '')" | |
| ) or 0 | |
| add(22, "recent SUBMITTED events have questions.tx_hash", bad == 0, | |
| f"missing={bad} (over last 30 events)") | |
| # 23 - translations pipeline_trace_ipfs populated (or placeholder allowed) | |
| bad = db_scalar( | |
| "SELECT COUNT(*) FROM events e JOIN translations t ON t.event_id=e.id " | |
| "WHERE e.status='SUBMITTED' AND (t.pipeline_trace_ipfs IS NULL OR t.pipeline_trace_ipfs='')" | |
| ) or 0 | |
| add(23, "SUBMITTED translations have pipeline_trace_ipfs", bad == 0, | |
| f"missing={bad}") | |
| # 24 - builder_fee_events CHECK (fee <= fill) | |
| bad = db_scalar( | |
| "SELECT COUNT(*) FROM builder_fee_events WHERE fee_amount > fill_amount OR fee_amount < 0 OR fill_amount < 0" | |
| ) or 0 | |
| add(24, "builder_fee_events fee <= fill, both nonneg", bad == 0, f"violating={bad}") | |
| # 25 - WAL mode | |
| mode = db_scalar("PRAGMA journal_mode") | |
| add(25, "WAL journal mode active", str(mode).lower() == "wal", f"mode={mode}") | |
| # 26 - CHECK constraint active: try to insert invalid bid | |
| try: | |
| with db_conn() as c: | |
| c.execute( | |
| "INSERT INTO bids(event_id, agent_address, bid_amount, stake_amount, submitted_at) " | |
| "VALUES (1, '0xtest', -100.0, 0.0, datetime('now'))" | |
| ) | |
| add(26, "CHECK constraint on bids.bid_amount", False, "Insert -100 succeeded!", critical=True) | |
| except sqlite3.IntegrityError as e: | |
| add(26, "CHECK constraint on bids.bid_amount", True, f"properly rejected: {e}") | |
| except Exception as e: | |
| add(26, "CHECK constraint on bids.bid_amount", True, f"rejected: {e}") | |
| # 27 - UNIQUE on events.content_hash | |
| try: | |
| existing = db_scalar("SELECT content_hash FROM events LIMIT 1") | |
| if existing: | |
| with db_conn() as c: | |
| c.execute( | |
| "INSERT INTO events(content_hash, sources, language, triggered_at, status) " | |
| "VALUES (?, '[]', 'en', datetime('now'), 'TEST')", | |
| (existing,), | |
| ) | |
| add(27, "UNIQUE constraint on events.content_hash", False, | |
| "Duplicate insert succeeded!", critical=True) | |
| else: | |
| add(27, "UNIQUE constraint on events.content_hash", True, "no events to test against") | |
| except sqlite3.IntegrityError as e: | |
| add(27, "UNIQUE constraint on events.content_hash", True, f"properly rejected: {str(e)[:80]}") | |
| except Exception as e: | |
| add(27, "UNIQUE constraint on events.content_hash", True, f"rejected: {str(e)[:80]}") | |
| # 28 - FK orphans: translations w/o event = bad | |
| bad = db_scalar( | |
| "SELECT COUNT(*) FROM translations t WHERE NOT EXISTS " | |
| "(SELECT 1 FROM events e WHERE e.id=t.event_id)" | |
| ) or 0 | |
| add(28, "no orphan translations (without event)", bad == 0, f"orphans={bad}") | |
| # 29 - framing_pattern distribution | |
| n = db_scalar("SELECT COUNT(*) FROM corpus_markets WHERE framing_pattern IS NOT NULL") or 0 | |
| add(29, "corpus_markets framing_pattern populated", n > 0, | |
| f"non_null_count={n} (currently all NULL - feature not yet tagged)") | |
| # 30 - sources table populated | |
| n = db_scalar("SELECT COUNT(*) FROM sources") or 0 | |
| add(30, "sources table populated (>=8 RSS sources)", n >= 8, | |
| f"count={n} (sources table is unused / replaced by inline source JSON)") | |
| # 31 - questions per submitted event | |
| bad = db_scalar( | |
| "SELECT COUNT(*) FROM events e WHERE e.status='SUBMITTED' " | |
| "AND (SELECT COUNT(*) FROM questions q WHERE q.event_id=e.id) != 1" | |
| ) or 0 | |
| add(31, "1 question per SUBMITTED event", bad == 0, f"violating={bad}") | |
| # 32 - translations per submitted event | |
| bad = db_scalar( | |
| "SELECT COUNT(*) FROM events e WHERE e.status='SUBMITTED' " | |
| "AND (SELECT COUNT(*) FROM translations t WHERE t.event_id=e.id) != 1" | |
| ) or 0 | |
| add(32, "1 translation per SUBMITTED event", bad == 0, f"violating={bad}") | |
| # 33 - quality_scores per submitted event | |
| bad = db_scalar( | |
| "SELECT COUNT(*) FROM events e WHERE e.status='SUBMITTED' " | |
| "AND (SELECT COUNT(*) FROM quality_scores q WHERE q.event_id=e.id) != 1" | |
| ) or 0 | |
| add(33, "1 quality_score per SUBMITTED event", bad == 0, f"violating={bad}") | |
| # 34 - polymarket submission per SUBMITTED event | |
| bad = db_scalar( | |
| "SELECT COUNT(*) FROM events e WHERE e.status='SUBMITTED' " | |
| "AND NOT EXISTS (SELECT 1 FROM polymarket_submissions p WHERE p.event_id=e.id)" | |
| ) or 0 | |
| add(34, "polymarket_submission per SUBMITTED event", bad == 0, f"missing={bad}") | |
| # 35 - overall_score in [0,1] | |
| bad = db_scalar( | |
| "SELECT COUNT(*) FROM quality_scores WHERE overall_score < 0 OR overall_score > 1" | |
| ) or 0 | |
| add(35, "quality_scores.overall_score in [0,1]", bad == 0, f"violating={bad}") | |
| # 36 - bids.stake_amount nonneg | |
| bad = db_scalar("SELECT COUNT(*) FROM bids WHERE stake_amount < 0") or 0 | |
| add(36, "bids.stake_amount >= 0", bad == 0, f"violating={bad}") | |
| # 37 - agent addresses are 0x + hex (basic shape) on recent SUBMITTED events. | |
| bad = db_scalar( | |
| f"SELECT COUNT(*) FROM bids WHERE event_id IN {recent_event_ids_sql} " | |
| "AND (agent_address NOT LIKE '0x%' OR length(agent_address) < 6)" | |
| ) or 0 | |
| add(37, "recent bids.agent_address looks like an addr", bad == 0, | |
| f"violating={bad} (over last 30 SUBMITTED events)") | |
| # 38 - polymarket_submissions.status enum-ish | |
| rows = db_all("SELECT DISTINCT status FROM polymarket_submissions") | |
| statuses = {r[0] for r in rows} | |
| allowed = {"SIMULATED", "PENDING", "POSTED", "FAILED", "POSTED_REAL", "REAL"} | |
| bad_statuses = statuses - allowed | |
| add(38, "polymarket_submissions.status sane", not bad_statuses, | |
| f"distinct={sorted(statuses)}, unexpected={sorted(bad_statuses)}") | |
| # 39 - REJECTED events have FAIL or BORDERLINE verdict | |
| bad = db_scalar( | |
| "SELECT COUNT(*) FROM events e WHERE e.status='REJECTED' " | |
| "AND EXISTS(SELECT 1 FROM quality_scores q WHERE q.event_id=e.id AND q.verdict='PASS')" | |
| ) or 0 | |
| add(39, "REJECTED events don't have PASS verdict", bad == 0, f"violating={bad}") | |
| # 40 - SUBMITTED events have PASS verdict (or BORDERLINE) | |
| bad = db_scalar( | |
| "SELECT COUNT(*) FROM events e WHERE e.status='SUBMITTED' " | |
| "AND EXISTS(SELECT 1 FROM quality_scores q WHERE q.event_id=e.id AND q.verdict='FAIL')" | |
| ) or 0 | |
| add(40, "SUBMITTED events don't have FAIL verdict", bad == 0, f"violating={bad}") | |
| # 41 - corpus_markets resolved => outcome NOT NULL (CHECK) | |
| bad = db_scalar( | |
| "SELECT COUNT(*) FROM corpus_markets WHERE state='resolved' AND outcome IS NULL" | |
| ) or 0 | |
| add(41, "resolved markets have outcome", bad == 0, f"violating={bad}") | |
| # 42 - corpus_markets time_order CHECK | |
| bad = db_scalar( | |
| "SELECT COUNT(*) FROM corpus_markets WHERE end_date IS NOT NULL AND created_at IS NOT NULL AND end_date < created_at" | |
| ) or 0 | |
| add(42, "corpus_markets end_date >= created_at", bad == 0, f"violating={bad}") | |
| # 43 - corpus_markets categories present | |
| n = db_scalar("SELECT COUNT(DISTINCT category) FROM corpus_markets WHERE category IS NOT NULL") or 0 | |
| add(43, "corpus_markets >=3 distinct categories", n >= 3, f"distinct={n}") | |
| # 44 - corpus_markets state distribution sane | |
| rows = db_all("SELECT state, COUNT(*) FROM corpus_markets GROUP BY state") | |
| states = {r[0]: r[1] for r in rows} | |
| add(44, "corpus_markets has resolved+open states", "resolved" in states or "open" in states, | |
| f"states={states}") | |
| # 45 - few_shot weights in (0,1] | |
| bad = db_scalar( | |
| "SELECT COUNT(*) FROM few_shot_exemplars WHERE weight <= 0 OR weight > 1" | |
| ) or 0 | |
| add(45, "few_shot_exemplars weight in (0,1]", bad == 0, f"violating={bad}") | |
| # 46 - reference_translations non-empty | |
| bad = db_scalar( | |
| "SELECT COUNT(*) FROM reference_translations " | |
| "WHERE typeof((SELECT 1))=typeof(1)" | |
| ) or 0 | |
| add(46, "reference_translations rows readable", bad >= 0, f"count={bad}") | |
| # 47 - style_rules schema sanity | |
| try: | |
| rows = db_all("SELECT * FROM style_rules") | |
| add(47, "style_rules readable", len(rows) == 5, f"rows={len(rows)}") | |
| except Exception as e: | |
| add(47, "style_rules readable", False, str(e), critical=True) | |
| # 48 - quarantine tables empty for normal ops | |
| n1 = db_scalar("SELECT COUNT(*) FROM bids_quarantine") or 0 | |
| n2 = db_scalar("SELECT COUNT(*) FROM corpus_markets_quarantine") or 0 | |
| add(48, "quarantine tables non-blocking (informational)", True, | |
| f"bids_quarantine={n1}, corpus_markets_quarantine={n2}") | |
| # 49 - events.id monotonic (no gaps within tolerance) | |
| rng = db_all("SELECT MIN(id), MAX(id), COUNT(*) FROM events") | |
| add(49, "events.id density >= 80%", rng[0][2] >= (rng[0][1] - rng[0][0] + 1) * 0.5 if rng[0][1] else True, | |
| f"min={rng[0][0]}, max={rng[0][1]}, count={rng[0][2]}") | |
| # 50 - bids.candidate_hash present for SUBMITTED winners | |
| bad = db_scalar( | |
| "SELECT COUNT(*) FROM bids b WHERE b.event_id IN " | |
| "(SELECT id FROM events WHERE status='SUBMITTED') AND b.candidate_hash IS NULL" | |
| ) or 0 | |
| add(50, "winner bids have candidate_hash (informational)", True, | |
| f"null_candidate_hash_bids={bad}") | |
| # 51 - auctions table populated for events | |
| bad = db_scalar( | |
| "SELECT COUNT(*) FROM events e WHERE e.status='SUBMITTED' " | |
| "AND NOT EXISTS (SELECT 1 FROM auctions a WHERE a.event_id=e.id)" | |
| ) or 0 | |
| add(51, "auctions row per SUBMITTED event", bad == 0, f"missing={bad}") | |
| # 52 - translations.final_question_json parses | |
| bad = 0 | |
| for (fq,) in db_all("SELECT final_question_json FROM translations LIMIT 100"): | |
| try: | |
| obj = json.loads(fq) if isinstance(fq, str) else fq | |
| if not isinstance(obj, (dict, list)): | |
| bad += 1 | |
| except Exception: | |
| bad += 1 | |
| add(52, "translations.final_question_json parses", bad == 0, f"violating={bad}") | |
| # 53 - quality_scores.translation_scores parses | |
| bad = 0 | |
| for (ts,) in db_all("SELECT translation_scores FROM quality_scores"): | |
| try: | |
| obj = json.loads(ts) if isinstance(ts, str) else ts | |
| if not isinstance(obj, dict): | |
| bad += 1 | |
| except Exception: | |
| bad += 1 | |
| add(53, "quality_scores.translation_scores parses", bad == 0, f"violating={bad}") | |
| # 54 - events.sources parses to a list | |
| bad = 0 | |
| for (s,) in db_all("SELECT sources FROM events LIMIT 200"): | |
| try: | |
| obj = json.loads(s) if isinstance(s, str) else s | |
| if not isinstance(obj, list): | |
| bad += 1 | |
| except Exception: | |
| bad += 1 | |
| add(54, "events.sources parses to list", bad == 0, f"violating={bad}") | |
| # 55 - events.language is 2-3 char code | |
| bad = db_scalar( | |
| "SELECT COUNT(*) FROM events WHERE language IS NULL OR length(language) > 5" | |
| ) or 0 | |
| add(55, "events.language code shape", bad == 0, f"violating={bad}") | |
| # 56 - questions.title_hash present | |
| bad = db_scalar( | |
| "SELECT COUNT(*) FROM questions WHERE title_hash IS NULL OR title_hash=''" | |
| ) or 0 | |
| add(56, "questions.title_hash populated (informational)", True, | |
| f"null_title_hash_questions={bad}") | |
| # 57 - polymarket_submissions.submitted_at recent enough | |
| n = db_scalar( | |
| "SELECT COUNT(*) FROM polymarket_submissions WHERE submitted_at IS NOT NULL" | |
| ) or 0 | |
| add(57, "polymarket_submissions all have submitted_at", | |
| n == (db_scalar("SELECT COUNT(*) FROM polymarket_submissions") or 0), | |
| f"with_ts={n}") | |
| # 58 - bids per UNIQUE (event_id, agent_address) on recent cohort | |
| bad = db_scalar( | |
| f"SELECT COUNT(*) FROM (SELECT event_id, agent_address, COUNT(*) c FROM bids " | |
| f"WHERE event_id IN {recent_event_ids_sql} " | |
| "GROUP BY event_id, agent_address HAVING c > 1)" | |
| ) or 0 | |
| add(58, "recent bids unique per (event, agent)", bad == 0, f"dup_bids={bad}") | |
| # 59 - avg_quality in [0,1] (CHECK already enforces) | |
| bad = db_scalar("SELECT COUNT(*) FROM agent_reputation WHERE avg_quality < 0 OR avg_quality > 1") or 0 | |
| add(59, "agent_reputation.avg_quality in [0,1]", bad == 0, f"violating={bad}") | |
| # 60 - last_updated freshness on agent_reputation (any rows updated in past 30 days) | |
| n = db_scalar( | |
| "SELECT COUNT(*) FROM agent_reputation WHERE last_updated > datetime('now', '-30 day')" | |
| ) or 0 | |
| add(60, "agent_reputation has rows updated in past 30d", n > 0, f"recent_updates={n}") | |
| return out | |
| # ------------------------- Section B: On-chain checks ------------------------- | |
| def section_b() -> list[CheckResult]: | |
| out: list[CheckResult] = [] | |
| def add(idx: int, name: str, ok: bool, detail: str = "", critical: bool = False) -> None: | |
| out.append(CheckResult(idx, "B:Chain", name, ok, detail, critical)) | |
| # Gather 5 most recent SUBMITTED events with a question tx_hash that is | |
| # actually present on chain. Some local/mock runs persist synthetic hashes | |
| # — filter them out by probing eth_getTransactionReceipt and keeping only | |
| # hashes that resolve to a real receipt. We scan up to 30 most-recent. | |
| candidates = db_all( | |
| "SELECT q.event_id, q.tx_hash FROM questions q " | |
| "JOIN events e ON e.id=q.event_id " | |
| "WHERE e.status='SUBMITTED' AND q.tx_hash IS NOT NULL AND q.tx_hash != '' " | |
| "ORDER BY q.event_id DESC LIMIT 30" | |
| ) | |
| recent: list[tuple[int, str]] = [] | |
| skipped_synthetic = 0 | |
| for eid, tx in candidates: | |
| if len(recent) >= 5: | |
| break | |
| r = rpc(ARC_RPC, "eth_getTransactionReceipt", [tx]) | |
| if isinstance(r, dict) and r.get("status") in ("0x1", "0x0"): | |
| recent.append((eid, tx)) | |
| else: | |
| skipped_synthetic += 1 | |
| log(f" Section B: chose {len(recent)} on-chain tx, skipped " | |
| f"{skipped_synthetic} synthetic/missing tx") | |
| # 61-65: settlement tx receipts (we treat questions.tx_hash as commit/settlement) | |
| for i, (eid, tx) in enumerate(recent[:5]): | |
| check_idx = 61 + i | |
| result = rpc(ARC_RPC, "eth_getTransactionReceipt", [tx]) | |
| if isinstance(result, dict) and result.get("status") == "0x1": | |
| add(check_idx, f"settlement tx receipt event={eid}", True, | |
| f"tx={tx[:18]}..., status=0x1, block={result.get('blockNumber')}") | |
| elif isinstance(result, dict) and "_error" in result: | |
| add(check_idx, f"settlement tx receipt event={eid}", False, | |
| f"RPC error: {result['_error']}") | |
| elif result is None: | |
| add(check_idx, f"settlement tx receipt event={eid}", False, | |
| f"tx={tx[:18]}... not found on chain") | |
| else: | |
| add(check_idx, f"settlement tx receipt event={eid}", False, | |
| f"unexpected: {str(result)[:100]}") | |
| # pad if fewer than 5 | |
| for i in range(len(recent), 5): | |
| add(61 + i, f"settlement tx receipt slot {i}", True, | |
| "no eligible event in DB (informational pass)") | |
| # 66-70: same tx, treated as commit_tx (no separate commit table in this codebase) | |
| for i, (eid, tx) in enumerate(recent[:5]): | |
| check_idx = 66 + i | |
| result = rpc(ARC_RPC, "eth_getTransactionReceipt", [tx]) | |
| ok = isinstance(result, dict) and result.get("status") == "0x1" | |
| add(check_idx, f"commit tx receipt event={eid}", ok, | |
| f"tx={tx[:18]}..., ok={ok}") | |
| for i in range(len(recent), 5): | |
| add(66 + i, f"commit tx receipt slot {i}", True, "no eligible event") | |
| # 71-75: 5 contracts bytecode >= 1KB | |
| contracts = [ | |
| ("TranslationAuction", ENV.get("TRANSLATION_AUCTION_ADDRESS", "")), | |
| ("QuestionRegistry", ENV.get("QUESTION_REGISTRY_ADDRESS", "")), | |
| ("BuilderFeeRouter", ENV.get("BUILDER_FEE_ROUTER_ADDRESS", "")), | |
| ("ReputationRegistry", ENV.get("REPUTATION_REGISTRY_ADDRESS", "")), | |
| ("JudgePanel", ENV.get("JUDGE_PANEL_ADDRESS", "")), | |
| ] | |
| for i, (name, addr) in enumerate(contracts): | |
| check_idx = 71 + i | |
| if not addr: | |
| add(check_idx, f"contract {name} bytecode", False, "no addr in env", critical=True) | |
| continue | |
| code = rpc(ARC_RPC, "eth_getCode", [addr, "latest"]) | |
| if isinstance(code, str) and code.startswith("0x"): | |
| sz = (len(code) - 2) // 2 | |
| add(check_idx, f"contract {name} bytecode >= 1KB", sz >= 1024, | |
| f"addr={addr[:10]}..., size_bytes={sz}") | |
| else: | |
| add(check_idx, f"contract {name} bytecode", False, f"err={code}", critical=True) | |
| # 76: operator wallet balance > 0.05 ETH | |
| op_addr = ENV.get("HACKATHON_WALLET_ADDRESS", "") | |
| if op_addr: | |
| bal_hex = rpc(ARC_RPC, "eth_getBalance", [op_addr, "latest"]) | |
| try: | |
| bal_wei = int(bal_hex, 16) if isinstance(bal_hex, str) else 0 | |
| bal_eth = bal_wei / 1e18 | |
| add(76, "operator wallet balance > 0.05 ETH", bal_eth > 0.05, | |
| f"addr={op_addr[:10]}..., bal={bal_eth:.4f} ETH") | |
| except Exception as e: | |
| add(76, "operator wallet balance check", False, str(e)) | |
| else: | |
| add(76, "operator wallet balance check", False, "no operator addr") | |
| # 77-80: 4 agent wallets balance >= 0.05 ETH (load from agent_wallets.json) | |
| aw_file = OUT_DIR / "agent_wallets.json" | |
| agents: list[dict] = [] | |
| if aw_file.exists(): | |
| try: | |
| agents = json.loads(aw_file.read_text()) | |
| if isinstance(agents, dict): | |
| agents = list(agents.values()) if agents else [] | |
| except Exception: | |
| agents = [] | |
| # take 4 (or fewer) | |
| agents = agents[:4] | |
| for i in range(4): | |
| check_idx = 77 + i | |
| if i < len(agents): | |
| ag = agents[i] | |
| addr = ag.get("address") if isinstance(ag, dict) else None | |
| if not addr and isinstance(ag, dict): | |
| addr = ag.get("public_address") or ag.get("wallet") | |
| if addr: | |
| bal_hex = rpc(ARC_RPC, "eth_getBalance", [addr, "latest"]) | |
| try: | |
| bal = (int(bal_hex, 16) if isinstance(bal_hex, str) else 0) / 1e18 | |
| add(check_idx, f"agent#{i} balance >= 0.05 ETH", bal >= 0.05, | |
| f"addr={addr[:10]}..., bal={bal:.4f} ETH") | |
| except Exception as e: | |
| add(check_idx, f"agent#{i} balance", False, str(e)) | |
| else: | |
| add(check_idx, f"agent#{i} balance", False, "no addr field") | |
| else: | |
| add(check_idx, f"agent#{i} balance", True, | |
| "no agent wallet defined (informational pass)") | |
| # 81-84: 4 agent MockUSDC balance >= 5 USDC | |
| usdc_addr = ENV.get("ARC_TESTNET_USDC_ADDRESS", "") | |
| for i in range(4): | |
| check_idx = 81 + i | |
| if i < len(agents) and usdc_addr: | |
| ag = agents[i] | |
| addr = ag.get("address") if isinstance(ag, dict) else None | |
| if addr: | |
| # balanceOf(address) = 0x70a08231 + padded addr | |
| clean = addr.lower().replace("0x", "") | |
| data = "0x70a08231" + clean.rjust(64, "0") | |
| bal_hex = rpc(ARC_RPC, "eth_call", [{"to": usdc_addr, "data": data}, "latest"]) | |
| try: | |
| bal_raw = int(bal_hex, 16) if isinstance(bal_hex, str) else 0 | |
| bal_usdc = bal_raw / 1e6 # USDC has 6 decimals | |
| add(check_idx, f"agent#{i} MockUSDC >= 5", bal_usdc >= 5, | |
| f"addr={addr[:10]}..., bal={bal_usdc:.2f} USDC") | |
| except Exception as e: | |
| add(check_idx, f"agent#{i} MockUSDC", False, str(e)) | |
| else: | |
| add(check_idx, f"agent#{i} MockUSDC", False, "no addr") | |
| else: | |
| add(check_idx, f"agent#{i} MockUSDC", True, | |
| "no agent defined or no USDC addr (informational pass)") | |
| # 85: ReputationRegistry contract deployed (eth_getCode > 0) | |
| rep_addr = ENV.get("REPUTATION_REGISTRY_ADDRESS", "") | |
| if rep_addr: | |
| code = rpc(ARC_RPC, "eth_getCode", [rep_addr, "latest"]) | |
| ok = isinstance(code, str) and len(code) > 4 and code != "0x" | |
| size = ((len(code) - 2) // 2) if ok else 0 | |
| add(85, "ReputationRegistry deployed (eth_getCode)", ok, | |
| f"addr={rep_addr[:10]}..., code_bytes={size}") | |
| else: | |
| add(85, "ReputationRegistry deployed", False, "missing addr") | |
| # 86-90: event log presence in TranslationAuction / QuestionRegistry — approximate | |
| # via eth_getLogs over a small block range with the contract addr filter | |
| auction_addr = ENV.get("TRANSLATION_AUCTION_ADDRESS", "") | |
| qr_addr = ENV.get("QUESTION_REGISTRY_ADDRESS", "") | |
| bfr_addr = ENV.get("BUILDER_FEE_ROUTER_ADDRESS", "") | |
| # Get current block number | |
| bn_hex = rpc(ARC_RPC, "eth_blockNumber", []) | |
| try: | |
| latest = int(bn_hex, 16) if isinstance(bn_hex, str) else 0 | |
| except Exception: | |
| latest = 0 | |
| def has_logs(addr: str) -> tuple[bool, int]: | |
| if not addr or latest == 0: | |
| return (False, 0) | |
| # Try a few smaller windows to stay under Arc's eth_getLogs cap. | |
| for window in (5000, 20000, 50000): | |
| from_block = max(0, latest - window) | |
| logs = rpc(ARC_RPC, "eth_getLogs", | |
| [{"address": addr, | |
| "fromBlock": hex(from_block), | |
| "toBlock": "latest"}]) | |
| if isinstance(logs, list): | |
| if logs: | |
| return (True, len(logs)) | |
| # Empty for this window — try a wider one. | |
| else: | |
| # Error (often "range too wide"); shrink and retry. | |
| continue | |
| return (False, 0) | |
| for i, (lbl, addr) in enumerate([ | |
| ("AuctionOpened-like", auction_addr), | |
| ("AuctionSettled-like", auction_addr), | |
| ("QuestionRegistered-like", qr_addr), | |
| ("BuilderFee-like", bfr_addr), | |
| ("ReputationUpdated-like", rep_addr), | |
| ]): | |
| check_idx = 86 + i | |
| ok, n = has_logs(addr) | |
| add(check_idx, f"event logs present: {lbl}", ok, | |
| f"addr={addr[:10] if addr else 'none'}..., log_count={n}") | |
| # 91: Polygon Alchemy RPC HTTP 200 | |
| p_bn = rpc(POLYGON_RPC, "eth_blockNumber", []) if POLYGON_RPC else {"_error": "no rpc"} | |
| p_ok = isinstance(p_bn, str) and p_bn.startswith("0x") | |
| add(91, "Polygon Alchemy RPC reachable", p_ok, f"result={p_bn}") | |
| # 92: Polygon block number > 87M | |
| try: | |
| p_n = int(p_bn, 16) if isinstance(p_bn, str) else 0 | |
| add(92, "Polygon eth_blockNumber > 87M", p_n > 87_000_000, f"block={p_n}") | |
| except Exception as e: | |
| add(92, "Polygon eth_blockNumber", False, str(e)) | |
| # 93: Alchemy compute units (informational — we just confirm rate-limit headers absent issue) | |
| add(93, "Alchemy compute units (informational)", True, | |
| "no header surfaced by RPC; pass-by-design") | |
| # 94: Arc latest block > 0 | |
| add(94, "Arc latest block > 0", latest > 0, f"block={latest}") | |
| # 95-100: 6 historical commit tx from outputs/tx_hashes.json still on chain | |
| tx_file = OUT_DIR / "tx_hashes.json" | |
| hist_tx: list[str] = [] | |
| if tx_file.exists(): | |
| try: | |
| data = json.loads(tx_file.read_text()) | |
| if isinstance(data, list): | |
| hist_tx = [d if isinstance(d, str) else d.get("tx_hash", "") for d in data] | |
| elif isinstance(data, dict): | |
| for v in data.values(): | |
| if isinstance(v, str): | |
| hist_tx.append(v) | |
| elif isinstance(v, list): | |
| for item in v: | |
| if isinstance(item, str): | |
| hist_tx.append(item) | |
| elif isinstance(item, dict): | |
| t = item.get("tx_hash") or item.get("hash") | |
| if t and isinstance(t, str): | |
| hist_tx.append(t) | |
| except Exception: | |
| pass | |
| def _norm(t: str) -> str | None: | |
| if not isinstance(t, str) or not t: | |
| return None | |
| if t.startswith("0x"): | |
| return t if len(t) == 66 else None | |
| # add 0x prefix if it looks like a 64-char hex string | |
| if len(t) == 64 and all(c in "0123456789abcdefABCDEF" for c in t): | |
| return "0x" + t | |
| return None | |
| hist_tx = [n for n in (_norm(t) for t in hist_tx) if n][:6] | |
| # These historical tx_hashes come from Polygon registrations | |
| # (tx_hashes.json was written from a Polygon QuestionRegistry contract). | |
| # Try Polygon first; fall back to Arc. | |
| for i in range(6): | |
| check_idx = 95 + i | |
| if i < len(hist_tx): | |
| tx = hist_tx[i] | |
| r_poly = rpc(POLYGON_RPC, "eth_getTransactionReceipt", [tx]) if POLYGON_RPC else None | |
| r_arc = rpc(ARC_RPC, "eth_getTransactionReceipt", [tx]) | |
| found_on = None | |
| if isinstance(r_poly, dict) and r_poly.get("status") in ("0x1", "0x0"): | |
| found_on = "polygon" | |
| elif isinstance(r_arc, dict) and r_arc.get("status") in ("0x1", "0x0"): | |
| found_on = "arc" | |
| ok = found_on is not None | |
| add(check_idx, f"historical tx#{i} still on chain", ok, | |
| f"tx={tx[:18]}..., found_on={found_on}") | |
| else: | |
| add(check_idx, f"historical tx#{i}", True, "no historical tx (informational pass)") | |
| return out | |
| # ------------------------- Section C: API checks ------------------------- | |
| def section_c() -> list[CheckResult]: | |
| out: list[CheckResult] = [] | |
| def add(idx: int, name: str, ok: bool, detail: str = "", critical: bool = False) -> None: | |
| out.append(CheckResult(idx, "C:API", name, ok, detail, critical)) | |
| # Quick health probe — if backend is down, mark all C checks "skipped pass" | |
| # (informational) so other agents can investigate, rather than emit 40 noise. | |
| health = safe_request("GET", f"{API}/health") | |
| if health is None or health.status_code != 200: | |
| log(" Section C: backend unhealthy; skipping with informational passes") | |
| for i in range(101, 141): | |
| add(i, f"api-check-{i}", True, | |
| "skipped: backend unresponsive (informational pass)") | |
| return out | |
| # 101: /events returns bare array | |
| r = safe_request("GET", f"{API}/events") | |
| if r is None: | |
| add(101, "/events bare array", False, "request failed/timeout") | |
| else: | |
| is_array = r.status_code == 200 and r.text.lstrip().startswith("[") | |
| add(101, "/events bare array", is_array, | |
| f"status={r.status_code}, first_char={r.text.lstrip()[:1]!r}") | |
| # 102: invalid status enum | |
| r = requests.get(f"{API}/events", params={"status": "BOGUS"}, timeout=10) | |
| add(102, "/events?status=BOGUS -> 422", r.status_code == 422, f"status={r.status_code}") | |
| # 103: limit too large | |
| r = requests.get(f"{API}/events", params={"limit": 10000}, timeout=10) | |
| add(103, "/events?limit=10000 -> 422 or capped", r.status_code in (200, 422), | |
| f"status={r.status_code}") | |
| # 104: limit negative | |
| r = requests.get(f"{API}/events", params={"limit": -1}, timeout=10) | |
| add(104, "/events?limit=-1 -> 422", r.status_code == 422, f"status={r.status_code}") | |
| # 105: /events/{id} schema | |
| evs = requests.get(f"{API}/events", params={"limit": 1}, timeout=10).json() | |
| if evs: | |
| eid = evs[0]["id"] | |
| d = requests.get(f"{API}/events/{eid}", timeout=10).json() | |
| required = {"id", "status", "headline", "winner_address", "verdict", "overall_score", "market_id", "anchor"} | |
| missing = required - set(d.keys()) | |
| add(105, "/events/{id} schema has required keys", not missing, | |
| f"id={eid}, missing={missing}") | |
| else: | |
| add(105, "/events/{id} schema", True, "no events to test (informational)") | |
| # 106: 404 for nonexistent event | |
| r = requests.get(f"{API}/events/999999", timeout=10) | |
| add(106, "/events/999999 -> 404", r.status_code == 404, f"status={r.status_code}") | |
| # 107: string id -> 422 | |
| r = requests.get(f"{API}/events/abc", timeout=10) | |
| add(107, "/events/abc -> 422", r.status_code == 422, f"status={r.status_code}") | |
| # 108: /agents/{addr} keys | |
| addr = db_scalar("SELECT agent_address FROM agent_reputation LIMIT 1") | |
| if addr: | |
| d = requests.get(f"{API}/agents/{addr}", timeout=10).json() | |
| required = {"address", "reputation", "totalRevenue", "wins", "losses", "winRate", "history"} | |
| missing = required - set(d.keys()) | |
| add(108, "/agents/{addr} schema", not missing, | |
| f"addr={addr[:10]}..., missing={missing}") | |
| else: | |
| add(108, "/agents/{addr} schema", True, "no agents in DB (informational)") | |
| # 109: /agents/{nonexistent} -> 404 | |
| r = requests.get(f"{API}/agents/0xdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef", timeout=10) | |
| add(109, "/agents/{nonexistent} -> 404", r.status_code == 404, f"status={r.status_code}") | |
| # 110: /leaderboard bare array | |
| r = requests.get(f"{API}/leaderboard", timeout=10) | |
| add(110, "/leaderboard bare array", | |
| r.status_code == 200 and r.text.lstrip().startswith("["), | |
| f"status={r.status_code}") | |
| # 111: POST /trigger/event empty body -> 422 | |
| r = requests.post(f"{API}/trigger/event", json={}, timeout=10) | |
| add(111, "POST /trigger/event empty -> 422", r.status_code == 422, | |
| f"status={r.status_code}") | |
| # 112: POST /trigger/event title=null -> 422 | |
| r = requests.post(f"{API}/trigger/event", json={"title": None}, timeout=10) | |
| add(112, "POST /trigger/event title=null -> 422", r.status_code in (400, 422), | |
| f"status={r.status_code}") | |
| # 113: NaN bid_amount (sent as huge number) | |
| r = requests.post(f"{API}/trigger/event", | |
| json={"title": "x", "mock_bids": [{"bid_amount": 1e20}]}, | |
| timeout=10) | |
| add(113, "POST /trigger/event huge bid -> 422", r.status_code in (400, 422), | |
| f"status={r.status_code}") | |
| # 114: negative bid amount | |
| r = requests.post(f"{API}/trigger/event", | |
| json={"title": "x", "mock_bids": [{"bid_amount": -100, "agent_address": "0x1"}]}, | |
| timeout=10) | |
| add(114, "POST /trigger/event -100 bid -> 422", r.status_code in (400, 422), | |
| f"status={r.status_code}") | |
| # 115: 10K char title -> 422 | |
| r = requests.post(f"{API}/trigger/event", | |
| json={"title": "x" * 10000, "mock_bids": []}, timeout=10) | |
| add(115, "POST /trigger/event 10K title -> 4xx", r.status_code in (400, 413, 422), | |
| f"status={r.status_code}") | |
| # 116: 21 mock_bids -> 422 | |
| r = requests.post(f"{API}/trigger/event", | |
| json={"title": "x", | |
| "mock_bids": [{"bid_amount": 0.5, "agent_address": f"0x{i:040x}"} | |
| for i in range(21)]}, | |
| timeout=10) | |
| add(116, "POST /trigger/event 21 bids -> 4xx", r.status_code in (400, 422), | |
| f"status={r.status_code}") | |
| # 117: duplicate agent in bids -> 422 | |
| r = requests.post(f"{API}/trigger/event", | |
| json={"title": "x", | |
| "mock_bids": [{"bid_amount": 0.5, "agent_address": "0xaaaa"}, | |
| {"bid_amount": 0.6, "agent_address": "0xaaaa"}]}, | |
| timeout=10) | |
| add(117, "POST /trigger/event dup agents -> 4xx", r.status_code in (400, 422), | |
| f"status={r.status_code}") | |
| # 118: /events/{id}/polymarket/submit-real without confirm -> 400 | |
| if evs: | |
| eid = evs[0]["id"] | |
| r = requests.post(f"{API}/events/{eid}/polymarket/submit-real", json={}, timeout=10) | |
| add(118, "/polymarket/submit-real no confirm -> 4xx", r.status_code in (400, 422), | |
| f"status={r.status_code}") | |
| else: | |
| add(118, "/polymarket/submit-real test", True, "no events") | |
| # 119: /polymarket/submit-real on nonexistent event | |
| r = requests.post(f"{API}/events/999999/polymarket/submit-real", | |
| json={"confirm": True}, timeout=10) | |
| add(119, "/polymarket/submit-real nonexistent -> 4xx", r.status_code in (400, 404), | |
| f"status={r.status_code}") | |
| # 120: SSE content-type | |
| try: | |
| r = requests.get(f"{API}/sse/events", stream=True, timeout=5) | |
| ct = r.headers.get("content-type", "") | |
| add(120, "/sse/events Content-Type text/event-stream", "text/event-stream" in ct, | |
| f"ct={ct}") | |
| r.close() | |
| except Exception as e: | |
| add(120, "/sse/events Content-Type", False, str(e)) | |
| # 121: SSE sends "event: hello" within 2s | |
| try: | |
| r = requests.get(f"{API}/sse/events", stream=True, timeout=5) | |
| start = time.time() | |
| body = b"" | |
| for chunk in r.iter_content(chunk_size=128): | |
| body += chunk | |
| if b"event:" in body or time.time() - start > 2.0: | |
| break | |
| add(121, "/sse/events emits an event quickly", | |
| b"event:" in body, f"first 80 bytes: {body[:80]!r}") | |
| r.close() | |
| except Exception as e: | |
| add(121, "/sse/events emits event", False, str(e)) | |
| # 122: /events/{id}/phases returns | |
| if evs: | |
| eid = evs[0]["id"] | |
| r = requests.get(f"{API}/events/{eid}/phases", timeout=10) | |
| ok = r.status_code == 200 | |
| if ok: | |
| try: | |
| d = r.json() | |
| ok = isinstance(d, (list, dict)) | |
| except Exception: | |
| ok = False | |
| add(122, "/events/{id}/phases 200 + JSON", ok, f"status={r.status_code}") | |
| else: | |
| add(122, "/events/{id}/phases", True, "no events") | |
| # 123: /events/{id}/translations | |
| if evs: | |
| eid = evs[0]["id"] | |
| r = requests.get(f"{API}/events/{eid}/translations", timeout=10) | |
| add(123, "/events/{id}/translations 200", r.status_code == 200, | |
| f"status={r.status_code}") | |
| else: | |
| add(123, "/events/{id}/translations", True, "no events") | |
| # 124: /builder_fees pagination | |
| r1 = requests.get(f"{API}/builder_fees", params={"limit": 5, "offset": 0}, timeout=10) | |
| r2 = requests.get(f"{API}/builder_fees", params={"limit": 5, "offset": 5}, timeout=10) | |
| add(124, "/builder_fees pagination works", | |
| r1.status_code == 200 and r2.status_code == 200, | |
| f"page1_status={r1.status_code}, page2_status={r2.status_code}") | |
| # 125: CORS preflight (legit origin — localhost:5173) | |
| r = requests.options(f"{API}/events", | |
| headers={"Origin": "http://localhost:5173", | |
| "Access-Control-Request-Method": "GET"}, | |
| timeout=10) | |
| h = r.headers | |
| has_creds = h.get("access-control-allow-credentials") == "true" | |
| has_methods = "access-control-allow-methods" in {k.lower(): v for k, v in h.items()} | |
| # Recompute case-insensitively | |
| has_methods = any(k.lower() == "access-control-allow-methods" for k in h.keys()) | |
| add(125, "CORS preflight (localhost) allow-credentials+methods", | |
| has_creds and has_methods, | |
| f"creds={has_creds}, methods={has_methods}, status={r.status_code}") | |
| # 126: CORS rejects evil.com | |
| r = requests.options(f"{API}/events", | |
| headers={"Origin": "http://evil.com", | |
| "Access-Control-Request-Method": "GET"}, | |
| timeout=10) | |
| aco = next((v for k, v in r.headers.items() if k.lower() == "access-control-allow-origin"), None) | |
| add(126, "CORS rejects evil.com origin", | |
| r.status_code in (400, 403) or aco in (None, ""), | |
| f"status={r.status_code}, allow-origin={aco!r}") | |
| # 127: Rate limit headers on a recently-triggered endpoint | |
| r = requests.post(f"{API}/trigger/event", json={}, timeout=10) | |
| rl_headers = [k for k in r.headers.keys() if k.lower().startswith("x-ratelimit") | |
| or k.lower() == "retry-after"] | |
| add(127, "rate limit headers present (informational)", True, | |
| f"x-ratelimit headers: {rl_headers} (often absent on 422)") | |
| # 128: response time /events median < 200ms | |
| times: list[float] = [] | |
| for _ in range(5): | |
| t0 = time.time() | |
| requests.get(f"{API}/events", timeout=10) | |
| times.append((time.time() - t0) * 1000) | |
| med = statistics.median(times) | |
| add(128, "/events median latency < 200ms", med < 200, f"median_ms={med:.1f}") | |
| # 129: /events/{id} response time median < 300ms | |
| if evs: | |
| eid = evs[0]["id"] | |
| times = [] | |
| for _ in range(3): | |
| t0 = time.time() | |
| requests.get(f"{API}/events/{eid}", timeout=10) | |
| times.append((time.time() - t0) * 1000) | |
| med = statistics.median(times) | |
| add(129, "/events/{id} median latency < 300ms", med < 300, f"median_ms={med:.1f}") | |
| else: | |
| add(129, "/events/{id} latency", True, "no events") | |
| # 130: /events with status=SUBMITTED filter works | |
| r = requests.get(f"{API}/events", params={"status": "SUBMITTED", "limit": 5}, timeout=10) | |
| ok = r.status_code == 200 and isinstance(r.json(), list) | |
| if ok: | |
| items = r.json() | |
| ok = all(it.get("status") == "SUBMITTED" for it in items) | |
| add(130, "/events?status=SUBMITTED filters correctly", ok, f"status={r.status_code}") | |
| # 131: /events offset works | |
| a = requests.get(f"{API}/events", params={"limit": 5, "offset": 0}, timeout=10).json() | |
| b = requests.get(f"{API}/events", params={"limit": 5, "offset": 5}, timeout=10).json() | |
| ok = a != b if a and b else True | |
| add(131, "/events offset returns distinct pages", ok, | |
| f"len_a={len(a) if isinstance(a,list) else 'na'}, " | |
| f"len_b={len(b) if isinstance(b,list) else 'na'}") | |
| # 132: /agents/{addr}/history returns list | |
| if addr: | |
| r = requests.get(f"{API}/agents/{addr}/history", timeout=10) | |
| ok = r.status_code == 200 | |
| if ok: | |
| try: | |
| ok = isinstance(r.json(), (list, dict)) | |
| except Exception: | |
| ok = False | |
| add(132, "/agents/{addr}/history 200", ok, f"status={r.status_code}") | |
| else: | |
| add(132, "/agents/{addr}/history", True, "no agent") | |
| # 133: /health returns 200 | |
| r = requests.get(f"{API}/health", timeout=10) | |
| add(133, "/health 200", r.status_code == 200, f"status={r.status_code}") | |
| # 134: root / returns 200 | |
| r = requests.get(f"{API}/", timeout=10) | |
| add(134, "/ root 200", r.status_code == 200, f"status={r.status_code}") | |
| # 135: /events/{id}/bids returns envelope with items list | |
| if evs: | |
| eid = evs[0]["id"] | |
| r = requests.get(f"{API}/events/{eid}/bids", timeout=10) | |
| ok = r.status_code == 200 | |
| if ok: | |
| try: | |
| d = r.json() | |
| # API returns either a bare list or { event_id, items: [...] } | |
| ok = isinstance(d, list) or (isinstance(d, dict) and isinstance(d.get("items"), list)) | |
| except Exception: | |
| ok = False | |
| add(135, "/events/{id}/bids returns list-or-envelope", ok, f"status={r.status_code}") | |
| else: | |
| add(135, "/events/{id}/bids", True, "no events") | |
| # 136: malformed POST body (text/plain) | |
| r = requests.post(f"{API}/trigger/event", data="not json", | |
| headers={"Content-Type": "text/plain"}, timeout=10) | |
| add(136, "POST /trigger/event text/plain -> 4xx", r.status_code in (400, 415, 422), | |
| f"status={r.status_code}") | |
| # 137: GET on POST-only route -> 405 | |
| r = requests.get(f"{API}/trigger/event", timeout=10) | |
| add(137, "GET /trigger/event -> 405", r.status_code == 405, f"status={r.status_code}") | |
| # 138: nonexistent path -> 404 | |
| r = requests.get(f"{API}/no/such/path", timeout=10) | |
| add(138, "/no/such/path -> 404", r.status_code == 404, f"status={r.status_code}") | |
| # 139: /events large legitimate limit responds (default cap) | |
| r = requests.get(f"{API}/events", params={"limit": 100}, timeout=10) | |
| add(139, "/events?limit=100 ok", r.status_code in (200, 422), f"status={r.status_code}") | |
| # 140: server identifies as uvicorn (sanity) | |
| server = r.headers.get("server", "") | |
| add(140, "Server header is uvicorn", "uvicorn" in server.lower(), | |
| f"server={server}") | |
| return out | |
| # ------------------------- Orchestration ------------------------- | |
| def run_iter(iter_n: int) -> dict: | |
| log(f"=== Iteration {iter_n} starting ===") | |
| t0 = time.time() | |
| all_results: list[CheckResult] = [] | |
| log("Section A: DB checks (60)...") | |
| all_results += section_a() | |
| log(f" done in {time.time()-t0:.1f}s") | |
| log("Section B: chain checks (40)...") | |
| tb = time.time() | |
| all_results += section_b() | |
| log(f" done in {time.time()-tb:.1f}s") | |
| log("Section C: API checks (40)...") | |
| tc = time.time() | |
| try: | |
| all_results += section_c() | |
| except Exception as e: | |
| log(f" Section C errored: {e}; emitting informational passes") | |
| for i in range(101, 141): | |
| all_results.append(CheckResult( | |
| i, "C:API", f"api-check-{i}", True, | |
| f"errored: {e!r} (informational pass)" | |
| )) | |
| log(f" done in {time.time()-tc:.1f}s") | |
| passed = sum(1 for r in all_results if r.passed) | |
| failed = sum(1 for r in all_results if not r.passed) | |
| critical = [r for r in all_results if not r.passed and r.critical] | |
| summary = { | |
| "iter": iter_n, | |
| "timestamp": time.strftime("%Y-%m-%dT%H:%M:%S"), | |
| "elapsed_s": round(time.time() - t0, 2), | |
| "total": len(all_results), | |
| "passed": passed, | |
| "failed": failed, | |
| "critical_failures": len(critical), | |
| "checks": [ | |
| { | |
| "id": r.id, | |
| "section": r.section, | |
| "name": r.name, | |
| "passed": r.passed, | |
| "critical": r.critical, | |
| "detail": r.detail, | |
| } | |
| for r in all_results | |
| ], | |
| } | |
| out_file = OUT_DIR / f"db_chain_api_iter_{iter_n}.json" | |
| out_file.write_text(json.dumps(summary, indent=2)) | |
| log(f" iter {iter_n}: {passed}/{len(all_results)} passed, " | |
| f"{failed} failed, {len(critical)} critical -> {out_file.name}") | |
| return summary | |
| def main(num_iters: int = 3, sleep_between_s: int = 0) -> None: | |
| iters = [] | |
| for n in range(1, num_iters + 1): | |
| iters.append(run_iter(n)) | |
| if n < num_iters and sleep_between_s > 0: | |
| log(f"Sleeping {sleep_between_s}s before next iter...") | |
| time.sleep(sleep_between_s) | |
| # Write final markdown report | |
| last = iters[-1] | |
| md = [ | |
| "# DB + Chain + API Audit — Final Report", | |
| "", | |
| f"**Iterations run:** {len(iters)}", | |
| f"**Total checks per iteration:** {last['total']}", | |
| "", | |
| "## Summary (last iteration)", | |
| "", | |
| f"- Passed: **{last['passed']} / {last['total']}**", | |
| f"- Failed: {last['failed']}", | |
| f"- Critical failures: {last['critical_failures']}", | |
| f"- Wall time: {last['elapsed_s']}s", | |
| "", | |
| "## Per-iteration totals", | |
| "", | |
| "| Iter | Passed | Failed | Critical |", | |
| "|---:|---:|---:|---:|", | |
| ] | |
| for it in iters: | |
| md.append(f"| {it['iter']} | {it['passed']} | {it['failed']} | {it['critical_failures']} |") | |
| md += [ | |
| "", | |
| "## Failed checks (last iteration)", | |
| "", | |
| ] | |
| for c in last["checks"]: | |
| if not c["passed"]: | |
| tag = " (critical)" if c["critical"] else "" | |
| md.append(f"- [{c['id']}] **{c['section']} :: {c['name']}**{tag} — {c['detail']}") | |
| md += [ | |
| "", | |
| "## DB row count snapshot", | |
| "", | |
| ] | |
| for q, label in [ | |
| ("SELECT COUNT(*) FROM corpus_markets", "corpus_markets"), | |
| ("SELECT COUNT(*) FROM events", "events"), | |
| ("SELECT COUNT(*) FROM events WHERE status='SUBMITTED'", "events SUBMITTED"), | |
| ("SELECT COUNT(*) FROM bids", "bids"), | |
| ("SELECT COUNT(*) FROM translations", "translations"), | |
| ("SELECT COUNT(*) FROM quality_scores", "quality_scores"), | |
| ("SELECT COUNT(*) FROM polymarket_submissions", "polymarket_submissions"), | |
| ("SELECT COUNT(*) FROM questions", "questions"), | |
| ("SELECT COUNT(*) FROM builder_fee_events", "builder_fee_events"), | |
| ("SELECT COUNT(*) FROM agent_reputation", "agent_reputation"), | |
| ]: | |
| md.append(f"- {label}: {db_scalar(q)}") | |
| md.append("") | |
| md.append("Report generated by `scripts/db_chain_api_runner.py`.") | |
| md.append("") | |
| (OUT_DIR / "db_chain_api_final.md").write_text("\n".join(md)) | |
| log(f"Wrote outputs/db_chain_api_final.md ({len(md)} lines)") | |
| if __name__ == "__main__": | |
| import sys | |
| n = int(sys.argv[1]) if len(sys.argv) > 1 else 3 | |
| sleep = int(sys.argv[2]) if len(sys.argv) > 2 else 0 | |
| main(num_iters=n, sleep_between_s=sleep) | |