#!/usr/bin/env python3 """ scripts/discover.py — find NEW markets, and the size each one dies at. venv/bin/python scripts/discover.py # top 40 tokens venv/bin/python scripts/discover.py --tokens 100 venv/bin/python scripts/discover.py --apply # pin what survives ──────────────────────────────────────────────────────────────────────────── WHY THIS AND NOT scan_tokens.py ──────────────────────────────────────────────────────────────────────────── Operator: "improve test scan token and signal we need to see new markets and tokens." scan_tokens.py asks Jupiter for a round trip and reads the answer. Jupiter's router is built to return the BEST path, which means it has already blended the venues and erased the difference between them — the one place a real spread can still be sitting. Every route this deployment has scanned that way has come back negative, 185,000 times. Their own /venues screen shows what that hides. One pair, one moment, seven venues: Meteora DLMM 72.43 Raydium 72.36 Raydium CLMM 72.41 Orca V2 72.26 Whirlpool 72.40 Phoenix 71.99 Meteora against Orca V2 is 23.5 bps. Against 0.80 bps of fees. At 1 SOL. So this asks the per-venue question across MANY tokens, and then — because a gap at 1 SOL means nothing — walks each survivor up a size ladder until it dies, and reports the largest size that still clears its fees. ──────────────────────────────────────────────────────────────────────────── WHAT IT REFUSES TO DO ──────────────────────────────────────────────────────────────────────────── It never reports a one-way gap as an opportunity. #149 shipped exactly that mistake and printed "$331,396" next to a venue with no order book. Every number here comes from a walked round trip: base→mid on the buy venue, mid→base on the sell venue, at that size, paying both legs' impact. A gap over _ABSURD_GAP_BPS is reported as an illiquid venue and never as an edge, for the same reason. Read-only. Quotes only. No key is loaded, nothing is signed or sent. """ from __future__ import annotations import argparse import asyncio import csv import json import os import statistics import sys import time import urllib.request from datetime import datetime, timezone from pathlib import Path from typing import Any, Optional sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) from modules.env_file import effective_haircut_bps, load_env_file load_env_file() _G, _Y, _R, _B, _0 = "\033[32m", "\033[33m", "\033[31m", "\033[1m", "\033[0m" # Venues deep enough on major pairs to be worth a separate quote. A venue # nobody routes through cannot disagree with anyone in a way you can trade. _VENUES = ["Whirlpool", "Orca V2", "Meteora DLMM", "Raydium", "Raydium CLMM", "Lifinity V2", "Phoenix"] # Same threshold scan_crossvenue uses. A one-way gap above this on a pair # anyone trades is a venue with no book at that size, not an edge. _ABSURD_GAP_BPS = 300.0 # Jupiter has moved this endpoint more than once, and each move broke this # script with a network error that reads like a dead box rather than a # renamed URL. The operator reported exactly that: "this not working # venv/bin/python scripts/discover.py --tokens 40". # # So: a CHAIN, tried in order, first one that answers wins. An explicit # JUPITER_TOKEN_LIST_URL still overrides everything and is tried alone — # if you named a URL, silently using a different one is worse than failing. _TOKEN_LIST_FALLBACKS = ( "https://lite-api.jup.ag/tokens/v1/tagged/verified", "https://api.jup.ag/tokens/v1/tagged/verified", "https://tokens.jup.ag/tokens?tags=verified", ) _TOKEN_LIST_URL = os.getenv("JUPITER_TOKEN_LIST_URL", "").strip() def _fetch_token_list(limit: int) -> list[dict]: """Jupiter's verified token list, newest markets included. Fails LOUDLY. A discovery tool that silently falls back to the tokens already configured would report "nothing new found" while never having looked — the same false negative that has cost this project days. Note the difference between the two kinds of fallback here. Trying another URL for the SAME question is fine — it is one list, published at several addresses. Falling back to the tokens already configured is not, because it changes the question from "what is new" to "what do I already have" while still printing an answer to the first one. """ urls = [_TOKEN_LIST_URL] if _TOKEN_LIST_URL else list(_TOKEN_LIST_FALLBACKS) data = None errors: list[str] = [] for url in urls: try: req = urllib.request.Request( url, headers={"User-Agent": "garden-angel-discover/1.0"}) with urllib.request.urlopen(req, timeout=20) as resp: data = json.loads(resp.read()) if url != urls[0]: print(f" (token list came from the fallback {url})") break except Exception as exc: # noqa: BLE001 errors.append(f"{url} -> {str(exc)[:90]}") if data is None: print(f"\n {_R}could not fetch the token list{_0} — tried " f"{len(urls)} endpoint(s):") for line in errors: print(f" {line}") print(" Override with JUPITER_TOKEN_LIST_URL, or pass --mints.") print(" NOT falling back to the configured tokens — that would report") print(" 'no new markets' without having looked.\n") return [] rows = data if isinstance(data, list) else (data.get("tokens") or []) out: list[dict] = [] for t in rows: mint, sym = t.get("address") or t.get("mint"), t.get("symbol") dec = t.get("decimals") if not (mint and sym) or dec is None: continue # Daily volume where the list carries it: the order the list arrives # in is not a ranking, and scanning alphabetically wastes the budget # on whatever starts with 'A'. out.append({"symbol": str(sym), "mint": str(mint), "decimals": int(dec), "volume": float(t.get("daily_volume") or 0.0)}) out.sort(key=lambda r: -r["volume"]) return out[:limit] async def _quote(client, in_mint: str, out_mint: str, amount: int, venue: Optional[str], slippage: int = 50) -> Optional[int]: from constants import JUPITER_HEADERS, JUPITER_QUOTE_API params = {"inputMint": in_mint, "outputMint": out_mint, "amount": str(amount), "slippageBps": str(slippage), "swapMode": "ExactIn"} if venue: params["dexes"] = venue params["onlyDirectRoutes"] = "true" try: r = await client.get(f"{JUPITER_QUOTE_API}/quote", params=params, headers=JUPITER_HEADERS, timeout=15.0) if r.status_code != 200: return None out = r.json().get("outAmount") return int(out) if out else None except Exception: # noqa: BLE001 return None async def _round_trip(client, base: dict, mid: dict, size: float, buy_on: str, sell_on: str, pace: float) -> Optional[float]: """Walk it: base→mid on one venue, mid→base on another. bps of size. THE ONLY NUMBER THAT MEANS ANYTHING. A one-way comparison infers a gap; this executes the inference against the same quoter the bot trades on, paying both legs' impact instead of neither. """ amount_in = int(size * (10 ** base["decimals"])) leg1 = await _quote(client, base["mint"], mid["mint"], amount_in, buy_on) await asyncio.sleep(pace) if not leg1: return None leg2 = await _quote(client, mid["mint"], base["mint"], leg1, sell_on) await asyncio.sleep(pace) if not leg2: return None return ((leg2 - amount_in) / amount_in) * 10_000.0 async def _two_hop_round_trip( client, base: dict, mid: dict, via: dict, size: float, buy_on: str, sell_on: str, pace: float, ) -> Optional[float]: """base -> via -> mid on one venue, then mid -> base on another. v1.2 — TWO HOPS, BECAUSE ONE HOP IS NOT WHERE THE DISAGREEMENT LIVES. Operator: "multi-hop paths are essential... cap at 2 hops - fast and covers most real opportunities." The reason this matters here specifically: a direct base->mid pair is the most-watched quote on Solana, and the whole point of the per-venue screen is to find a venue that disagrees. On the majors, none do — 21 hours of /observe says so. But a route through an intermediate token prices TWO pools per leg, and the mispricing that has been arbitraged out of USDC/SOL has not necessarily been arbitraged out of USDC->JUP->SOL, because far fewer bots quote it. Capped at 2 by intent, not by laziness. Each extra hop multiplies the quote budget, adds its own impact and its own failure mode, and this deployment's Jupiter budget is already 85% used. Two hops is where the ratio of new ground covered to budget spent is best. Returns bps of `size`, or None if any leg has no route. """ amount_in = int(size * (10 ** base["decimals"])) leg1 = await _quote(client, base["mint"], via["mint"], amount_in, buy_on) await asyncio.sleep(pace) if not leg1: return None leg2 = await _quote(client, via["mint"], mid["mint"], leg1, buy_on) await asyncio.sleep(pace) if not leg2: return None leg3 = await _quote(client, mid["mint"], base["mint"], leg2, sell_on) await asyncio.sleep(pace) if not leg3: return None return ((leg3 - amount_in) / amount_in) * 10_000.0 def _build_ladder(rungs: list[float], probe: float) -> list[float]: """The ladder, with the screening size guaranteed to be its first rung. v1.1 — THE SCREEN AND THE LADDER WERE MEASURING DIFFERENT MARKETS. Screening happened at --probe (default 100). The ladder started at 250. So a pair whose edge is real at 100 and gone by 250 was found by the screen, failed the first rung, hit `if not best: continue`, and was dropped without ever being printed. The tool could only ever confirm an edge at a size it had never screened at, and every pair it rejected was rejected at a size 2.5x larger than the one that made it interesting. That is very likely the whole reason this reports "No pair survived" while /venues keeps showing 23.5 bps between Meteora and Orca V2: those gaps are real at small size, and the first question the ladder asked was about a size where they are not. Prepending the probe size makes the first rung the size the gap was actually observed at, so a survivor is confirmed at its own size and THEN walked up. Deduplicated and sorted, so passing --ladder with the probe already in it is harmless. """ return sorted({round(float(x), 6) for x in ([probe] + list(rungs)) if float(x) > 0}) async def main() -> int: ap = argparse.ArgumentParser(description=__doc__) ap.add_argument("--tokens", type=int, default=40, help="how many tokens to screen, by daily volume") ap.add_argument("--base", default="USDC") ap.add_argument("--probe", type=float, default=100.0, help="screening size — small, so the survey is cheap") ap.add_argument("--ladder", default="250,1000,2500,5000", help="sizes to walk a survivor up until it dies. The " "screening size is always prepended — see _build_ladder") ap.add_argument("--mints", default="", help="comma-separated SYMBOL:MINT:DECIMALS, skips the list") ap.add_argument("--yes", action="store_true", help="run even if the survey exceeds the remaining budget") ap.add_argument("--apply", action="store_true", help="pin what survives into SOLANA_EXTRA_ROUTES") ap.add_argument("--hops", type=int, default=1, choices=(1, 2), help="1 = base->mid->base (default). 2 = also try " "base->via->mid->base through --via tokens, where " "far fewer bots are quoting") ap.add_argument("--via", default="SOL,USDT,JUP", help="intermediate tokens for --hops 2") args = ap.parse_args() import httpx from constants import SOLANA_TOKENS base_meta = SOLANA_TOKENS.get(args.base) if not base_meta: print(f"unknown base {args.base}") return 1 base = {"symbol": args.base, **base_meta} if args.mints: cands = [] for spec in args.mints.split(","): parts = spec.split(":") if len(parts) == 3: cands.append({"symbol": parts[0], "mint": parts[1], "decimals": int(parts[2]), "volume": 0.0}) else: cands = _fetch_token_list(args.tokens) if not cands: return 1 haircut, _src = effective_haircut_bps() fee_bps = haircut + float(os.getenv("SOLANA_FLASH_FEE_BPS", "0.5") or 0.5) pace = max(0.25, float(os.getenv("SOLANA_JUPITER_MIN_INTERVAL_SECS", "0.3") or 0.3)) ladder = _build_ladder( [float(x) for x in args.ladder.split(",") if x.strip()], args.probe, ) # ── BUDGET GUARD ────────────────────────────────────────────────── # # This survey is not free and the budget is not ours alone. Screening # 40 tokens across 7 venues is 280 quotes before a single ladder is # walked, against an hourly cap of 5,600 that the operator's /doctor # reports at 5,552 used — 99%. Run blind, this survey would rate-limit # the live scanner it is meant to help, and the Jupiter key is shared # with their Rust bot on top of that. # # So the cost is stated first and consent is required above a share of # the cap. Not a silent cap: a survey that quietly screened 5 tokens # instead of 40 would report "no new markets" from a sample that could # never have found one. screen_cost = len(cands) * len(_VENUES) ladder_cost = len(ladder) * 2 est = screen_cost + ladder_cost * 3 # assume ~3 survivors walk print() print(f"{_B} budget{_0}: ~{est} Jupiter quote(s) " f"({screen_cost} screening + ladders)") try: from modules.route_scorer import get_scorer st = get_scorer().status() used, total = st.get("budget_used") or 0, st.get("budget_total") or 0 if total: free = max(0, total - used) print(f" {used:,}/{total:,} used this hour — {free:,} free") if est > free: print(f"\n {_R}This survey needs more quotes than are left this " f"hour.{_0}") print(f" Running it would rate-limit the live scanner, and the") print(f" key is shared with the Rust bot.") print(f"\n Either wait for the hour to roll, or lower --tokens to " f"about {max(1, free // len(_VENUES))}.") if not args.yes: print(" Pass --yes to run anyway.\n") return 1 print(f" {_Y}--yes given, proceeding anyway.{_0}") except Exception: # noqa: BLE001 — never block on the guard itself print(" (live budget unreadable — cost above is the estimate)") print() print(f"{_B} 🔭 discover — new markets, and the size each one dies at{_0}") print(f" {len(cands)} token(s) vs {base['symbol']} · screen at " f"{args.probe:,.0f} · fees {fee_bps:.2f} bps · ~{pace:.2f}s pacing") print(" " + "─" * 66) results: list[dict] = [] near_misses: list[dict] = [] screened = 0 async with httpx.AsyncClient() as client: for n, tok in enumerate(cands, 1): if tok["mint"] == base["mint"]: continue # 1 — per-venue price at the small screening size. prices: dict[str, float] = {} amount = int(args.probe * (10 ** base["decimals"])) for venue in _VENUES: out = await _quote(client, base["mint"], tok["mint"], amount, venue) await asyncio.sleep(pace) if out: prices[venue] = out / (10 ** tok["decimals"]) if len(prices) < 2: continue # Only counts once a real comparison was possible — a token only # one venue quoted was never screened, it was skipped, and # conflating the two is what let "no edge found" stand in for # "no venue answered". screened += 1 rich = max(prices, key=lambda v: prices[v]) # most mid per base -> BUY poor = min(prices, key=lambda v: prices[v]) # values mid highest -> SELL gap = (prices[rich] - prices[poor]) / prices[poor] * 10_000.0 if gap < fee_bps or gap > _ABSURD_GAP_BPS: continue # 2 — walk it up the ladder until it stops clearing its fees. best: Optional[dict] = None died_at: Optional[tuple[float, float]] = None # (size, net_bps) for size in ladder: rt = await _round_trip(client, base, tok, size, rich, poor, pace) if rt is None: died_at = (size, float("nan")) break net = rt - fee_bps if net <= 0: died_at = (size, net) break best = {"size": size, "roundtrip_bps": rt, "net_bps": net, "net_usd": size * net / 10_000.0} if not best: # v1.2 — before giving up on this pair, try it through an # intermediate. A direct pair that prices flat is exactly the # case where a two-hop path is worth the extra quotes: the # majors are efficiently arbitraged, the routes THROUGH them # are much less so. if args.hops == 2: for via_sym in [v.strip().upper() for v in args.via.split(",")]: via_meta = SOLANA_TOKENS.get(via_sym) if not via_meta or via_sym in (base["symbol"], tok["symbol"]): continue via = {"symbol": via_sym, **via_meta} rt2 = await _two_hop_round_trip( client, base, tok, via, ladder[0], rich, poor, pace) if rt2 is None: continue net2 = rt2 - fee_bps if net2 > 0: best = {"size": ladder[0], "roundtrip_bps": rt2, "net_bps": net2, "net_usd": ladder[0] * net2 / 10_000.0} print(f" {_G}✓{_0} {base['symbol']}-{tok['symbol']}" f" via {via_sym} · net {net2:+.2f} bps " f"(2-hop)") break if not best: # v1.1 — SAY SO, rather than dropping it in silence. # # This used to be a bare `continue`. A pair that showed a gap # at the screening size and then failed the first rung # vanished with no output at all, so a run that examined a # real disagreement and correctly rejected it looked exactly # like a run that found nothing to examine. "No pair # survived" was doing double duty for two completely # different findings, and the operator has been reading the # weaker one for 21 hours. # # The distinction matters: a screen gap that dies on the # first rung is price impact (the honest negative), while a # screen gap that dies on the THIRD rung is a real but # shallow market — a size problem, not an absence. if died_at is not None: size_d, net_d = died_at detail = ("no route" if net_d != net_d # NaN else f"net {net_d:+.2f} bps after {fee_bps:.2f} fees") print(f" {_Y}·{_0} {base['symbol']}-{tok['symbol']:<12} " f"screened {gap:+6.2f} bps, died at " f"{size_d:>7,.0f} ({detail})") near_misses.append({ "pair": f"{base['symbol']}-{tok['symbol']}", "gap_bps": round(gap, 2), "died_at": size_d, "buy_on": rich, "sell_on": poor, }) continue row = {"pair": f"{base['symbol']}-{tok['symbol']}", "mint": tok["mint"], "decimals": tok["decimals"], "buy_on": rich, "sell_on": poor, "gap_bps": round(gap, 2), "fees_bps": round(fee_bps, 2), "venues": len(prices), **{k: round(v, 4) if isinstance(v, float) else v for k, v in best.items()}} results.append(row) print(f" {_G}✓{_0} {row['pair']:<16} survives to {best['size']:>7,.0f} · " f"net {best['net_bps']:+6.2f} bps (${best['net_usd']:.2f}) · " f"BUY {rich} SELL {poor}") if n % 10 == 0: print(f" {_Y}·{_0} {n}/{len(cands)} screened, " f"{len(results)} survivor(s)") print(" " + "─" * 66) if not results: print() # v1.1 — three genuinely different findings that all used to print # the same sentence. Which one it is changes what to do next, so it # is now stated rather than left to be guessed. if screened == 0: print(f" {_R}Nothing was screened.{_0} No token returned two or more") print(" venue quotes, so no comparison was ever possible. This is a") print(" COVERAGE problem, not a market finding — check the venue list") print(" and the token list before concluding anything about edge.") elif not near_misses: print(f" No pair even showed a gap at {args.probe:,.0f} " f"{base['symbol']}.") print(f" {screened} pair(s) screened across {len(_VENUES)} venues and " f"every one priced") print(" within fees. The venues agree; there is nothing to arbitrage") print(" at this size, which is the honest answer.") else: first_rung = ladder[0] at_screen = [m for m in near_misses if m["died_at"] <= first_rung] deeper = [m for m in near_misses if m["died_at"] > first_rung] print(f" {len(near_misses)} pair(s) showed a gap and none survived a " f"walked round trip.") print() if at_screen: print(f" {len(at_screen)} died at the FIRST rung " f"({first_rung:,.0f}) — the gap does not survive being") print(" traded at the size it was found at. That is price") print(" impact, not edge.") if deeper: print(f" {len(deeper)} survived the first rung and died higher up " f"— these are REAL") print(" but shallow. The edge exists; the size does not. Worth") print(" re-running with a lower ladder to find where they live:") best_depth = max(m["died_at"] for m in deeper) print(f" --ladder {first_rung:,.0f},{best_depth:,.0f} " f"--probe {args.probe:,.0f}") print() return 0 results.sort(key=lambda r: -r["net_usd"]) out_dir = Path("data"); out_dir.mkdir(parents=True, exist_ok=True) path = out_dir / f"discover_{datetime.now(timezone.utc):%Y%m%d-%H%M%S}.csv" with path.open("w", newline="", encoding="utf-8") as fh: w = csv.DictWriter(fh, fieldnames=list(results[0])) w.writeheader(); w.writerows(results) print(f"\n {len(results)} market(s) survived. Written to {path}\n") for r in results[:10]: print(f" {r['pair']:<16} {r['net_usd']:>8.2f} USD at {r['size']:,.0f}") print(f" {'':<16} BUY {r['buy_on']} · SELL {r['sell_on']}") print() print(" Each line was WALKED — both legs quoted at that size, both impacts") print(" paid. The size shown is the largest on the ladder that still") print(" cleared its fees; the next rung up did not.") if args.apply: extra = os.getenv("SOLANA_EXTRA_TOKENS", "").strip() routes = os.getenv("SOLANA_EXTRA_ROUTES", "").strip() new_tok = ",".join(f"{r['pair'].split('-')[1]}:{r['mint']}:{r['decimals']}" for r in results[:5]) new_rt = ",".join(f"{base['symbol']}>{r['pair'].split('-')[1]}" for r in results[:5]) print() print(f" {_B}to pin the top {min(5, len(results))}:{_0}") print(f" ./scripts/setenv.sh \\") print(f" SOLANA_EXTRA_TOKENS={','.join(x for x in (extra, new_tok) if x)} \\") print(f" SOLANA_EXTRA_ROUTES={','.join(x for x in (routes, new_rt) if x)}") print(" then restart. --apply prints the command; it does not run it,") print(" because adding a route spends quote budget every cycle.") print() return 0 if __name__ == "__main__": raise SystemExit(asyncio.run(main()))