scripts/discover.py

#!/usr/bin/env python3
"""
scripts/discover.py — find NEW markets, and the size each one dies at.

    venv/bin/python scripts/discover.py                    # top 40 tokens
    venv/bin/python scripts/discover.py --tokens 100
    venv/bin/python scripts/discover.py --apply            # pin what survives

────────────────────────────────────────────────────────────────────────────
WHY THIS AND NOT scan_tokens.py
────────────────────────────────────────────────────────────────────────────
Operator: "improve test scan token and signal we need to see new markets and
tokens."

scan_tokens.py asks Jupiter for a round trip and reads the answer. Jupiter's
router is built to return the BEST path, which means it has already blended
the venues and erased the difference between them — the one place a real
spread can still be sitting. Every route this deployment has scanned that
way has come back negative, 185,000 times.

Their own /venues screen shows what that hides. One pair, one moment, seven
venues:

    Meteora DLMM  72.43     Raydium       72.36
    Raydium CLMM  72.41     Orca V2       72.26
    Whirlpool     72.40     Phoenix       71.99

Meteora against Orca V2 is 23.5 bps. Against 0.80 bps of fees. At 1 SOL.

So this asks the per-venue question across MANY tokens, and then — because
a gap at 1 SOL means nothing — walks each survivor up a size ladder until
it dies, and reports the largest size that still clears its fees.

────────────────────────────────────────────────────────────────────────────
WHAT IT REFUSES TO DO
────────────────────────────────────────────────────────────────────────────
It never reports a one-way gap as an opportunity. #149 shipped exactly that
mistake and printed "$331,396" next to a venue with no order book. Every
number here comes from a walked round trip: base→mid on the buy venue,
mid→base on the sell venue, at that size, paying both legs' impact.

A gap over _ABSURD_GAP_BPS is reported as an illiquid venue and never as an
edge, for the same reason.

Read-only. Quotes only. No key is loaded, nothing is signed or sent.
"""

from __future__ import annotations

import argparse
import asyncio
import csv
import json
import os
import statistics
import sys
import time
import urllib.request
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Optional

sys.path.insert(0, str(Path(__file__).resolve().parent.parent))

from modules.env_file import effective_haircut_bps, load_env_file

load_env_file()

_G, _Y, _R, _B, _0 = "\033[32m", "\033[33m", "\033[31m", "\033[1m", "\033[0m"

# Venues deep enough on major pairs to be worth a separate quote. A venue
# nobody routes through cannot disagree with anyone in a way you can trade.
_VENUES = ["Whirlpool", "Orca V2", "Meteora DLMM", "Raydium",
           "Raydium CLMM", "Lifinity V2", "Phoenix"]

# Same threshold scan_crossvenue uses. A one-way gap above this on a pair
# anyone trades is a venue with no book at that size, not an edge.
_ABSURD_GAP_BPS = 300.0

_TOKEN_LIST_URL = os.getenv(
    "JUPITER_TOKEN_LIST_URL", "https://tokens.jup.ag/tokens?tags=verified")


def _fetch_token_list(limit: int) -> list[dict]:
    """Jupiter's verified token list, newest markets included.

    Fails LOUDLY. A discovery tool that silently falls back to the tokens
    already configured would report "nothing new found" while never having
    looked — the same false negative that has cost this project days.
    """
    try:
        req = urllib.request.Request(
            _TOKEN_LIST_URL, headers={"User-Agent": "garden-angel-discover/1.0"})
        with urllib.request.urlopen(req, timeout=20) as resp:
            data = json.loads(resp.read())
    except Exception as exc:  # noqa: BLE001
        print(f"\n  {_R}could not fetch the token list{_0}: {str(exc)[:140]}")
        print(f"  url: {_TOKEN_LIST_URL}")
        print("  Override with JUPITER_TOKEN_LIST_URL, or pass --mints.")
        print("  NOT falling back to the configured tokens — that would report")
        print("  'no new markets' without having looked.\n")
        return []

    rows = data if isinstance(data, list) else (data.get("tokens") or [])
    out: list[dict] = []
    for t in rows:
        mint, sym = t.get("address") or t.get("mint"), t.get("symbol")
        dec = t.get("decimals")
        if not (mint and sym) or dec is None:
            continue
        # Daily volume where the list carries it: the order the list arrives
        # in is not a ranking, and scanning alphabetically wastes the budget
        # on whatever starts with 'A'.
        out.append({"symbol": str(sym), "mint": str(mint), "decimals": int(dec),
                    "volume": float(t.get("daily_volume") or 0.0)})
    out.sort(key=lambda r: -r["volume"])
    return out[:limit]


async def _quote(client, in_mint: str, out_mint: str, amount: int,
                 venue: Optional[str], slippage: int = 50) -> Optional[int]:
    from constants import JUPITER_HEADERS, JUPITER_QUOTE_API

    params = {"inputMint": in_mint, "outputMint": out_mint,
              "amount": str(amount), "slippageBps": str(slippage),
              "swapMode": "ExactIn"}
    if venue:
        params["dexes"] = venue
        params["onlyDirectRoutes"] = "true"
    try:
        r = await client.get(f"{JUPITER_QUOTE_API}/quote", params=params,
                             headers=JUPITER_HEADERS, timeout=15.0)
        if r.status_code != 200:
            return None
        out = r.json().get("outAmount")
        return int(out) if out else None
    except Exception:  # noqa: BLE001
        return None


async def _round_trip(client, base: dict, mid: dict, size: float,
                      buy_on: str, sell_on: str, pace: float) -> Optional[float]:
    """Walk it: base→mid on one venue, mid→base on another. bps of size.

    THE ONLY NUMBER THAT MEANS ANYTHING. A one-way comparison infers a gap;
    this executes the inference against the same quoter the bot trades on,
    paying both legs' impact instead of neither.
    """
    amount_in = int(size * (10 ** base["decimals"]))
    leg1 = await _quote(client, base["mint"], mid["mint"], amount_in, buy_on)
    await asyncio.sleep(pace)
    if not leg1:
        return None
    leg2 = await _quote(client, mid["mint"], base["mint"], leg1, sell_on)
    await asyncio.sleep(pace)
    if not leg2:
        return None
    return ((leg2 - amount_in) / amount_in) * 10_000.0


async def _two_hop_round_trip(
    client, base: dict, mid: dict, via: dict, size: float,
    buy_on: str, sell_on: str, pace: float,
) -> Optional[float]:
    """base -> via -> mid on one venue, then mid -> base on another.

    v1.2 — TWO HOPS, BECAUSE ONE HOP IS NOT WHERE THE DISAGREEMENT LIVES.

    Operator: "multi-hop paths are essential... cap at 2 hops - fast and
    covers most real opportunities."

    The reason this matters here specifically: a direct base->mid pair is
    the most-watched quote on Solana, and the whole point of the per-venue
    screen is to find a venue that disagrees. On the majors, none do — 21
    hours of /observe says so. But a route through an intermediate token
    prices TWO pools per leg, and the mispricing that has been arbitraged
    out of USDC/SOL has not necessarily been arbitraged out of
    USDC->JUP->SOL, because far fewer bots quote it.

    Capped at 2 by intent, not by laziness. Each extra hop multiplies the
    quote budget, adds its own impact and its own failure mode, and this
    deployment's Jupiter budget is already 85% used. Two hops is where the
    ratio of new ground covered to budget spent is best.

    Returns bps of `size`, or None if any leg has no route.
    """
    amount_in = int(size * (10 ** base["decimals"]))
    leg1 = await _quote(client, base["mint"], via["mint"], amount_in, buy_on)
    await asyncio.sleep(pace)
    if not leg1:
        return None
    leg2 = await _quote(client, via["mint"], mid["mint"], leg1, buy_on)
    await asyncio.sleep(pace)
    if not leg2:
        return None
    leg3 = await _quote(client, mid["mint"], base["mint"], leg2, sell_on)
    await asyncio.sleep(pace)
    if not leg3:
        return None
    return ((leg3 - amount_in) / amount_in) * 10_000.0


def _build_ladder(rungs: list[float], probe: float) -> list[float]:
    """The ladder, with the screening size guaranteed to be its first rung.

    v1.1 — THE SCREEN AND THE LADDER WERE MEASURING DIFFERENT MARKETS.

    Screening happened at --probe (default 100). The ladder started at 250.
    So a pair whose edge is real at 100 and gone by 250 was found by the
    screen, failed the first rung, hit `if not best: continue`, and was
    dropped without ever being printed. The tool could only ever confirm an
    edge at a size it had never screened at, and every pair it rejected was
    rejected at a size 2.5x larger than the one that made it interesting.

    That is very likely the whole reason this reports "No pair survived"
    while /venues keeps showing 23.5 bps between Meteora and Orca V2: those
    gaps are real at small size, and the first question the ladder asked
    was about a size where they are not.

    Prepending the probe size makes the first rung the size the gap was
    actually observed at, so a survivor is confirmed at its own size and
    THEN walked up. Deduplicated and sorted, so passing --ladder with the
    probe already in it is harmless.
    """
    return sorted({round(float(x), 6) for x in ([probe] + list(rungs)) if float(x) > 0})


async def main() -> int:
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("--tokens", type=int, default=40,
                    help="how many tokens to screen, by daily volume")
    ap.add_argument("--base", default="USDC")
    ap.add_argument("--probe", type=float, default=100.0,
                    help="screening size — small, so the survey is cheap")
    ap.add_argument("--ladder", default="250,1000,2500,5000",
                    help="sizes to walk a survivor up until it dies. The "
                         "screening size is always prepended — see _build_ladder")
    ap.add_argument("--mints", default="",
                    help="comma-separated SYMBOL:MINT:DECIMALS, skips the list")
    ap.add_argument("--yes", action="store_true",
                    help="run even if the survey exceeds the remaining budget")
    ap.add_argument("--apply", action="store_true",
                    help="pin what survives into SOLANA_EXTRA_ROUTES")
    ap.add_argument("--hops", type=int, default=1, choices=(1, 2),
                    help="1 = base->mid->base (default). 2 = also try "
                         "base->via->mid->base through --via tokens, where "
                         "far fewer bots are quoting")
    ap.add_argument("--via", default="SOL,USDT,JUP",
                    help="intermediate tokens for --hops 2")
    args = ap.parse_args()

    import httpx

    from constants import SOLANA_TOKENS

    base_meta = SOLANA_TOKENS.get(args.base)
    if not base_meta:
        print(f"unknown base {args.base}")
        return 1
    base = {"symbol": args.base, **base_meta}

    if args.mints:
        cands = []
        for spec in args.mints.split(","):
            parts = spec.split(":")
            if len(parts) == 3:
                cands.append({"symbol": parts[0], "mint": parts[1],
                              "decimals": int(parts[2]), "volume": 0.0})
    else:
        cands = _fetch_token_list(args.tokens)
    if not cands:
        return 1

    haircut, _src = effective_haircut_bps()
    fee_bps = haircut + float(os.getenv("SOLANA_FLASH_FEE_BPS", "0.5") or 0.5)
    pace = max(0.25, float(os.getenv("SOLANA_JUPITER_MIN_INTERVAL_SECS", "0.3") or 0.3))
    ladder = _build_ladder(
        [float(x) for x in args.ladder.split(",") if x.strip()], args.probe,
    )

    # ── BUDGET GUARD ──────────────────────────────────────────────────
    #
    # This survey is not free and the budget is not ours alone. Screening
    # 40 tokens across 7 venues is 280 quotes before a single ladder is
    # walked, against an hourly cap of 5,600 that the operator's /doctor
    # reports at 5,552 used — 99%. Run blind, this survey would rate-limit
    # the live scanner it is meant to help, and the Jupiter key is shared
    # with their Rust bot on top of that.
    #
    # So the cost is stated first and consent is required above a share of
    # the cap. Not a silent cap: a survey that quietly screened 5 tokens
    # instead of 40 would report "no new markets" from a sample that could
    # never have found one.
    screen_cost = len(cands) * len(_VENUES)
    ladder_cost = len(ladder) * 2
    est = screen_cost + ladder_cost * 3  # assume ~3 survivors walk
    print()
    print(f"{_B}  budget{_0}: ~{est} Jupiter quote(s) "
          f"({screen_cost} screening + ladders)")
    try:
        from modules.route_scorer import get_scorer

        st = get_scorer().status()
        used, total = st.get("budget_used") or 0, st.get("budget_total") or 0
        if total:
            free = max(0, total - used)
            print(f"          {used:,}/{total:,} used this hour — {free:,} free")
            if est > free:
                print(f"\n  {_R}This survey needs more quotes than are left this "
                      f"hour.{_0}")
                print(f"  Running it would rate-limit the live scanner, and the")
                print(f"  key is shared with the Rust bot.")
                print(f"\n  Either wait for the hour to roll, or lower --tokens to "
                      f"about {max(1, free // len(_VENUES))}.")
                if not args.yes:
                    print("  Pass --yes to run anyway.\n")
                    return 1
                print(f"  {_Y}--yes given, proceeding anyway.{_0}")
    except Exception:  # noqa: BLE001 — never block on the guard itself
        print("          (live budget unreadable — cost above is the estimate)")

    print()
    print(f"{_B}  🔭 discover — new markets, and the size each one dies at{_0}")
    print(f"  {len(cands)} token(s) vs {base['symbol']} · screen at "
          f"{args.probe:,.0f} · fees {fee_bps:.2f} bps · ~{pace:.2f}s pacing")
    print("  " + "─" * 66)

    results: list[dict] = []
    near_misses: list[dict] = []
    screened = 0
    async with httpx.AsyncClient() as client:
        for n, tok in enumerate(cands, 1):
            if tok["mint"] == base["mint"]:
                continue

            # 1 — per-venue price at the small screening size.
            prices: dict[str, float] = {}
            amount = int(args.probe * (10 ** base["decimals"]))
            for venue in _VENUES:
                out = await _quote(client, base["mint"], tok["mint"], amount, venue)
                await asyncio.sleep(pace)
                if out:
                    prices[venue] = out / (10 ** tok["decimals"])
            if len(prices) < 2:
                continue
            # Only counts once a real comparison was possible — a token only
            # one venue quoted was never screened, it was skipped, and
            # conflating the two is what let "no edge found" stand in for
            # "no venue answered".
            screened += 1

            rich = max(prices, key=lambda v: prices[v])   # most mid per base -> BUY
            poor = min(prices, key=lambda v: prices[v])   # values mid highest -> SELL
            gap = (prices[rich] - prices[poor]) / prices[poor] * 10_000.0
            if gap < fee_bps or gap > _ABSURD_GAP_BPS:
                continue

            # 2 — walk it up the ladder until it stops clearing its fees.
            best: Optional[dict] = None
            died_at: Optional[tuple[float, float]] = None   # (size, net_bps)
            for size in ladder:
                rt = await _round_trip(client, base, tok, size, rich, poor, pace)
                if rt is None:
                    died_at = (size, float("nan"))
                    break
                net = rt - fee_bps
                if net <= 0:
                    died_at = (size, net)
                    break
                best = {"size": size, "roundtrip_bps": rt, "net_bps": net,
                        "net_usd": size * net / 10_000.0}
            if not best:
                # v1.2 — before giving up on this pair, try it through an
                # intermediate. A direct pair that prices flat is exactly the
                # case where a two-hop path is worth the extra quotes: the
                # majors are efficiently arbitraged, the routes THROUGH them
                # are much less so.
                if args.hops == 2:
                    for via_sym in [v.strip().upper() for v in args.via.split(",")]:
                        via_meta = SOLANA_TOKENS.get(via_sym)
                        if not via_meta or via_sym in (base["symbol"], tok["symbol"]):
                            continue
                        via = {"symbol": via_sym, **via_meta}
                        rt2 = await _two_hop_round_trip(
                            client, base, tok, via, ladder[0], rich, poor, pace)
                        if rt2 is None:
                            continue
                        net2 = rt2 - fee_bps
                        if net2 > 0:
                            best = {"size": ladder[0], "roundtrip_bps": rt2,
                                    "net_bps": net2,
                                    "net_usd": ladder[0] * net2 / 10_000.0}
                            print(f"  {_G}✓{_0} {base['symbol']}-{tok['symbol']}"
                                  f" via {via_sym} · net {net2:+.2f} bps "
                                  f"(2-hop)")
                            break

            if not best:
                # v1.1 — SAY SO, rather than dropping it in silence.
                #
                # This used to be a bare `continue`. A pair that showed a gap
                # at the screening size and then failed the first rung
                # vanished with no output at all, so a run that examined a
                # real disagreement and correctly rejected it looked exactly
                # like a run that found nothing to examine. "No pair
                # survived" was doing double duty for two completely
                # different findings, and the operator has been reading the
                # weaker one for 21 hours.
                #
                # The distinction matters: a screen gap that dies on the
                # first rung is price impact (the honest negative), while a
                # screen gap that dies on the THIRD rung is a real but
                # shallow market — a size problem, not an absence.
                if died_at is not None:
                    size_d, net_d = died_at
                    detail = ("no route" if net_d != net_d      # NaN
                              else f"net {net_d:+.2f} bps after {fee_bps:.2f} fees")
                    print(f"  {_Y}·{_0} {base['symbol']}-{tok['symbol']:<12} "
                          f"screened {gap:+6.2f} bps, died at "
                          f"{size_d:>7,.0f} ({detail})")
                    near_misses.append({
                        "pair": f"{base['symbol']}-{tok['symbol']}",
                        "gap_bps": round(gap, 2), "died_at": size_d,
                        "buy_on": rich, "sell_on": poor,
                    })
                continue

            row = {"pair": f"{base['symbol']}-{tok['symbol']}",
                   "mint": tok["mint"], "decimals": tok["decimals"],
                   "buy_on": rich, "sell_on": poor,
                   "gap_bps": round(gap, 2), "fees_bps": round(fee_bps, 2),
                   "venues": len(prices), **{k: round(v, 4) if isinstance(v, float) else v
                                             for k, v in best.items()}}
            results.append(row)
            print(f"  {_G}✓{_0} {row['pair']:<16} survives to {best['size']:>7,.0f} · "
                  f"net {best['net_bps']:+6.2f} bps (${best['net_usd']:.2f}) · "
                  f"BUY {rich} SELL {poor}")

            if n % 10 == 0:
                print(f"  {_Y}·{_0} {n}/{len(cands)} screened, "
                      f"{len(results)} survivor(s)")

    print("  " + "─" * 66)
    if not results:
        print()
        # v1.1 — three genuinely different findings that all used to print
        # the same sentence. Which one it is changes what to do next, so it
        # is now stated rather than left to be guessed.
        if screened == 0:
            print(f"  {_R}Nothing was screened.{_0} No token returned two or more")
            print("  venue quotes, so no comparison was ever possible. This is a")
            print("  COVERAGE problem, not a market finding — check the venue list")
            print("  and the token list before concluding anything about edge.")
        elif not near_misses:
            print(f"  No pair even showed a gap at {args.probe:,.0f} "
                  f"{base['symbol']}.")
            print(f"  {screened} pair(s) screened across {len(_VENUES)} venues and "
                  f"every one priced")
            print("  within fees. The venues agree; there is nothing to arbitrage")
            print("  at this size, which is the honest answer.")
        else:
            first_rung = ladder[0]
            at_screen = [m for m in near_misses if m["died_at"] <= first_rung]
            deeper = [m for m in near_misses if m["died_at"] > first_rung]
            print(f"  {len(near_misses)} pair(s) showed a gap and none survived a "
                  f"walked round trip.")
            print()
            if at_screen:
                print(f"    {len(at_screen)} died at the FIRST rung "
                      f"({first_rung:,.0f}) — the gap does not survive being")
                print("      traded at the size it was found at. That is price")
                print("      impact, not edge.")
            if deeper:
                print(f"    {len(deeper)} survived the first rung and died higher up "
                      f"— these are REAL")
                print("      but shallow. The edge exists; the size does not. Worth")
                print("      re-running with a lower ladder to find where they live:")
                best_depth = max(m["died_at"] for m in deeper)
                print(f"        --ladder {first_rung:,.0f},{best_depth:,.0f} "
                      f"--probe {args.probe:,.0f}")
        print()
        return 0

    results.sort(key=lambda r: -r["net_usd"])
    out_dir = Path("data"); out_dir.mkdir(parents=True, exist_ok=True)
    path = out_dir / f"discover_{datetime.now(timezone.utc):%Y%m%d-%H%M%S}.csv"
    with path.open("w", newline="", encoding="utf-8") as fh:
        w = csv.DictWriter(fh, fieldnames=list(results[0]))
        w.writeheader(); w.writerows(results)

    print(f"\n  {len(results)} market(s) survived. Written to {path}\n")
    for r in results[:10]:
        print(f"    {r['pair']:<16} {r['net_usd']:>8.2f} USD at {r['size']:,.0f}")
        print(f"    {'':<16} BUY {r['buy_on']} · SELL {r['sell_on']}")
    print()
    print("  Each line was WALKED — both legs quoted at that size, both impacts")
    print("  paid. The size shown is the largest on the ladder that still")
    print("  cleared its fees; the next rung up did not.")

    if args.apply:
        extra = os.getenv("SOLANA_EXTRA_TOKENS", "").strip()
        routes = os.getenv("SOLANA_EXTRA_ROUTES", "").strip()
        new_tok = ",".join(f"{r['pair'].split('-')[1]}:{r['mint']}:{r['decimals']}"
                           for r in results[:5])
        new_rt = ",".join(f"{base['symbol']}>{r['pair'].split('-')[1]}"
                          for r in results[:5])
        print()
        print(f"  {_B}to pin the top {min(5, len(results))}:{_0}")
        print(f"    ./scripts/setenv.sh \\")
        print(f"      SOLANA_EXTRA_TOKENS={','.join(x for x in (extra, new_tok) if x)} \\")
        print(f"      SOLANA_EXTRA_ROUTES={','.join(x for x in (routes, new_rt) if x)}")
        print("    then restart. --apply prints the command; it does not run it,")
        print("    because adding a route spends quote budget every cycle.")
    print()
    return 0


if __name__ == "__main__":
    raise SystemExit(asyncio.run(main()))