tests/polymath_daemon.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
XERO POLYMATH DAEMON — auto-feed diversified knowledge, evolve continuously.

    PYTHONPATH=modules python3 tests/polymath_daemon.py [--minutes M] [--max-units N] [--batch B]

Rotates across every registered public database (science, medicine, genetics,
chemistry, art, history, literature, poetry, geography, biodiversity, wisdom),
alternating sources so none is rate-limited, and feeds each fresh unit into the
organism — which integrates it and applies an evolutionary nudge. Diversified
data every cycle; the organism becomes a polymath. Fully RESUMABLE
(testing_logs/HARVEST_STATE.json + POLYMATH_PROGRESS.json) and self-throttling,
so it can run "at full blast" for days without exhausting a single API.

Defaults to running until every database is exhausted (the infinite ones never
are — so it runs until you stop it). Ctrl-C is safe; it flushes and resumes.
"""
from __future__ import annotations

import json
import os
import sys
import time

ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, os.path.join(ROOT, "modules"))
PROG = os.path.join(ROOT, "testing_logs", "POLYMATH_PROGRESS.json")
DONE = os.path.join(ROOT, "testing_logs", "POLYMATH_DONE.json")

from xero_polymath import PolymathHarvester
from xero_curriculum import Culturer
from xero_observer import Observer
from xero_storage import manager
from xero_autonomy import StagnationWatch


def _flag(name, default=None):
    if name in sys.argv:
        i = sys.argv.index(name)
        return sys.argv[i + 1] if i + 1 < len(sys.argv) else True
    return default


MINUTES = float(_flag("--minutes", 0) or 0)
MAX_UNITS = int(_flag("--max-units", 0) or 0)
BATCH = int(_flag("--batch", 6) or 6)


def _resume_cycle() -> int:
    try:
        return int(json.load(open(PROG)).get("cycle", 0))
    except Exception:
        return 0


def main() -> int:
    h = PolymathHarvester(batch=BATCH)
    cult = Culturer(online=False)            # we feed it directly via .learn()
    obs = Observer(periodic_every=400, milestone_every=2000)
    sm = manager(budget_gb=float(os.environ.get("XERO_BUDGET_GB", "64")))
    watch = StagnationWatch(window=12, min_rel_gain=0.003)

    cycle = _resume_cycle()
    domains_seen: dict[str, int] = {}
    t0 = time.time()
    deadline = t0 + MINUTES * 60 if MINUTES else 0
    print(f"POLYMATH DAEMON · sources={len(h.state)} · batch={BATCH} · "
          f"{'forever' if not MINUTES else f'{MINUTES}min'} · resume@{cycle}")

    ticks = 0
    try:
        while True:
            if deadline and time.time() > deadline:
                print("time limit reached."); break
            if MAX_UNITS and cycle >= MAX_UNITS:
                print("unit target reached."); break
            r = h.tick()
            ticks += 1
            if r.get("status") == "exhausted":
                print("ALL DATABASES EXHAUSTED."); break
            if r.get("status") != "ok":
                continue
            domains_seen[r["domain"]] = domains_seen.get(r["domain"], 0) + r["new"]
            for u in r["new_units"]:
                cult.learn(u, cycle, iteration=0)
                cycle += 1
            watch.observe(cult.fitness)

            if ticks % 8 == 0:
                cult.flush()
                guard = sm.enforce()
                summary = h.summary()
                snap = {"cycle": cycle, "fitness": round(cult.fitness, 4),
                        "anomaly": watch.is_stagnant(),
                        "unique_units_learned": len(cult.learned),
                        "traditions_count": len({v.get("tradition") for v in cult.learned.values()}),
                        "domains": domains_seen, "harvest": summary,
                        "rate_cps": round(cycle / max(1e-9, time.time() - t0), 2)}
                rep = obs.observe(snap)
                with open(PROG, "w") as f:
                    json.dump({**snap, "elapsed_s": round(time.time() - t0, 1),
                               "sources_exhausted": summary["exhausted"],
                               "total_fetched": summary["total_fetched"]}, f, indent=2, default=str)
                tag = "POKE" if snap["anomaly"] else r["domain"]
                print(f"  cycle {cycle} · {tag} +{r['new']} ({r['source']}) · fit={snap['fitness']} · "
                      f"units={snap['unique_units_learned']} · fetched={summary['total_fetched']}"
                      + (f" · {rep['kind']} report" if rep else ""))
    except KeyboardInterrupt:
        print("\nstopping (state saved; resume any time)…")

    cult.flush(); sm.snapshot_learned()
    summ = h.summary()
    final = {"organism": "XERO", "cycle": cycle, "fitness": round(cult.fitness, 4),
             "unique_units_learned": len(cult.learned), "domains": domains_seen,
             "harvest": summ, "elapsed_s": round(time.time() - t0, 1),
             "sources_exhausted": summ["exhausted"], "total_fetched": summ["total_fetched"],
             "generated": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())}
    with open(PROG, "w") as f:
        json.dump(final, f, indent=2, default=str)
    if h.all_exhausted():
        with open(DONE, "w") as f:
            json.dump(final, f, indent=2, default=str)
    print(f"DONE · {cycle} cycles · fitness {round(cult.fitness, 4)} · "
          f"{len(cult.learned)} units · fetched {summ['total_fetched']} across "
          f"{len([d for d in summ['by_domain'] if summ['by_domain'][d]])} domains")
    return 0


if __name__ == "__main__":
    sys.exit(main())