tools/aggregate.py

#!/usr/bin/env python3
"""aggregate.py — VADUGWI word-rating review report (offline, human-reviewed).

Read the private dataset `deucebucket/vadugwi-data` (anonymous word/scenario
ratings collected by the Space) and produce a human-review report of CANDIDATE
corrections. NOTHING here is auto-applied to clanker — the engine's weights are
genetically-optimized champions and FP-zero is non-negotiable. A human adopts any
deltas deliberately.

Two candidate sets (spec §11.4 / §11.7):
  KNOWN words   -> weight-correction candidates: per word, mean human user_v vs
                   engine compute_vadug(word).v, ranked by |systematic disagreement|.
  UNKNOWN words -> new-vocabulary candidates: per word, mean human rating mapped
                   via SCALE_MAP to a candidate valence (0-255), expressed as a
                   candidate dv vs the 128 neutral center, with sample size and
                   agreement (stddev).

Writes tools/aggregate_report.md. If the dataset is empty, says so and exits
cleanly (expected until the Space starts collecting submissions).

Spec: docs/.../2026-06-17-vadugwi-read-the-room-design.md §11.4, §11.7.
"""
import json
import math
import os
import sys
from collections import defaultdict

CLANKER = "/home/deucebucket/ai-drive/clanker"
sys.path.insert(0, CLANKER)

HERE = os.path.dirname(os.path.abspath(__file__))
REPO = os.path.dirname(HERE)
REPORT = os.path.join(HERE, "aggregate_report.md")

DATA_REPO = "deucebucket/vadugwi-data"

# Mirror of compute.js SCALE_MAP (-6..+6 -> 0..255, 128 = neutral center).
SCALE_MAP = {
    -6: 0, -5: 0, -4: 51, -3: 102, -2: 128, -1: 140,
    0: 153, 1: 166, 2: 179, 3: 204, 4: 230, 5: 255, 6: 255,
}
NEUTRAL = 128


def scale(raw):
    try:
        return SCALE_MAP[int(round(float(raw)))]
    except (ValueError, TypeError, KeyError):
        return None


def load_rows():
    """Return list of word-rating rows, or None if the dataset is unreachable/empty.

    Each row of interest: {word, user_rating, source?, lexicon_v?}.
    """
    try:
        from huggingface_hub import HfApi, hf_hub_download
    except Exception as e:
        print(f"huggingface_hub unavailable: {e}")
        return None
    try:
        api = HfApi()
        info = api.dataset_info(DATA_REPO, files_metadata=True)
    except Exception as e:
        print(f"{DATA_REPO} unreachable: {type(e).__name__}: {e}")
        return None

    data_files = [s.rfilename for s in info.siblings
                  if s.rfilename.endswith((".json", ".jsonl"))
                  and not s.rfilename.startswith(".")]
    if not data_files:
        print(f"{DATA_REPO}: no data files yet (empty dataset).")
        return []

    rows = []
    for fn in data_files:
        try:
            path = hf_hub_download(DATA_REPO, fn, repo_type="dataset")
        except Exception as e:
            print(f"download {fn} failed: {e}")
            continue
        rows.extend(_iter_rows(path))
    return rows


def _iter_rows(path):
    out = []
    try:
        if path.endswith(".jsonl"):
            with open(path) as f:
                for line in f:
                    line = line.strip()
                    if line:
                        out.append(json.loads(line))
        else:
            data = json.load(open(path))
            out = data if isinstance(data, list) else data.get("rows", [])
    except Exception as e:
        print(f"parse {os.path.basename(path)} failed: {e}")
    # keep only word rows (carry a `word` + a rating)
    keep = []
    for r in out:
        if not isinstance(r, dict):
            continue
        if "word" not in r:
            continue
        if r.get("user_rating") is None and r.get("rating") is None:
            continue
        keep.append(r)
    return keep


def engine_valence(word):
    from engine.pendulum import compute_vadug
    result, _ = compute_vadug(word)
    return result.v


def mean(xs):
    return sum(xs) / len(xs) if xs else float("nan")


def stdev(xs):
    if len(xs) < 2:
        return 0.0
    m = mean(xs)
    return math.sqrt(sum((x - m) ** 2 for x in xs) / (len(xs) - 1))


def write_empty_report(reason):
    lines = [
        "# VADUGWI aggregate report",
        "",
        f"_Dataset_: `{DATA_REPO}`",
        "",
        f"**No ratings to aggregate yet.** {reason}",
        "",
        "This is expected before the Space begins collecting anonymous "
        "submissions. Re-run `python3 tools/aggregate.py` once rows exist.",
        "",
        "All outputs of this tool are CANDIDATES for human review and are never "
        "auto-applied to the clanker engine.",
    ]
    with open(REPORT, "w") as f:
        f.write("\n".join(lines) + "\n")
    print(f"Wrote empty report -> {REPORT}")


def main():
    rows = load_rows()
    if rows is None:
        write_empty_report(
            "The dataset could not be reached (auth/network) — nothing aggregated."
        )
        return
    if not rows:
        write_empty_report("The dataset is empty (no submissions logged).")
        return

    # group ratings by (word, source)
    known = defaultdict(list)    # word -> [user_v, ...]
    unknown = defaultdict(list)  # word -> [user_v, ...]
    raw_by_word = defaultdict(list)  # word -> [raw rating, ...] (unknown candidates)

    for r in rows:
        word = str(r.get("word", "")).strip().lower()
        if not word:
            continue
        raw = r.get("user_rating", r.get("rating"))
        uv = scale(raw)
        if uv is None:
            continue
        source = (r.get("source") or "").lower()
        if source == "unknown":
            unknown[word].append(uv)
            raw_by_word[word].append(float(raw))
        else:
            known[word].append(uv)

    # KNOWN: human mean vs engine valence
    known_rows = []
    for word, uvs in known.items():
        try:
            ev = engine_valence(word)
        except Exception:
            continue
        hm = mean(uvs)
        known_rows.append({
            "word": word, "n": len(uvs), "human_v": hm,
            "engine_v": ev, "disagree": hm - ev, "sd": stdev(uvs),
        })
    known_rows.sort(key=lambda d: -abs(d["disagree"]))

    # UNKNOWN: human mean -> candidate dv vs 128
    unknown_rows = []
    for word, uvs in unknown.items():
        hm = mean(uvs)
        unknown_rows.append({
            "word": word, "n": len(uvs), "human_v": hm,
            "cand_dv": hm - NEUTRAL, "sd": stdev(uvs),
        })
    unknown_rows.sort(key=lambda d: (-d["n"], -abs(d["cand_dv"])))

    _write_report(rows, known_rows, unknown_rows)


def _write_report(rows, known_rows, unknown_rows):
    L = []
    L.append("# VADUGWI aggregate report")
    L.append("")
    L.append(f"_Dataset_: `{DATA_REPO}`  ·  word rows aggregated: {len(rows)}")
    L.append("")
    L.append("> All entries below are **CANDIDATES for human review**. Nothing is "
             "auto-applied to the clanker engine — weights are genetically-optimized "
             "champions and FP-zero is non-negotiable. A human adopts deltas "
             "deliberately into `engine/forces_curated.py`.")
    L.append("")

    L.append("## Known words — weight-correction candidates")
    L.append("")
    L.append("Mean human valence vs engine `compute_vadug(word).v`, ranked by "
             "|systematic disagreement|. Large positive disagree = humans read the "
             "word warmer than the lexicon; negative = colder.")
    L.append("")
    if known_rows:
        L.append("| word | n | human_v | engine_v | disagree | agree(sd) |")
        L.append("|------|---|---------|----------|----------|-----------|")
        for d in known_rows[:100]:
            L.append(f"| {d['word']} | {d['n']} | {d['human_v']:.0f} | "
                     f"{d['engine_v']:.0f} | {d['disagree']:+.0f} | {d['sd']:.0f} |")
    else:
        L.append("_No known-word ratings yet._")
    L.append("")

    L.append("## Unknown words — new-vocabulary candidates")
    L.append("")
    L.append("Words the engine had no entry for. Mean human rating mapped via "
             "SCALE_MAP to a candidate valence (0-255), expressed as a candidate "
             "**dv vs 128** (the neutral center). Larger sample + smaller sd = "
             "more trustworthy.")
    L.append("")
    if unknown_rows:
        L.append("| word | n | human_v | candidate dv | agree(sd) |")
        L.append("|------|---|---------|--------------|-----------|")
        for d in unknown_rows[:100]:
            L.append(f"| {d['word']} | {d['n']} | {d['human_v']:.0f} | "
                     f"{d['cand_dv']:+.0f} | {d['sd']:.0f} |")
    else:
        L.append("_No unknown-word ratings yet._")
    L.append("")
    L.append("---")
    L.append("Candidate dv is a valence-only proposal; the other axes "
             "(da,dd,du,dg) still require human judgment before any vocabulary edit.")

    with open(REPORT, "w") as f:
        f.write("\n".join(L) + "\n")
    print(f"Wrote report ({len(known_rows)} known, {len(unknown_rows)} unknown "
          f"candidates) -> {REPORT}")


if __name__ == "__main__":
    main()