Blame for eval_m1.py

"""M1 Competition 111 forecasting eval (macro MAPE) for byte-level spark-gpt OEIS checkpoints.

A PyTorch port of eval_m1_competition_mape_mlx.py from the NextTerm-440M repository (the protocol
behind the NextTerm model cards: Naive2 17.7987, NextTerm-440M 17.6239, NextTerm-47M 18.7621),
unchanged in substance:
  * each series' context is scaled to integers by the largest number of decimal places visible
    in the context, joined by commas, with a trailing comma
  * greedy decoding with EOS and PAD suppressed, stopping once `horizon` commas are generated
    (or after --max-new-tokens); series are decoded together in left-padded batches with
    per-row stopping (same greedy result as one at a time, up to batching numerics)
  * the first `horizon` generated integers, divided back by the scale, are the forecast;
    missing terms are scored as 0 (missing_policy "zero")
  * series MAPE = mean APE over the horizon (zero actuals skipped); macro MAPE = mean over series
    (the number on the model cards); median MAPE = median over series is also reported, since
    one runaway forecast (e.g. a parabola on a flat series, APE ~700) can swing the macro mean
    by several points
  * prompts of --max-context-tokens or more tokens (BOS included) are skipped

Usage: python eval_m1.py <hf_dir> <m1_competition_111.jsonl> <out_prefix>
Writes <out_prefix>.series.jsonl and <out_prefix>.summary.json.
"""
import argparse
import json
import math
from decimal import Decimal, InvalidOperation, localcontext
from statistics import mean, median

import torch
from transformers import AutoModelForCausalLM, AutoTokenizer


def parse_decimal(raw):
    try:
        value = Decimal(str(raw).strip())
    except InvalidOperation as exc:
        raise ValueError(f"Could not parse decimal value {raw!r}") from exc
    if not value.is_finite():
        raise ValueError(f"Non-finite decimal value {raw!r}")
    return value


def context_scale(raw_context):
    places = 0
    for raw in raw_context:
        exponent = parse_decimal(raw).as_tuple().exponent
        if exponent < 0:
            places = max(places, -exponent)
    return 10 ** places


def scale_decimal_to_int(value, scale):
    scaled = value * Decimal(scale)
    with localcontext() as ctx:
        ctx.prec = max(50, len(scaled.as_tuple().digits) + 10)
        rounded = scaled.to_integral_value()
    if rounded != scaled:
        raise ValueError(f"Context scale {scale} does not make {value} integral")
    return int(rounded)


def parse_generated_terms(text, limit):
    terms, current, malformed = [], [], False

    def flush():
        nonlocal malformed
        if not current:
            return
        token = "".join(current)
        current.clear()
        if token == "-":
            malformed = True
            return
        try:
            terms.append(int(token))
        except ValueError:
            malformed = True

    for ch in text:
        if len(terms) >= limit:
            break
        if ch.isdigit() or (ch == "-" and not current):
            current.append(ch)
        elif ch == ",":
            flush()
        elif ch.isspace():
            continue
        else:
            malformed = True
            flush()
            break
    if len(terms) < limit:
        flush()
    return terms[:limit], malformed


def ape(actual, prediction):
    if actual == 0:
        return None
    return float(abs(actual - prediction) / abs(actual) * Decimal(100))


def mape_for_predictions(actuals, predictions, missing_policy, fallback):
    apes, missing = [], 0
    for actual, pred in zip(actuals, predictions):
        if pred is None:
            missing += 1
            if missing_policy == "skip":
                apes.append(None)
                continue
            pred = Decimal(0) if missing_policy == "zero" else fallback
        apes.append(ape(actual, pred))
    valid = [x for x in apes if x is not None and math.isfinite(x)]
    return (mean(valid) if valid else None), apes, missing


def main():
    p = argparse.ArgumentParser()
    p.add_argument("hf_dir")
    p.add_argument("data")
    p.add_argument("out_prefix")
    p.add_argument("--max-new-tokens", type=int, default=1024)
    p.add_argument("--max-context-tokens", type=int, default=2048)
    p.add_argument("--missing-policy", choices=["zero", "last_context", "skip"], default="zero")
    p.add_argument("--batch-size", type=int, default=128,
                   help="series decoded together (greedy, left-padded, per-row stopping)")
    args = p.parse_args()

    tok = AutoTokenizer.from_pretrained(args.hf_dir)
    model = AutoModelForCausalLM.from_pretrained(args.hf_dir, dtype=torch.float32).cuda().eval()
    bos, eos, pad = tok.bos_token_id, tok.eos_token_id, tok.pad_token_id
    comma = ord(",")
    suppress = torch.tensor([t for t in (eos, pad) if t is not None], device="cuda")

    rows = [json.loads(line) for line in open(args.data) if line.strip()]
    series = []
    skipped_long = 0
    for row in rows:
        raw_context = [str(x) for x in row["context"]]
        context = [parse_decimal(x) for x in raw_context]
        scale = context_scale(raw_context)
        prompt_text = ",".join(str(scale_decimal_to_int(v, scale)) for v in context) + ","
        prompt = [bos] + list(prompt_text.encode("utf-8"))
        assert tok(prompt_text).input_ids == prompt, "tokenizer disagrees with BOS + bytes"
        if args.max_context_tokens > 0 and len(prompt) >= args.max_context_tokens:
            skipped_long += 1
            continue
        series.append((row, raw_context, context, scale, prompt, int(row["horizon"])))

    @torch.no_grad()
    def greedy_batch(batch):
        """Greedy decode a left-padded batch with a KV cache, EOS/PAD suppressed; each row
        stops once it has generated `horizon` commas (or after --max-new-tokens)."""
        width = max(len(s[4]) for s in batch)
        ids = torch.full((len(batch), width), pad, dtype=torch.long, device="cuda")
        mask = torch.zeros((len(batch), width), dtype=torch.long, device="cuda")
        for r, s in enumerate(batch):
            ids[r, width - len(s[4]):] = torch.tensor(s[4], device="cuda")
            mask[r, width - len(s[4]):] = 1
        pos = (mask.cumsum(-1) - 1).clamp(min=0)
        out = model(ids, attention_mask=mask, position_ids=pos, use_cache=True)
        past, logits = out.past_key_values, out.logits[:, -1]
        horizon = torch.tensor([s[5] for s in batch], device="cuda")
        commas = torch.zeros(len(batch), dtype=torch.long, device="cuda")
        done = torch.zeros(len(batch), dtype=torch.bool, device="cuda")
        last_pos = pos[:, -1]
        generated = []
        for _ in range(args.max_new_tokens):
            logits = logits.float()
            logits[:, suppress] = float("-inf")
            nxt = logits.argmax(-1)
            generated.append(torch.where(done, torch.full_like(nxt, pad), nxt))
            commas += (nxt == comma) & ~done
            done |= commas >= horizon
            if bool(done.all()):
                break
            mask = torch.cat([mask, torch.ones_like(mask[:, :1])], dim=1)
            last_pos = last_pos + 1
            out = model(nxt[:, None], attention_mask=mask, position_ids=last_pos[:, None],
                        past_key_values=past, use_cache=True)
            past, logits = out.past_key_values, out.logits[:, -1]
        gen = torch.stack(generated, dim=1).tolist()
        return [bytes(t for t in g if t < 256).decode("utf-8", "replace") for g in gen]

    texts = []
    for b in range(0, len(series), args.batch_size):
        texts.extend(greedy_batch(series[b:b + args.batch_size]))

    records = []
    for (row, raw_context, context, scale, prompt, horizon), text in zip(series, texts):
        target = [parse_decimal(str(x)) for x in row["target"]]
        naive2 = [parse_decimal(str(x)) for x in row["naive2_forecast"]]
        terms, malformed = parse_generated_terms(text, horizon)
        predictions = [Decimal(t) / Decimal(scale) for t in terms] + [None] * (horizon - len(terms))
        fallback = context[-1] if context else Decimal(0)
        mape, apes, missing = mape_for_predictions(target, predictions, args.missing_policy, fallback)
        naive2_mape, naive2_apes, _ = mape_for_predictions(target, naive2, "skip", fallback)
        records.append({
            "row_index": row["row_index"], "series_name": row["series_name"], "period": row["period"],
            "horizon": horizon, "scale": scale, "prompt_tokens": len(prompt),
            "target": [str(x) for x in target],
            "prediction": [str(x) if x is not None else None for x in predictions],
            "parsed_terms": len(terms), "missing_terms": missing, "malformed": malformed,
            "mape": mape, "apes": apes, "naive2_mape": naive2_mape, "naive2_apes": naive2_apes,
            "generated_text": text,
        })

    def finite(rs, key):
        return [float(r[key]) for r in rs if r[key] is not None and math.isfinite(float(r[key]))]

    def macro(rs, key):
        vals = finite(rs, key)
        return mean(vals) if vals else None

    def med(rs, key):
        vals = finite(rs, key)
        return median(vals) if vals else None

    by_period = {}
    for period in sorted({r["period"] for r in records}):
        rs = [r for r in records if r["period"] == period]
        by_period[period] = {"series": len(rs),
                             "model_median_series_mape": med(rs, "mape"),
                             "model_macro_mape": macro(rs, "mape"),
                             "naive2_median_series_mape": med(rs, "naive2_mape"),
                             "naive2_macro_mape": macro(rs, "naive2_mape")}
    summary = {
        "checkpoint": args.hf_dir,
        "series_evaluated": len(records),
        "skipped_long": skipped_long,
        "model_median_series_mape": med(records, "mape"),
        "model_macro_mape": macro(records, "mape"),
        "naive2_median_series_mape": med(records, "naive2_mape"),
        "naive2_macro_mape": macro(records, "naive2_mape"),
        "parsed_full_series": sum(1 for r in records if r["parsed_terms"] >= r["horizon"]),
        "total_missing_terms": sum(r["missing_terms"] for r in records),
        "malformed_series": sum(1 for r in records if r["malformed"]),
        "by_period": by_period,
    }
    with open(args.out_prefix + ".series.jsonl", "w") as f:
        for r in records:
            f.write(json.dumps(r) + "\n")
    with open(args.out_prefix + ".summary.json", "w") as f:
        json.dump(summary, f, indent=2)
    print(json.dumps({k: summary[k] for k in ["series_evaluated", "skipped_long",
                                              "model_median_series_mape", "naive2_median_series_mape",
                                              "model_macro_mape", "naive2_macro_mape", "parsed_full_series",
                                              "total_missing_terms", "malformed_series"]}))


if __name__ == "__main__":
    main()