harness/config.py

#!/usr/bin/env python3
"""Everything environment-specific, in one place.

The first version of this harness hard-coded absolute build-machine paths, a scratch directory
under /tmp, `http://localhost:1234/v1` and one model name, so nobody else could run it: measured
2026-08-04, 46 such lines across 22 scripts, plus 679 benchmark rows whose `pyc_path` pointed into
a scratch directory that no longer exists. Nothing here is hard-coded to a machine: paths are
resolved relative to this file, and the model endpoint comes from the environment.

    PYBYTECODE_ENDPOINT   OpenAI-compatible base URL   (default http://localhost:1234/v1)
    PYBYTECODE_MODEL      model name at that endpoint  (default pybytecode-v3-1.5b)
    PYBYTECODE_API_KEY    bearer token, if the server wants one (default "not-needed")

Only `generate.py` reads the endpoint. Every grading path is offline: given the cached
generations in this repository, all published numbers reproduce with no model and no GPU.
"""
from __future__ import annotations

import os
from pathlib import Path

HARNESS_DIR = Path(__file__).resolve().parent
RELEASE_DIR = HARNESS_DIR.parent
BENCHMARKS_DIR = RELEASE_DIR / "benchmarks"
GENERATIONS_DIR = RELEASE_DIR / "generations"
RESULTS_DIR = RELEASE_DIR / "results"

ENDPOINT = os.environ.get("PYBYTECODE_ENDPOINT", "http://localhost:1234/v1")
MODEL = os.environ.get("PYBYTECODE_MODEL", "pybytecode-v3-1.5b")
API_KEY = os.environ.get("PYBYTECODE_API_KEY", "not-needed")


def resolve_bench_asset(bench_file: Path, stored_path: str, kind: str, index: int) -> Path:
    """Resolve a row's .pyc / .py path RELATIVE TO ITS OWN bench.jsonl.

    Benchmarks built by the original harness stored absolute scratch-directory paths, which do
    not exist on any other machine (and no longer exist on the build machine either). The files
    themselves have always been committed next to bench.jsonl, so the stored string is ignored
    whenever the co-located file exists. A stored RELATIVE path is honoured as written.

    `kind` is "pyc" or "src"; `index` is the row's `i`.
    """
    bench_dir = bench_file.resolve().parent
    if stored_path and not stored_path.startswith("/"):
        p = bench_dir / stored_path
        if p.exists():
            return p
    ext = ".pyc" if kind == "pyc" else ".py"
    p = bench_dir / kind / f"{index:05d}{ext}"
    if p.exists():
        return p
    # last resort: trust the stored absolute path, so a failure names the real missing file
    return Path(stored_path) if stored_path else p


def have_pylingual() -> bool:
    """PyLingual is an OPTIONAL, user-installed extra. It is GPL-3.0 and is never vendored."""
    try:
        import pylingual.equivalence_check  # noqa: F401
        return True
    except Exception:  # noqa: BLE001
        return False