Spaces:
Running
Running
File size: 3,711 Bytes
67acd34 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 | from __future__ import annotations
from dataclasses import dataclass
from pathlib import Path
from typing import Iterable
import pandas as pd
from datasets import load_dataset
ROOT_DIR = Path(__file__).resolve().parents[2]
EVALS_DIR = ROOT_DIR / "evals"
PROMPTS_DIR = EVALS_DIR / "prompts"
RESULTS_DIR = ROOT_DIR / "results"
RUNS_DIR = RESULTS_DIR / "runs"
REPORTS_DIR = RESULTS_DIR / "reports"
MAPPINGS_DIR = ROOT_DIR / "mappings" / "topobench_mappings"
SUBMODULES_DIR = ROOT_DIR / "submodules"
DATASET_REPOS = {
"plain": "topobench/topobench",
"intformat": "topobench/topobench_intformat",
"intformat_json": "topobench/topobench_intformat_json",
}
@dataclass(frozen=True)
class PuzzleSpec:
key: str
dataset_name: str
prompt_stem: str
verifier_type: str
default_args: str
def prompt_path(self, variant: str) -> Path:
suffix = {
"plain": ".txt",
"intformat": "_intformat.txt",
"intformat_json": "_intformat_json.txt",
}[variant]
return PROMPTS_DIR / f"{self.prompt_stem}{suffix}"
PUZZLES: dict[str, PuzzleSpec] = {
"bridges": PuzzleSpec(
key="bridges",
dataset_name="bridges",
prompt_stem="bridges",
verifier_type="bridges",
default_args="5x5deL",
),
"flow_free": PuzzleSpec(
key="flow_free",
dataset_name="flow_free",
prompt_stem="flow_free",
verifier_type="flow_free",
default_args="5x5",
),
"galaxies": PuzzleSpec(
key="galaxies",
dataset_name="galaxies",
prompt_stem="galaxies",
verifier_type="galaxies",
default_args="4x4",
),
"loopy": PuzzleSpec(
key="loopy",
dataset_name="loopy",
prompt_stem="loopy",
verifier_type="loopy",
default_args="5x5t0",
),
"pattern": PuzzleSpec(
key="pattern",
dataset_name="pattern",
prompt_stem="pattern",
verifier_type="pattern",
default_args="5x5",
),
"undead": PuzzleSpec(
key="undead",
dataset_name="undead",
prompt_stem="undead",
verifier_type="undead",
default_args="4x4",
),
}
def get_puzzle(name: str) -> PuzzleSpec:
try:
return PUZZLES[name]
except KeyError as exc:
valid = ", ".join(sorted(PUZZLES))
raise ValueError(f"Unknown puzzle '{name}'. Valid values: {valid}") from exc
def list_puzzle_names() -> list[str]:
return sorted(PUZZLES)
def expand_puzzles(names: Iterable[str]) -> list[PuzzleSpec]:
items = list(names)
if not items or items == ["all"]:
return [PUZZLES[name] for name in list_puzzle_names()]
return [get_puzzle(name) for name in items]
def load_prompt(puzzle: PuzzleSpec, variant: str) -> str:
return puzzle.prompt_path(variant).read_text(encoding="utf-8")
def load_dataset_frame(
puzzle: PuzzleSpec,
*,
variant: str,
difficulty: str,
limit: int | None,
split: str = "test",
) -> pd.DataFrame:
repo_id = DATASET_REPOS[variant]
frame = load_dataset(repo_id, split=split).to_pandas()
frame = frame[frame["puzzlename"] == puzzle.dataset_name].copy()
if "include" in frame.columns:
frame = frame[frame["include"].fillna(False)]
if difficulty != "all":
frame = frame[frame["difficulty"] == difficulty]
frame = frame.sort_values(["difficulty", "filename"]).reset_index(drop=True)
if limit is not None:
frame = frame.head(limit).copy()
return frame.reset_index(drop=True)
def build_row_lookup(frame: pd.DataFrame) -> dict[str, pd.Series]:
return {row["filename"]: row for _, row in frame.iterrows()}
|