"""Task-type generators for synthetic filesystem benchmarks. Each generator produces a self-consistent task: * ``build(env_dir, llm, rng)`` writes the initial files into the test environment and returns a ``spec`` dict describing what it created. * ``description(spec)`` renders the natural-language ``description.md`` the model will see. * ``verify_src(spec)`` renders a self-contained ``verify.py`` whose checks *recompute* the correct answer from the resulting files (no external answer key), using only the standard library. * ``solve(work_dir, spec)`` is the oracle: it performs the correct solution in-place so the pipeline can prove ``verify.py`` accepts the intended answer. The description and the verifier are written together so they always agree. """ import json import random import shutil from collections import defaultdict from pathlib import Path from typing import Dict, List # --------------------------------------------------------------------------- # # Shared helpers # --------------------------------------------------------------------------- # _FILLER = " lorem ipsum dolor sit amet consectetur adipiscing elit" def _ascii_filler(n: int) -> str: if n <= 0: return "" return (_FILLER * (n // len(_FILLER) + 1))[:n] def pad_to(body: str, target_bytes: int) -> str: """Return text whose UTF-8 size is exactly ``target_bytes``.""" b = body.encode("utf-8") if len(b) > target_bytes: return b[:target_bytes].decode("utf-8", errors="ignore") return body + _ascii_filler(target_bytes - len(b.decode("utf-8").encode("utf-8"))) def _write(path: Path, content: str) -> None: path.parent.mkdir(parents=True, exist_ok=True) path.write_text(content, encoding="utf-8") # Header shared by every generated verify.py. _VERIFY_HEADER = '''#!/usr/bin/env python3 """Auto-generated verifier (synthetic task). Recomputes the answer; do not edit.""" import json import os import sys from pathlib import Path SYSTEM_FILES = {".DS_Store", "Thumbs.db", ".DS_Store?", "._.DS_Store"} def get_test_dir() -> Path: d = os.environ.get("FILESYSTEM_TEST_DIR") if not d: raise ValueError("FILESYSTEM_TEST_DIR environment variable is required") return Path(d) def fail(msg): print("\\u274c " + msg) sys.exit(1) def ok(msg): print("\\u2705 " + msg) ''' class Generator: KEY = "base" CATEGORY_NAME = "Base" DIFFICULTY = "L2" TAGS: List[str] = [] def __init__(self, difficulty: str = "medium"): # Generators that support difficulty tiers read self.difficulty; the rest # simply ignore it. self.difficulty = difficulty def build(self, env_dir: Path, llm, rng: random.Random) -> Dict: raise NotImplementedError def description(self, spec: Dict) -> str: raise NotImplementedError def verify_src(self, spec: Dict) -> str: raise NotImplementedError def solve(self, work_dir: Path, spec: Dict) -> None: raise NotImplementedError def _render_verify(body: str, consts: dict) -> str: return _VERIFY_HEADER + body.replace("__CONSTS__", json.dumps(json.dumps(consts))) _AUTHORS = ["Ada Lovelace", "Alan Turing", "Grace Hopper", "Donald Knuth", "Barbara Liskov"] _SONG_TITLES = [ "Blue Horizon", "Midnight Drive", "Paper Moon", "Echoes", "Golden Hour", "Silent Tide", "Neon Rain", "Wandering", "Afterglow", "Velvet Sky", "Lighthouse", "Crossroads", ] _STUDENTS = [ "Liam Carter", "Olivia Reed", "Noah Patel", "Emma Davies", "Mason Cole", "Ava Brooks", "Lucas Gray", "Mia Foster", "Ethan Ward", "Sofia Bennett", ] _WORDS = ( "system module config network buffer kernel thread cache socket render " "matrix tensor sample dataset gradient logging parser schema invoice client" ).split() def _para(rng: random.Random, n_lines: int, inject=None) -> list: lines = [] for _ in range(n_lines): line = " ".join(rng.choice(_WORDS) for _ in range(rng.randint(4, 9))).capitalize() + "." lines.append(line) if inject: # inject is a list of (index, text); replace those lines for idx, text in inject: lines[idx % len(lines)] = text return lines