"""Paths, hashing and file helpers every step shares. Data, base weights and runs live outside the repository, under `$LAYA_TASKS_HOME` (default `~/.laya-tasks`), one folder a task. None of them belong in a model repository's code folder, and keeping them apart makes that easy to check. """ from __future__ import annotations import hashlib import json import os from pathlib import Path from typing import Any, Dict, Iterable, List def home() -> Path: """The root for every task's data and runs.""" return Path(os.environ.get("LAYA_TASKS_HOME", "~/.laya-tasks")).expanduser() def task_home(task: str) -> Path: return home() / task def data_dir(task: str) -> Path: return task_home(task) / "data" / "v1" def run_dir(task: str, run: str) -> Path: return task_home(task) / "runs" / run def sha256_file(path: Path) -> str: h = hashlib.sha256() with open(path, "rb") as f: for block in iter(lambda: f.read(1 << 20), b""): h.update(block) return h.hexdigest() def write_jsonl(path: Path, items: Iterable[Dict[str, Any]]) -> None: """Write one JSON object a line, through a temporary file, so a crash never leaves half a file in place.""" tmp = path.with_suffix(path.suffix + ".tmp") with open(tmp, "w", encoding="utf-8", newline="\n") as f: for item in items: f.write(json.dumps(item, ensure_ascii=False, sort_keys=True) + "\n") os.replace(tmp, path) def read_jsonl(path: Path) -> List[Dict[str, Any]]: with open(path, encoding="utf-8") as f: return [json.loads(line) for line in f if line.strip()] def write_json(path: Path, value: Any) -> None: with open(path, "w", encoding="utf-8", newline="\n") as f: json.dump(value, f, indent=2, ensure_ascii=False) f.write("\n") def read_json(path: Path) -> Any: with open(path, encoding="utf-8") as f: return json.load(f) def normalise(text: str) -> str: """Text compared for copies between train and test: case and spacing ignored.""" return " ".join(text.lower().split())