laya-ledgar / code /layatasks /common.py
VibeHash's picture
Release laya-ledgar, run ce-seed0
779ff0b verified
Raw History Blame Contribute Delete
2.08 kB
"""Paths, hashing and file helpers every step shares.
Data, base weights and runs live outside the repository, under `$LAYA_TASKS_HOME` (default `~/.laya-tasks`), one
folder a task. None of them belong in a model repository's code folder, and keeping them apart makes that easy to
check.
"""
from __future__ import annotations
import hashlib
import json
import os
from pathlib import Path
from typing import Any, Dict, Iterable, List
def home() -> Path:
"""The root for every task's data and runs."""
return Path(os.environ.get("LAYA_TASKS_HOME", "~/.laya-tasks")).expanduser()
def task_home(task: str) -> Path:
return home() / task
def data_dir(task: str) -> Path:
return task_home(task) / "data" / "v1"
def run_dir(task: str, run: str) -> Path:
return task_home(task) / "runs" / run
def sha256_file(path: Path) -> str:
h = hashlib.sha256()
with open(path, "rb") as f:
for block in iter(lambda: f.read(1 << 20), b""):
h.update(block)
return h.hexdigest()
def write_jsonl(path: Path, items: Iterable[Dict[str, Any]]) -> None:
"""Write one JSON object a line, through a temporary file, so a crash never leaves half a file in place."""
tmp = path.with_suffix(path.suffix + ".tmp")
with open(tmp, "w", encoding="utf-8", newline="\n") as f:
for item in items:
f.write(json.dumps(item, ensure_ascii=False, sort_keys=True) + "\n")
os.replace(tmp, path)
def read_jsonl(path: Path) -> List[Dict[str, Any]]:
with open(path, encoding="utf-8") as f:
return [json.loads(line) for line in f if line.strip()]
def write_json(path: Path, value: Any) -> None:
with open(path, "w", encoding="utf-8", newline="\n") as f:
json.dump(value, f, indent=2, ensure_ascii=False)
f.write("\n")
def read_json(path: Path) -> Any:
with open(path, encoding="utf-8") as f:
return json.load(f)
def normalise(text: str) -> str:
"""Text compared for copies between train and test: case and spacing ignored."""
return " ".join(text.lower().split())