File size: 2,741 Bytes
7ed86c3 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 | """Load the Norma macronization/scansion benchmark.
Two sources:
hf (default) -- the canonical, actively-maintained release at
https://huggingface.co/datasets/anonymous-stoicheia/norma. This now supersedes the old
git-based copy: at time of writing it has 1378 macronize rows (614 test +
764 dev) vs the git copy's 932 (both dev and test folded into one file,
disambiguated by split_manifest.json), plus the same 614 syllabify test rows.
git -- a local checkout of an existing macronization project ($MACRONIZER_SRC),
kept for exact reproducibility of numbers reported before the HF migration.
Either way, load_norma() returns the same shape: {"test": [...], "dev": [...]},
each item a {"text": ..., "source": ..., "task": "macronize"|"syllabify"} dict,
so callers don't need to know which source produced them.
"""
from __future__ import annotations
import argparse
import json
import os
from pathlib import Path
HF_REPO = "anonymous-stoicheia/norma"
def _load_git(src: Path) -> dict:
lines = [json.loads(l) for l in open(src / "data/norma/test.jsonl", encoding="utf-8")]
manifest = json.loads((src / "data/norma/split_manifest.json").read_text())
dev_idx = set(manifest["dev_line_indices"])
test = [d for i, d in enumerate(lines) if i not in dev_idx]
dev = [d for i, d in enumerate(lines) if i in dev_idx]
return {"test": test, "dev": dev}
def _load_hf() -> dict:
from huggingface_hub import hf_hub_download
test_path = hf_hub_download(HF_REPO, "data/test.jsonl", repo_type="dataset")
dev_path = hf_hub_download(HF_REPO, "data/validation.jsonl", repo_type="dataset")
test = [json.loads(l) for l in open(test_path, encoding="utf-8")]
dev = [json.loads(l) for l in open(dev_path, encoding="utf-8")]
return {"test": test, "dev": dev}
def load_norma(source: str = "hf", src: Path | None = None) -> dict:
"""source="hf" (default): pull the canonical release from the Hub, no local
clone needed. source="git": read src/data/norma/{test.jsonl,split_manifest.json}
from a local macron-data checkout (src defaults to $MACRONIZER_SRC)."""
if source == "hf":
return _load_hf()
if source == "git":
src = src or Path(os.path.expandvars(os.environ.get(
"MACRONIZER_SRC", "$MACRONIZER_SRC")))
return _load_git(src)
raise ValueError(f"unknown Norma source: {source!r} (expected 'hf' or 'git')")
def add_norma_source_arg(ap: argparse.ArgumentParser) -> None:
"""Shared CLI flag so every Norma-consuming entry point takes the same option."""
ap.add_argument("--norma-source", choices=["hf", "git"], default="hf",
help="where to load the Norma benchmark from (default: hf)")
|