"""CLI: download FiNER-139, align labels, split into train/val/test. Usage: python scripts/prepare_data.py python scripts/prepare_data.py --seed 123 """ from __future__ import annotations import argparse import logging import sys from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) from rich.console import Console from rich.table import Table from collections import Counter from finner.data.load import load_all from finner.data.split import split_and_save from finner.labels import ENTITY_TYPES logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s") console = Console() def label_distribution(examples: list[dict]) -> dict: counts = Counter() for ex in examples: for tag in ex["ner_tags"]: if tag != "O": counts[tag.split("-", 1)[1]] += 1 return dict(counts) def main(): parser = argparse.ArgumentParser(description="Prepare FinNER training data") parser.add_argument("--seed", type=int, default=42) args = parser.parse_args() console.print("[bold cyan]Step 1: Loading FiNER-139...[/bold cyan]") examples = load_all() console.print(f"Loaded [bold]{len(examples)}[/bold] examples total.") console.print("\n[bold cyan]Step 2: Splitting (stratified, 70/15/15)...[/bold cyan]") train, val, test = split_and_save(examples, seed=args.seed) # Print distribution stats table = Table(title="Label distribution by split", show_header=True) table.add_column("Entity Type", style="cyan") table.add_column("Train", justify="right") table.add_column("Val", justify="right") table.add_column("Test", justify="right") train_dist = label_distribution(train) val_dist = label_distribution(val) test_dist = label_distribution(test) for etype in ENTITY_TYPES: table.add_row( etype, str(train_dist.get(etype, 0)), str(val_dist.get(etype, 0)), str(test_dist.get(etype, 0)), ) console.print(table) console.print(f"\n[bold green]✓ Done.[/bold green] Splits saved to data/splits/") console.print(f" train: {len(train):,} val: {len(val):,} test: {len(test):,}") console.print("\n[yellow]Note: O-class dominance is expected and will be addressed in Phase 3.[/yellow]") if __name__ == "__main__": main()