hf_publish.py

"""Service layer for getting the combined dataset out to the world -
either pushed to a HF Hub repo, or written out as a local JSONL file.
"""
from __future__ import annotations

import json
import os

from datasets import Dataset


def push_dataset(records: list, repo_id: str, private: bool, token: str) -> str:
    if not token:
        raise ValueError("No HF token available - sign in first.")
    if not repo_id or "/" not in repo_id:
        raise ValueError("repo_id must look like 'username/dataset-name'.")

    ds = Dataset.from_list(records)
    ds.push_to_hub(repo_id, private=private, token=token)
    return f"https://huggingface.co/datasets/{repo_id}"


def write_jsonl(records: list, output_path: str) -> str:
    os.makedirs(os.path.dirname(output_path) or ".", exist_ok=True)
    with open(output_path, "w", encoding="utf-8") as f:
        for record in records:
            f.write(json.dumps(record, ensure_ascii=False) + "\n")
    return output_path