utils.py

import os
import json
import uuid
from pathlib import Path

from huggingface_hub import CommitScheduler


def create_scheduler():
    """
    Create a scheduler to commit feedback to the dataset repository."""
    feedback_file = Path("question_answers/") / f"data_{uuid.uuid4()}.json"
    feedback_folder = feedback_file.parent
    token = os.getenv("HF_TOKEN")

    scheduler = CommitScheduler(
        repo_id="paper-extractor-bot-history",
        repo_type="dataset",
        folder_path=feedback_folder,
        path_in_repo="data",
        every=5,
        token=token,
    )
    return scheduler, feedback_file


def save_feedback(
    scheduler: CommitScheduler, feedback_file: Path, question: str, answer: str
) -> None:
    """
    Add the question and answer to a JSON Lines file.
    """
    with scheduler.lock:
        with feedback_file.open("a") as f:
            f.write(json.dumps({"input": question, "answer": answer}))
            f.write("\n")