ds.py

import json
from datasets import load_dataset
from tqdm import tqdm

JSONL_PATH = "/workspace/BiomedEnriched.jsonl"

commercial = load_dataset(
    "almanach/Biomed-Enriched",
    split="commercial",
    streaming=True
)

noncommercial = load_dataset(
    "almanach/Biomed-Enriched",
    split="noncommercial",
    streaming=True
)

with open(JSONL_PATH, "w", encoding="utf-8") as f:
    for i, row in enumerate(tqdm(commercial, desc="commercial")):
        rec = {
            "key": f"commercial_{i}",
            "split": "commercial",
            "text": row.get("text"),
            "path": row.get("path"),
            "license_url": row.get("license_url"),
            "authors": row.get("authors"),
            "document_type": row.get("document_type"),
            "domain": row.get("domain"),
            "educational_score": row.get("educational_score"),
        }
        f.write(json.dumps(rec, ensure_ascii=False) + "\n")

    for i, row in enumerate(tqdm(noncommercial, desc="noncommercial")):
        rec = {
            "key": f"noncommercial_{i}",
            "split": "noncommercial",
            "path": row.get("path"),
            "license_url": row.get("license_url"),
            "authors": row.get("authors"),
            "document_type": row.get("document_type"),
            "domain": row.get("domain"),
            "educational_score": row.get("educational_score"),
        }
        f.write(json.dumps(rec, ensure_ascii=False) + "\n")