feat: add hugging face datasets setup tooling

.gitignore

@@ -0,0 +1,4 @@
+.venv/
+__pycache__/
+*.pyc
+data/

README.md

@@ -45,8 +45,44 @@ vector = embedder.encode("Book a flight from SFO to NYC")
 # Returns: 21D numpy array in Poincare Ball coordinates
 ```
 
+## Dataset Setup (PowerShell)
+
+Use this repo as a working directory for Hugging Face datasets:
+
+```powershell
+cd C:\Users\issda\hf-repos\phdm-21d-embedding
+python -m venv .venv
+.\.venv\Scripts\Activate.ps1
+python -m pip install --upgrade pip
+python -m pip install -r requirements-datasets.txt
+```
+
+Set your token in the current shell session:
+
+```powershell
+$env:HF_TOKEN="hf_your_token_here"
+```
+
+Load and preview a dataset split:
+
+```powershell
+python scripts/load_hf_dataset.py --dataset-id issdandavis/scbe-aethermoore-knowledge-base --split train --limit 3
+```
+
+Push local JSONL files to a dataset repo:
+
+```powershell
+python scripts/push_jsonl_dataset.py --dataset-id issdandavis/scbe-aethermoore-knowledge-base --train .\data\train.jsonl --validation .\data\validation.jsonl
+```
+
+Expected JSONL row format example:
+
+```json
+{"text":"Example source content","source":"notion","category":"policy"}
+```
+
 ## Related
 
 - [SCBE-AETHERMOORE GitHub](https://github.com/issdandavis/SCBE-AETHERMOORE)
 - [Knowledge Base Dataset](https://huggingface.co/datasets/issdandavis/scbe-aethermoore-knowledge-base)
-- [Interaction Logs Dataset](https://huggingface.co/datasets/issdandavis/scbe-interaction-logs)
+- [Interaction Logs Dataset](https://huggingface.co/datasets/issdandavis/scbe-interaction-logs)

requirements-datasets.txt

@@ -0,0 +1,4 @@
+datasets>=2.19.0
+huggingface_hub>=0.24.0
+pyarrow>=15.0.0
+pandas>=2.2.0

scripts/load_hf_dataset.py

@@ -0,0 +1,85 @@
+#!/usr/bin/env python
+"""Quick dataset loader for Hugging Face Hub datasets."""
+
+from __future__ import annotations
+
+import argparse
+import os
+
+from datasets import load_dataset
+from huggingface_hub import HfApi
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Load and preview a dataset split from Hugging Face Hub."
+    )
+    parser.add_argument(
+        "--dataset-id",
+        default="issdandavis/scbe-aethermoore-knowledge-base",
+        help="Dataset repo id on Hugging Face Hub (for example: username/dataset-name).",
+    )
+    parser.add_argument("--split", default="train", help="Split to load.")
+    parser.add_argument(
+        "--limit",
+        type=int,
+        default=3,
+        help="How many examples to print from the split.",
+    )
+    parser.add_argument(
+        "--streaming",
+        action="store_true",
+        help="Stream examples without downloading the full dataset.",
+    )
+    parser.add_argument(
+        "--token",
+        default=os.environ.get("HF_TOKEN"),
+        help="HF access token. Defaults to HF_TOKEN env var.",
+    )
+    return parser.parse_args()
+
+
+def main() -> None:
+    args = parse_args()
+    token = args.token
+
+    if token:
+        try:
+            user = HfApi(token=token).whoami()["name"]
+            print(f"Authenticated as: {user}")
+        except Exception as exc:  # pragma: no cover - network/auth failure path
+            print(f"Token check failed ({exc}). Retrying without token for public access.")
+            token = None
+    else:
+        print("No HF token provided. Public datasets only.")
+
+    try:
+        ds = load_dataset(
+            path=args.dataset_id,
+            split=args.split,
+            token=token,
+            streaming=args.streaming,
+        )
+    except Exception as exc:  # pragma: no cover - network/hub failure path
+        raise SystemExit(
+            f"Failed to load dataset '{args.dataset_id}' split '{args.split}': {exc}"
+        ) from exc
+
+    if args.streaming:
+        print(f"Loaded streaming split '{args.split}' from '{args.dataset_id}'.")
+        for idx, row in enumerate(ds):
+            print(f"[{idx}] {row}")
+            if idx + 1 >= args.limit:
+                break
+        return
+
+    print(f"Loaded split '{args.split}' from '{args.dataset_id}'.")
+    print(f"Rows: {len(ds)}")
+    print(f"Columns: {ds.column_names}")
+    print(f"Features: {ds.features}")
+    for idx in range(min(args.limit, len(ds))):
+        print(f"[{idx}] {ds[idx]}")
+
+
+if __name__ == "__main__":
+    main()

scripts/push_jsonl_dataset.py

@@ -0,0 +1,101 @@
+#!/usr/bin/env python
+"""Build and push a DatasetDict from local JSONL files."""
+
+from __future__ import annotations
+
+import argparse
+import os
+from pathlib import Path
+
+from datasets import DatasetDict, load_dataset
+from huggingface_hub import create_repo
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Push train/validation/test JSONL files to a Hugging Face dataset repo."
+    )
+    parser.add_argument(
+        "--dataset-id",
+        required=True,
+        help="Dataset repo id (for example: username/dataset-name).",
+    )
+    parser.add_argument("--train", help="Path to train JSONL file.")
+    parser.add_argument("--validation", help="Path to validation JSONL file.")
+    parser.add_argument("--test", help="Path to test JSONL file.")
+    parser.add_argument(
+        "--private",
+        action="store_true",
+        help="Create/update the dataset as private.",
+    )
+    parser.add_argument(
+        "--token",
+        default=os.environ.get("HF_TOKEN"),
+        help="HF access token. Defaults to HF_TOKEN env var.",
+    )
+    return parser.parse_args()
+
+
+def validate_split_path(name: str, path: str | None) -> Path | None:
+    if not path:
+        return None
+
+    file_path = Path(path).expanduser().resolve()
+    if not file_path.exists():
+        raise FileNotFoundError(f"{name} file not found: {file_path}")
+    if file_path.suffix.lower() != ".jsonl":
+        raise ValueError(f"{name} file must be a .jsonl file: {file_path}")
+    return file_path
+
+
+def main() -> None:
+    args = parse_args()
+
+    split_paths = {
+        "train": validate_split_path("train", args.train),
+        "validation": validate_split_path("validation", args.validation),
+        "test": validate_split_path("test", args.test),
+    }
+    split_paths = {k: str(v) for k, v in split_paths.items() if v is not None}
+
+    if not split_paths:
+        raise ValueError("Provide at least one split: --train, --validation, or --test.")
+
+    if not args.token:
+        raise ValueError("Set HF_TOKEN or pass --token to push a dataset.")
+
+    try:
+        create_repo(
+            repo_id=args.dataset_id,
+            repo_type="dataset",
+            private=args.private,
+            exist_ok=True,
+            token=args.token,
+        )
+    except Exception as exc:  # pragma: no cover - network/hub failure path
+        raise SystemExit(f"Failed to create/access dataset repo '{args.dataset_id}': {exc}") from exc
+
+    split_datasets = {}
+    for split_name, path in split_paths.items():
+        split_datasets[split_name] = load_dataset(
+            "json",
+            data_files={split_name: path},
+            split=split_name,
+        )
+        print(f"Loaded {split_name}: {len(split_datasets[split_name])} rows from {path}")
+
+    dataset_dict = DatasetDict(split_datasets)
+    try:
+        dataset_dict.push_to_hub(
+            repo_id=args.dataset_id,
+            private=args.private,
+            token=args.token,
+        )
+    except Exception as exc:  # pragma: no cover - network/hub failure path
+        raise SystemExit(f"Failed to push dataset '{args.dataset_id}': {exc}") from exc
+
+    print(f"Pushed dataset to: https://huggingface.co/datasets/{args.dataset_id}")
+
+
+if __name__ == "__main__":
+    main()