Upload 9 files

README.md

@@ -40,14 +40,20 @@ server-side between sessions.
 
 ## Project layout
 
+Everything lives flat in the repo root (no subfolders, so it uploads fine
+through the Spaces web UI one file at a time):
+
 - `app.py` - Gradio UI and event wiring.
-- `core/` - pure logic: schema detection (`schema_detect.py`) and row mapping
-  (`field_mapper.py`). No network calls in here, which makes it the easy part
-  to unit test on its own - see the inline examples in each module's tests
-  if you add a `tests/` folder later.
-- `services/` - everything that talks to Hugging Face: peeking at a dataset's
-  shape (`hf_inspect.py`), pulling the real rows (`hf_dataset_loader.py`), and
-  pushing/exporting the result (`hf_publish.py`).
+- `models.py` - the `DatasetEntry` / `FieldMapping` dataclasses shared by
+  everything else.
+- `schema_detect.py` - pure schema-detection logic. No network calls in
+  here, which makes it the easy part to unit test on its own.
+- `field_mapper.py` - pure row-to-triplet extraction logic, also network-free.
+- `hf_inspect.py` - peeks at a dataset's shape via the lightweight
+  `datasets-server` API (with a streaming fallback) without downloading it.
+- `hf_dataset_loader.py` - pulls the real rows, up to the per-dataset limit.
+- `hf_publish.py` - pushes the combined dataset to the Hub, or writes it
+  out as JSONL.
 
 ## Known limitations
 
@@ -57,13 +63,13 @@ server-side between sessions.
   Anything else routes to manual mapping on purpose - a near-miss column name
   (e.g. `question`/`chosen` instead of `prompt`/`chosen`) won't get silently
   mis-mapped. The known-pairs list is a plain Python list at the top of
-  `core/schema_detect.py` if you want to extend it for patterns you hit a lot.
+  `schema_detect.py` if you want to extend it for patterns you hit a lot.
 - Datasets that bundle extra per-row enrichment beyond a simple triplet (for
   example, appending a vulnerability note to an answer, or synthesizing a
   prompt from a code snippet plus a language tag) won't be replicated by the
   generic mapper - it pulls the two fields you point it at, verbatim. If you
   need that kind of enrichment again, it's a small addition to
-  `core/field_mapper.py`.
+  `field_mapper.py`.
 - Gated datasets need the signed-in user's token to actually have read
   access; there's no separate "request access" flow built in here.
 - The `gr.OAuthToken` / `gr.OAuthProfile` injection is wired to the documented

app.py

@@ -24,9 +24,12 @@ import tempfile
 
 import gradio as gr
 
-from core import field_mapper, schema_detect
-from core.models import DatasetEntry, FieldMapping
-from services import hf_dataset_loader, hf_inspect, hf_publish
+import field_mapper
+import schema_detect
+import hf_dataset_loader
+import hf_inspect
+import hf_publish
+from models import DatasetEntry, FieldMapping
 
 _STATUS_LABELS = {
     "empty": "Not detected yet.",

field_mapper.py

@@ -0,0 +1,55 @@
+"""Turn one raw dataset row into a normalized chat-format record, given a
+FieldMapping. Pure function - no network, no HF imports, easy to test.
+"""
+from __future__ import annotations
+
+from typing import Optional
+
+from models import FieldMapping
+
+
+def _stringify(value) -> Optional[str]:
+    if value is None:
+        return None
+    if isinstance(value, str):
+        return value if value.strip() else None
+    return str(value)
+
+
+def extract_triplet(row: dict, mapping: FieldMapping, system_prompt: str) -> Optional[dict]:
+    """Returns {"messages": [...]} in OpenAI/ShareGPT chat format, or None
+    if this row didn't have usable user/assistant text.
+    """
+    user_text: Optional[str] = None
+    asst_text: Optional[str] = None
+
+    if mapping.kind == "conversation_list":
+        items = row.get(mapping.config["list_field"]) or []
+        role_key = mapping.config["role_key"]
+        content_key = mapping.config["content_key"]
+        human_tag = mapping.config["human_tag"]
+        gpt_tag = mapping.config["gpt_tag"]
+        user_text = _stringify(
+            next((item.get(content_key) for item in items if item.get(role_key) == human_tag), None)
+        )
+        asst_text = _stringify(
+            next((item.get(content_key) for item in items if item.get(role_key) == gpt_tag), None)
+        )
+
+    elif mapping.kind == "flat_pair":
+        user_text = _stringify(row.get(mapping.config["user_field"]))
+        asst_text = _stringify(row.get(mapping.config["assistant_field"]))
+
+    else:
+        return None
+
+    if not user_text or not asst_text:
+        return None
+
+    return {
+        "messages": [
+            {"role": "system", "content": system_prompt or ""},
+            {"role": "user", "content": user_text},
+            {"role": "assistant", "content": asst_text},
+        ]
+    }

hf_dataset_loader.py

@@ -0,0 +1,33 @@
+"""Service layer for pulling up to N rows out of a HF dataset, for real
+(not just a peek). Streams where the dataset supports it so we never
+download more than was asked for.
+"""
+from __future__ import annotations
+
+from typing import Optional
+
+from datasets import load_dataset
+
+
+def load_limited(
+    repo_id: str,
+    subset: str,
+    split: str,
+    limit: int,
+    token: Optional[str] = None,
+) -> list:
+    try:
+        ds = load_dataset(repo_id, subset or None, split=split, streaming=True, token=token)
+        rows = []
+        for i, row in enumerate(ds):
+            if i >= limit:
+                break
+            rows.append(dict(row))
+        return rows
+    except Exception:
+        # Some datasets (mostly older script-based ones) don't support
+        # streaming. Fall back to a full load and slice - slower, but
+        # correct, and rare enough not to special-case earlier.
+        ds = load_dataset(repo_id, subset or None, split=split, token=token)
+        n = min(limit, len(ds))
+        return [dict(ds[i]) for i in range(n)]

hf_inspect.py

@@ -0,0 +1,89 @@
+"""Service layer for peeking at a HF dataset's shape without pulling the
+whole thing down. Tries the lightweight datasets-server API first; falls
+back to a short streaming pull for datasets that API doesn't cover
+(gated, script-based, or just not indexed yet).
+"""
+from __future__ import annotations
+
+from typing import Optional
+
+import requests
+from datasets import load_dataset
+
+_ROWS_URL = "https://datasets-server.huggingface.co/rows"
+_SPLITS_URL = "https://datasets-server.huggingface.co/splits"
+_REQUEST_TIMEOUT_SECONDS = 15
+
+
+class DatasetInspectError(Exception):
+    """Raised when we genuinely can't get a peek at a dataset, after
+    trying both the fast path and the streaming fallback."""
+
+
+def list_splits(repo_id: str, token: Optional[str] = None) -> list:
+    """Returns [{"config": ..., "split": ...}, ...] for the dataset."""
+    headers = {"Authorization": f"Bearer {token}"} if token else {}
+    resp = requests.get(
+        _SPLITS_URL, params={"dataset": repo_id}, headers=headers, timeout=_REQUEST_TIMEOUT_SECONDS
+    )
+    if resp.status_code != 200:
+        raise DatasetInspectError(
+            f"Couldn't list splits for '{repo_id}' (HTTP {resp.status_code}): {resp.text[:300]}"
+        )
+    data = resp.json()
+    return [{"config": s["config"], "split": s["split"]} for s in data.get("splits", [])]
+
+
+def peek_rows(
+    repo_id: str,
+    subset: str,
+    split: str,
+    sample_size: int = 8,
+    token: Optional[str] = None,
+) -> list:
+    """Returns up to `sample_size` raw rows as plain dicts."""
+    if not repo_id.strip():
+        raise DatasetInspectError("No dataset repo id given.")
+    try:
+        return _peek_via_datasets_server(repo_id, subset, split, sample_size, token)
+    except DatasetInspectError:
+        return _peek_via_streaming(repo_id, subset, split, sample_size, token)
+
+
+def _peek_via_datasets_server(
+    repo_id: str, subset: str, split: str, sample_size: int, token: Optional[str]
+) -> list:
+    headers = {"Authorization": f"Bearer {token}"} if token else {}
+    params = {
+        "dataset": repo_id,
+        "config": subset or "default",
+        "split": split,
+        "offset": 0,
+        "length": sample_size,
+    }
+    resp = requests.get(_ROWS_URL, params=params, headers=headers, timeout=_REQUEST_TIMEOUT_SECONDS)
+    if resp.status_code != 200:
+        raise DatasetInspectError(f"datasets-server returned HTTP {resp.status_code} for '{repo_id}'")
+    data = resp.json()
+    rows = data.get("rows", [])
+    if not rows:
+        raise DatasetInspectError(f"datasets-server returned no rows for '{repo_id}'")
+    return [r["row"] for r in rows]
+
+
+def _peek_via_streaming(
+    repo_id: str, subset: str, split: str, sample_size: int, token: Optional[str]
+) -> list:
+    try:
+        ds = load_dataset(repo_id, subset or None, split=split, streaming=True, token=token)
+    except Exception as exc:
+        raise DatasetInspectError(f"Couldn't load '{repo_id}': {exc}") from exc
+
+    rows = []
+    for i, row in enumerate(ds):
+        if i >= sample_size:
+            break
+        rows.append(dict(row))
+    if not rows:
+        raise DatasetInspectError(f"'{repo_id}' (config={subset or 'default'}, split={split}) has no rows")
+    return rows

hf_publish.py

@@ -0,0 +1,28 @@
+"""Service layer for getting the combined dataset out to the world -
+either pushed to a HF Hub repo, or written out as a local JSONL file.
+"""
+from __future__ import annotations
+
+import json
+import os
+
+from datasets import Dataset
+
+
+def push_dataset(records: list, repo_id: str, private: bool, token: str) -> str:
+    if not token:
+        raise ValueError("No HF token available - sign in first.")
+    if not repo_id or "/" not in repo_id:
+        raise ValueError("repo_id must look like 'username/dataset-name'.")
+
+    ds = Dataset.from_list(records)
+    ds.push_to_hub(repo_id, private=private, token=token)
+    return f"https://huggingface.co/datasets/{repo_id}"
+
+
+def write_jsonl(records: list, output_path: str) -> str:
+    os.makedirs(os.path.dirname(output_path) or ".", exist_ok=True)
+    with open(output_path, "w", encoding="utf-8") as f:
+        for record in records:
+            f.write(json.dumps(record, ensure_ascii=False) + "\n")
+    return output_path

models.py

@@ -0,0 +1,47 @@
+"""Data models for dataset entries and field mappings.
+
+These are plain dataclasses so they can live inside a gr.State without any
+serialization step - Gradio keeps server-side state as real Python objects
+per session, so we just mutate them in place.
+"""
+from __future__ import annotations
+
+import uuid
+from dataclasses import dataclass, field
+from typing import Literal, Optional
+
+MappingKind = Literal["conversation_list", "flat_pair", "unmapped"]
+EntryStatus = Literal["empty", "detecting", "needs_mapping", "ready", "error"]
+
+
+@dataclass
+class FieldMapping:
+    """How to pull a (system, user, assistant) triplet out of one raw row.
+
+    `config` holds whatever the given `kind` needs:
+      - conversation_list: list_field, role_key, content_key, human_tag, gpt_tag
+      - flat_pair: user_field, assistant_field
+    """
+
+    kind: MappingKind
+    config: dict = field(default_factory=dict)
+
+
+@dataclass
+class DatasetEntry:
+    """One row in the dataset-builder list."""
+
+    uid: str = field(default_factory=lambda: uuid.uuid4().hex[:8])
+    repo_id: str = ""
+    subset: str = ""
+    split: str = "train"
+    limit: int = 1000
+    system_prompt: str = ""
+
+    mapping: Optional[FieldMapping] = None
+    detected_columns: list = field(default_factory=list)
+    detected_list_info: Optional[dict] = None
+    sample_rows: list = field(default_factory=list)
+
+    status: EntryStatus = "empty"
+    error_message: str = ""

schema_detect.py

@@ -0,0 +1,174 @@
+"""Pure functions for guessing how a raw HF dataset row maps onto a
+(system, user, assistant) triplet, based on a handful of sample rows.
+
+Nothing here touches the network - callers fetch sample rows separately
+(see hf_inspect.py) and hand them in. That split is deliberate:
+this logic is the part worth unit-testing on its own, against fixed
+sample data, without spinning up HTTP calls every time.
+"""
+from __future__ import annotations
+
+from typing import Optional
+
+from models import FieldMapping
+
+# Known flat-column name pairs, checked case-insensitively against the
+# dataset's actual column names. Ordered roughly by how common they are
+# across instruction-tuning datasets on the Hub.
+_KNOWN_FLAT_PAIRS = [
+    ("instruction", "output"),
+    ("instruction", "response"),
+    ("prompt", "chosen"),
+    ("prompt", "completion"),
+    ("query", "answer"),
+    ("query", "answers"),
+    ("question", "solution"),
+    ("question", "answer"),
+    ("input", "output"),
+]
+
+# Known (human_tag, assistant_tag) pairs for the role key inside a
+# conversation-list column (e.g. ShareGPT's "from", OpenAI's "role").
+_KNOWN_TAG_PAIRS = [
+    ("human", "gpt"),
+    ("user", "assistant"),
+    ("user", "model"),
+]
+
+
+def _is_list_of_dicts_column(values: list) -> bool:
+    for v in values:
+        if not isinstance(v, list) or not v:
+            return False
+        if not all(isinstance(item, dict) for item in v):
+            return False
+    return True
+
+
+def detect_flat_pair(columns: list) -> Optional[FieldMapping]:
+    """Match the dataset's column names against known flat pairs."""
+    lower_to_actual = {c.lower(): c for c in columns}
+    for user_name, asst_name in _KNOWN_FLAT_PAIRS:
+        if user_name in lower_to_actual and asst_name in lower_to_actual:
+            return FieldMapping(
+                kind="flat_pair",
+                config={
+                    "user_field": lower_to_actual[user_name],
+                    "assistant_field": lower_to_actual[asst_name],
+                },
+            )
+    return None
+
+
+def _guess_role_and_content_key(values: list, keys: set):
+    """Pick which key behaves like a role tag (short, low-cardinality
+    strings) and which behaves like free-text content (longer, varied).
+    Returns (role_key, content_key), or (None, None) if it can't tell.
+    """
+    candidates = list(keys)
+    if len(candidates) < 2:
+        return None, None
+
+    avg_lengths = {}
+    distinct_counts = {}
+    for key in candidates:
+        all_values = [item.get(key) for value_list in values for item in value_list if key in item]
+        string_values = [v for v in all_values if isinstance(v, str)]
+        if not string_values:
+            avg_lengths[key] = float("inf")
+            distinct_counts[key] = len(set(map(str, all_values)))
+            continue
+        avg_lengths[key] = sum(len(v) for v in string_values) / len(string_values)
+        distinct_counts[key] = len(set(string_values))
+
+    role_key = min(candidates, key=lambda k: (distinct_counts[k], avg_lengths[k]))
+    remaining = [k for k in candidates if k != role_key]
+    content_key = max(remaining, key=lambda k: avg_lengths[k])
+    return role_key, content_key
+
+
+def detect_list_column(sample_rows: list) -> Optional[dict]:
+    """Find a column whose values look like a conversation list (ShareGPT's
+    `conversations`, OpenAI's `messages`, or anything shaped like them) and
+    figure out which sub-key is the role and which is the content.
+
+    Returns a dict describing what was found - used both to try full
+    auto-detection and to pre-fill the manual-mapping UI when auto-detect
+    isn't confident. Returns None if nothing list-shaped turned up.
+    """
+    if not sample_rows:
+        return None
+
+    columns = list(sample_rows[0].keys())
+    for col in columns:
+        values = [row.get(col) for row in sample_rows]
+        if not _is_list_of_dicts_column(values):
+            continue
+
+        common_keys = None
+        for value_list in values:
+            for item in value_list:
+                keys = set(item.keys())
+                common_keys = keys if common_keys is None else (common_keys & keys)
+        if not common_keys or len(common_keys) < 2:
+            continue
+
+        role_key, content_key = _guess_role_and_content_key(values, common_keys)
+        if role_key is None:
+            continue
+
+        tag_values = sorted(
+            {
+                item.get(role_key)
+                for value_list in values
+                for item in value_list
+                if role_key in item and item.get(role_key) is not None
+            }
+        )
+
+        return {
+            "list_field": col,
+            "role_key": role_key,
+            "content_key": content_key,
+            "tag_values": tag_values,
+        }
+
+    return None
+
+
+def detect_conversation_list(sample_rows: list) -> Optional[FieldMapping]:
+    """Full auto-detect for conversation-list columns. Only returns a
+    mapping when both the human and assistant tags are recognized -
+    anything less certain falls through to manual mapping on purpose.
+    """
+    found = detect_list_column(sample_rows)
+    if not found:
+        return None
+
+    tags_lower = {str(t).lower(): t for t in found["tag_values"] if t is not None}
+    for human_tag, gpt_tag in _KNOWN_TAG_PAIRS:
+        if human_tag in tags_lower and gpt_tag in tags_lower:
+            return FieldMapping(
+                kind="conversation_list",
+                config={
+                    "list_field": found["list_field"],
+                    "role_key": found["role_key"],
+                    "content_key": found["content_key"],
+                    "human_tag": tags_lower[human_tag],
+                    "gpt_tag": tags_lower[gpt_tag],
+                },
+            )
+    return None
+
+
+def auto_detect(sample_rows: list) -> Optional[FieldMapping]:
+    """Try every detector in order of confidence. Returns None if nothing
+    lands cleanly - caller should fall back to manual mapping.
+    """
+    if not sample_rows:
+        return None
+    mapping = detect_conversation_list(sample_rows)
+    if mapping:
+        return mapping
+    columns = list(sample_rows[0].keys())
+    return detect_flat_pair(columns)