app.py

"""
Gradio app for Hugging Face Spaces: view detokenized samples by category.
No processing — only serves pre-built sampled.jsonl. Data path via DATA_PATH env (default: sampled.jsonl).
"""

import json
import os
from pathlib import Path

import gradio as gr


def load_sampled_jsonl(path: Path) -> dict[str, list[dict]]:
    """Load JSONL into category -> list of {path, text, num_tokens}."""
    by_cat: dict[str, list[dict]] = {}
    with open(path, encoding="utf-8") as f:
        for line in f:
            line = line.strip()
            if not line:
                continue
            rec = json.loads(line)
            cat = rec.get("category", "unknown")
            if cat not in by_cat:
                by_cat[cat] = []
            by_cat[cat].append(rec)
    return by_cat


PAGE_SIZE = 500


def format_page(records: list[dict], page: int) -> str:
    """Format one page of samples (up to PAGE_SIZE) as one string. Page is 1-based."""
    if not records:
        return ""
    start = (page - 1) * PAGE_SIZE
    end = min(start + PAGE_SIZE, len(records))
    page_recs = records[start:end]
    sep = "\n\n" + "—" * 50 + "\n\n"
    parts = [r.get("text", "") for r in page_recs]
    return sep.join(parts)


def create_demo():
    data_path = Path(os.environ.get("DATA_PATH", "sampled.jsonl")).resolve()
    if not data_path.is_file():
        raise FileNotFoundError(
            f"Data file not found: {data_path}. Upload sampled.jsonl to this Space (or set DATA_PATH)."
        )
    by_cat = load_sampled_jsonl(data_path)
    categories = sorted(by_cat.keys())

    def _page_label(page: int, max_page: int) -> str:
        return f"Page {page} of {max_page}"

    def on_category_change(cat: str | None):
        if not cat or cat not in by_cat:
            return "", (1, 1), "Page 1 of 1", gr.update(choices=["1"], value="1")
        recs = by_cat[cat]
        max_page = max(1, (len(recs) + PAGE_SIZE - 1) // PAGE_SIZE)
        text = format_page(recs, 1)
        choices = [str(i) for i in range(1, max_page + 1)]
        return text, (1, max_page), _page_label(1, max_page), gr.update(choices=choices, value="1")

    def on_next(cat: str | None, state: tuple):
        page, max_page = state
        if not cat or cat not in by_cat:
            return "", state, _page_label(1, state[1]), gr.update()
        page = min(max_page, page + 1)
        recs = by_cat[cat]
        text = format_page(recs, page)
        return text, (page, max_page), _page_label(page, max_page), gr.update(value=str(page))

    def on_prev(cat: str | None, state: tuple):
        page, max_page = state
        if not cat or cat not in by_cat:
            return "", state, _page_label(1, state[1]), gr.update()
        page = max(1, page - 1)
        recs = by_cat[cat]
        text = format_page(recs, page)
        return text, (page, max_page), _page_label(page, max_page), gr.update(value=str(page))

    def on_page_select(cat: str | None, page_str: str | None, state: tuple):
        if not cat or cat not in by_cat or not page_str:
            return "", state, _page_label(state[0], state[1])
        try:
            page = int(page_str)
        except ValueError:
            return "", state, _page_label(state[0], state[1])
        _, max_page = state
        page = max(1, min(max_page, page))
        recs = by_cat[cat]
        text = format_page(recs, page)
        return text, (page, max_page), _page_label(page, max_page)

    custom_css = """
    .pagination-row { display: flex !important; flex-wrap: nowrap !important; align-items: center !important; gap: 12px !important; }
    .gradio-container textarea { font-size: 18px !important; line-height: 1.6; }
    .gradio-container .label { font-size: 17px !important; }
    .gradio-container input, .gradio-container select { font-size: 17px !important; }
    .home-page .prose, .home-page p, .home-page h1, .home-page .markdown { font-size: 24px !important; color: #212529 !important; }
    .home-page h1 { font-size: 32px !important; color: #212529 !important; }
    .home-page .card-wrap { display: inline-block; margin: 1em 0; border-radius: 16px; box-shadow: 0 8px 24px rgba(0,0,0,0.08); padding: 4px; background: #f8f9fa; border: 1px solid #e9ecef; }
    .home-page .card-wrap .primary, .home-page .card-wrap button { font-size: 26px !important; padding: 40px 64px !important; min-height: 100px !important; border-radius: 12px !important; border: none !important; background: linear-gradient(180deg, #ffffff 0%, #f1f3f5 100%) !important; box-shadow: 0 2px 8px rgba(0,0,0,0.06) !important; cursor: pointer !important; transition: box-shadow 0.2s ease, transform 0.2s ease !important; font-weight: 600 !important; color: #212529 !important; }
    .home-page .card-wrap .primary:hover, .home-page .card-wrap button:hover { box-shadow: 0 6px 20px rgba(0,0,0,0.12) !important; transform: translateY(-2px) !important; }
    .home-page .card-desc, .home-page .card-wrap .markdown, .home-page .card-wrap p, .home-page .card-wrap div[class*="markdown"] { font-size: 20px !important; color: #212529 !important; margin-top: 0.5em !important; }
    """
    with gr.Blocks(title="Token sample viewer", css=custom_css) as demo:
        home_col = gr.Column(visible=True, elem_classes=["home-page"])
        with home_col:
            gr.Markdown("# Home")
            gr.Markdown("Select a dataset to view samples:")
            with gr.Column(elem_classes=["card-wrap"]):
                card_btn = gr.Button("OLMo3", elem_classes=["primary"])
                gr.Markdown("OLMo3 / Dolma 3 pre-training data (Allen AI)", elem_classes=["card-desc"])

        olmo3_col = gr.Column(visible=False)
        with olmo3_col:
            back_btn = gr.Button("← Back to Home")
            gr.Markdown("""
**Dataset:** OLMo3 / Dolma 3 pre-training data (Allen AI).  
Source: Hugging Face (e.g. `allenai/dolma3_pool`, `dolma3_mix_*`). Tokenized with Dolma; each **category** is one data split.  
Text below is detokenized from `.npy` for inspection; document boundaries are not preserved.
            """)
            gr.Markdown("## View detokenized samples by category")
            cat_dd = gr.Dropdown(
                choices=categories,
                label="Category",
                value=categories[0] if categories else None,
            )
            page_state = gr.State((1, 1))
            with gr.Row(elem_classes=["pagination-row"]):
                prev_btn = gr.Button("← Previous")
                next_btn = gr.Button("Next →")
                page_label = gr.Markdown("Page 1 of 1", show_label=False)
                page_dd = gr.Dropdown(choices=["1"], value="1", label="Go to page")
            text_out = gr.Textbox(
                label="Samples (max 500 per page)",
                lines=25,
            )

        def show_olmo3():
            return gr.update(visible=False), gr.update(visible=True)

        def show_home():
            return gr.update(visible=True), gr.update(visible=False)

        card_btn.click(fn=show_olmo3, outputs=[home_col, olmo3_col])
        back_btn.click(fn=show_home, outputs=[home_col, olmo3_col])
        cat_dd.change(
            fn=on_category_change,
            inputs=[cat_dd],
            outputs=[text_out, page_state, page_label, page_dd],
        )
        next_btn.click(
            fn=on_next,
            inputs=[cat_dd, page_state],
            outputs=[text_out, page_state, page_label, page_dd],
        )
        prev_btn.click(
            fn=on_prev,
            inputs=[cat_dd, page_state],
            outputs=[text_out, page_state, page_label, page_dd],
        )
        page_dd.change(
            fn=on_page_select,
            inputs=[cat_dd, page_dd, page_state],
            outputs=[text_out, page_state, page_label],
        )
        demo.load(
            fn=on_category_change,
            inputs=[cat_dd],
            outputs=[text_out, page_state, page_label, page_dd],
        )
    return demo


demo = create_demo()
port = int(os.environ.get("PORT", "7860"))
demo.launch(server_name="0.0.0.0", server_port=port)