app.py

"""Sketchnote — Gradio app.

Upload a PDF, pick a chapter range, and get a whiteboard sketch-animation video
with synced Kokoro narration, chapter by chapter. Heavy models (MiniCPM,
Nemotron Parse, optional SDXL-Turbo) run on Modal; light work runs here.

Built for the Hugging Face "Build Small" hackathon — every model is < 32B and
fully open-weight / self-hosted (no proprietary hosted model APIs).
"""
from __future__ import annotations

import logging
import traceback

import gradio as gr

from pipeline import llm, pdf_parser, sketch, tts, video
from pipeline import visuals as visuals_mod

logging.basicConfig(level=logging.INFO)
log = logging.getLogger("sketchnote.app")

VOICES = ["af_heart", "af_bella", "af_sarah", "am_michael", "am_adam"]


def _page_range(start: int, end: int):
    start, end = int(start or 0), int(end or 0)
    if start > 0 and end >= start:
        return (start - 1, end - 1)  # UI is 1-based; parser is 0-based
    return None


def run_pipeline(pdf_file, max_chapters, page_start, page_end, voice, use_sdxl,
                 progress=gr.Progress()):
    """Ingest -> per chapter {summarize, visual, tts, sketch, mux} -> concat."""
    if not pdf_file:
        return None, "Please upload a PDF first."
    pdf_path = pdf_file if isinstance(pdf_file, str) else pdf_file.name

    progress(0.05, desc="Reading PDF and splitting chapters…")
    chapters = pdf_parser.extract_chapters(
        pdf_path, max_chapters=int(max_chapters), page_range=_page_range(page_start, page_end))
    if not chapters:
        return None, "Could not extract any chapters from this PDF."

    clips, transcript, warnings = [], [], []
    n = len(chapters)
    for i, ch in enumerate(chapters):
        frac = 0.1 + 0.8 * (i / max(1, n))
        progress(frac, desc=f"Chapter {i + 1}/{n}: {ch['title'][:40]}…")
        try:
            clip = _build_chapter(ch, voice, use_sdxl)
            clips.append(clip["path"])
            transcript.append(clip["md"])
            if clip.get("warning"):
                warnings.append(clip["warning"])
        except Exception:  # noqa: BLE001 — never let one chapter kill the run
            log.error("Chapter %d failed:\n%s", i + 1, traceback.format_exc())
            transcript.append(f"### {i + 1}. {ch['title']}\n\n_(skipped — error)_\n")

    if not clips:
        return None, "All chapters failed to render. See logs.\n\n" + "\n".join(transcript)

    progress(0.92, desc="Stitching chapters into the final video…")
    final = video.concat(clips)
    progress(1.0, desc="Done!")

    warn_banner = ""
    if warnings:
        unique = list(dict.fromkeys(warnings))  # dedupe, preserve order
        warn_banner = "\n".join(f"> ⚠️ **{w}**" for w in unique) + "\n\n"
    return final, warn_banner + "\n".join(transcript)


def _build_chapter(ch: dict, voice: str, use_sdxl: bool) -> dict:
    """Render a single chapter to a muxed clip using per-beat synchronization.

    For each beat:
      1. Synthesize just that beat's sentence with Kokoro → get duration d.
      2. Render the cumulative visual (nodes 0..k). With ``use_sdxl`` this is the
         chapter's SDXL illustration with our labels overlaid; otherwise the
         hand-drawn storyboard diagram.
      3. animate_beat reveals only the new label k over d seconds (prior labels
         stay drawn; drawing finishes at ~75 % then holds).
      4. Mux beat audio onto the beat clip.
    All beat clips are concatenated into the chapter clip.

    Returns {path, md, warning}.
    """
    summary = llm.summarize_chapter(ch["title"], ch["text"])
    beats = summary.get("beats") or []
    warning = summary.get("warning")  # surfaced in the UI transcript

    # Graceful fallback: if we somehow have no beats at all, one sentence.
    if not beats:
        beats = [{"say": ch["text"][:300] or f"This section covers {ch['title']}.",
                  "node": ch["title"][:40], "connects_to": None}]

    # Image-only mode: generate ONE text-free illustration for the chapter and
    # overlay our labels on it. Falls back to the diagram if SDXL is unavailable.
    bg_path = visuals_mod.chapter_illustration(ch["title"], beats) if use_sdxl else None

    beat_clips: list[str] = []
    beat_mds: list[str] = []
    prev_png: str | None = None

    for k, beat in enumerate(beats):
        log.info("Chapter %r beat %d/%d: node=%r", ch["title"], k + 1, len(beats),
                 beat["node"])
        # Cumulative visual up to (and including) beat k. The FULL beats list is
        # passed so layout is computed once; ``upto`` fills in nodes 1..k+1 at
        # their final fixed positions (prior nodes never move between beats).
        if bg_path:
            full_png = visuals_mod.build_image_label_frame(
                beats, ch["title"], bg_path, upto=k + 1)
        else:
            full_png = visuals_mod.build_storyboard_frame(
                beats, ch["title"], upto=k + 1)

        wav_path, duration = tts.synthesize(beat["say"], voice=voice)
        silent = sketch.animate_beat(full_png, prev_png, target_duration=duration)
        clip = video.mux(silent, wav_path)
        beat_clips.append(clip)
        beat_mds.append(f"- **{beat['node']}**: {beat['say']}")
        prev_png = full_png

    chapter_clip = video.concat(beat_clips) if len(beat_clips) > 1 else beat_clips[0]

    warn_md = f"\n\n> ⚠️ **{warning}**" if warning else ""
    md = (f"### {ch['title']}{warn_md}\n\n"
          + "\n".join(beat_mds) + "\n")
    return {"path": chapter_clip, "md": md, "warning": warning}


def build_ui() -> gr.Blocks:
    with gr.Blocks(title="Sketchnote") as demo:
        gr.Markdown(
            "# ✏️ Sketchnote\n"
            "Turn a PDF (e.g. a textbook) into a **whiteboard sketch-animation "
            "video with synced narration**, chapter by chapter. "
            "All models are open-weight and under 32B parameters.")
        with gr.Row():
            with gr.Column(scale=1):
                pdf_in = gr.File(label="Upload PDF", file_types=[".pdf"], type="filepath")
                max_ch = gr.Slider(1, 8, value=3, step=1, label="Max chapters")
                with gr.Row():
                    p_start = gr.Number(value=0, precision=0, label="First page (0 = auto)")
                    p_end = gr.Number(value=0, precision=0, label="Last page (0 = auto)")
                voice = gr.Dropdown(VOICES, value="af_heart", label="Narration voice")
                use_sdxl = gr.Checkbox(value=True,
                                       label="AI illustration + labels (SDXL-Turbo "
                                             "on Modal) — off uses a hand-drawn "
                                             "concept diagram")
                go = gr.Button("Generate video", variant="primary")
                gr.Examples(examples=[["assets/sample.pdf"]], inputs=[pdf_in],
                            label="Try the sample PDF")
            with gr.Column(scale=1):
                video_out = gr.Video(label="Sketchnote video")
                transcript_out = gr.Markdown(label="Per-chapter transcript")
        go.click(run_pipeline,
                 inputs=[pdf_in, max_ch, p_start, p_end, voice, use_sdxl],
                 outputs=[video_out, transcript_out])
        gr.Markdown(
            "_Tip: keep the chapter range small for a fast demo. If Modal isn't "
            "deployed, Sketchnote falls back to a non-AI extractive summary so it "
            "still produces a video._")
    return demo


demo = build_ui()

if __name__ == "__main__":
    demo.queue().launch()