Deploy AI Prof hackathon submission

.env.example

@@ -0,0 +1,59 @@
+# AI Prof configuration.
+# Leave everything blank to run in MOCK mode (no weights needed — great for UI dev / demo recording).
+
+# --- Eyes: MiniCPM-V (vision) ---
+# Hosted on Modal through llama.cpp (modal_app_vision.py).
+# Bring up with: modal run modal_app_vision.py::download_model
+#                modal run modal_app_vision.py::warm
+#                modal deploy modal_app_vision.py
+# Set VISION_BASE_URL to the printed serve URL, with or without /v1.
+VISION_BASE_URL=
+VISION_API_KEY=sk-no-key-required
+VISION_MODEL=minicpm-v
+
+# --- Brain: Nemotron 3 Nano (explanation + Q&A) ---
+# Any OpenAI-compatible chat endpoint (vLLM, llama.cpp, etc.)
+#   vllm serve nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16 --port 8000
+BRAIN_BASE_URL=
+BRAIN_API_KEY=sk-no-key-required
+BRAIN_MODEL=nemotron-3-nano
+
+# --- Voice (TTS): VoxCPM2 served via vLLM-Omni (modal_app_vox.py) ---
+# Used by the WebRTC voice layer (ai_prof/rtc.py) to speak AI Prof answers aloud.
+# Bring up with: modal run modal_app_vox.py::download_model
+#                modal run modal_app_vox.py::warm
+#                modal deploy modal_app_vox.py
+# Set TTS_BASE_URL to either the printed *.modal.run URL or that URL + /v1.
+# Leave blank to fall back to a silent-audio mock (voice UI still works, just mute).
+TTS_BASE_URL=
+TTS_API_KEY=sk-no-key-required
+TTS_MODEL=voxcpm2
+# Use a precomputed vLLM-Omni voice name when available. Otherwise AI Prof
+# creates one voice on the first utterance and reuses it for the lecture.
+TTS_VOICE=default
+
+# --- Voice (STT): distil-whisper-large-v3 via faster-whisper-server (modal_app_vox.py) ---
+# Used to transcribe student microphone audio (push-to-talk + WebRTC interjections).
+# Bring up with: modal run modal_app_vox.py::download_stt
+#                modal deploy modal_app_vox.py
+# Set STT_BASE_URL to the `transcribe` function URL, with or without /v1.
+# Leave blank to fall back to a "[voice input]" mock.
+STT_BASE_URL=
+STT_API_KEY=sk-no-key-required
+STT_MODEL=distil-whisper-large-v3
+
+# Rendering DPI for slide images (higher = sharper, slower vision pass)
+SLIDE_DPI=150
+
+# --- Processed deck cache ---
+# Every uploaded PDF is hashed. Rendered slide images, text layers, MiniCPM-V
+# readings, and the complete deck index are cached locally under DECK_CACHE_DIR.
+DECK_CACHE_DIR=.cache/ai-prof/decks
+
+# Optional shared Hugging Face dataset cache. Public repos can be read without a
+# token; private repos require HF_TOKEN. Keep writes disabled for arbitrary user
+# uploads unless the dataset is private or you have explicit permission to
+# publish the lecture material.
+HF_DECK_CACHE_REPO=
+HF_TOKEN=
+HF_DECK_CACHE_WRITE=false

.gitignore

@@ -0,0 +1,39 @@
+# Python
+__pycache__/
+*.py[cod]
+*.egg-info/
+.venv/
+venv/
+env/
+.python-version
+
+# Models / weights / data (keep large + private out of git)
+*.gguf
+*.safetensors
+*.bin
+models/
+weights/
+slides/
+data/
+.cache/
+*.pdf
+
+# Secrets / env
+.env
+.env.*
+!.env.example
+*.key
+
+# Agent instructions (kept local, not published)
+AGENTS.md
+CLAUDE.md
+
+# OS / editor
+.DS_Store
+.idea/
+.vscode/
+*.log
+
+# Gradio
+.gradio/
+flagged/

ARCHITECTURE.md

@@ -0,0 +1,240 @@
+# AI Prof Architecture
+
+This document captures the intended production and demo architecture for AI Prof.
+The core experience is a professor-like agent that understands an entire lecture,
+controls the visible slide and whiteboard, speaks naturally, and responds to student
+interruptions without losing its place.
+
+## Deployment
+
+```text
+Hugging Face Space
+  Gradio UI
+  session state
+  professor orchestrator
+  slide and whiteboard rendering
+            |
+            v
+Modal inference services
+  MiniCPM-V slide vision
+  Nemotron professor agent
+  VoxCPM text-to-speech
+  Whisper or Moonshine speech-to-text
+            |
+            v
+Hugging Face storage
+  published demo deck
+  processed deck manifests
+  slide images and readings
+```
+
+- **Hugging Face Space:** public Gradio application and lightweight orchestration.
+- **Modal:** GPU-backed, self-hosted model inference that can scale down when idle.
+- **Local development:** the same Gradio app in mock mode or pointed at local model
+  servers.
+- **Hugging Face storage:** persistent processed lectures, including one polished
+  public demo deck that works immediately for judges and visitors.
+
+## Deck Preparation
+
+The professor should not begin teaching after only slide 1 is processed. Navigation
+and question answering depend on knowing what exists across the whole lecture.
+
+On upload:
+
+1. Hash the PDF to identify an existing processed deck.
+2. If cached, load its manifest and slide assets.
+3. Otherwise, render every page and extract the PDF text layer.
+4. Run MiniCPM-V over every slide.
+5. Build a compact deck index from all slide readings.
+6. Persist the processed result when appropriate.
+7. Enable the lecture only when the complete index is ready.
+
+Gradio displays preparation progress while this runs. It does not need to hold the
+heavy processing itself; it can call Modal jobs and stream their progress.
+
+### Processed Deck Format
+
+```text
+decks/<pdf_sha256>/
+  manifest.json
+  source.pdf
+  slides/
+    001.png
+    002.png
+  readings/
+    001.json
+    002.json
+```
+
+Each index entry should remain compact enough to keep the entire deck map in the
+agent context:
+
+```json
+{
+  "slide": 7,
+  "title": "Convolution",
+  "summary": "Applying a kernel across an image",
+  "concepts": ["kernel", "stride", "weighted sum"],
+  "equations": ["g(x,y) = sum_i sum_j h(i,j)f(x-i,y-j)"],
+  "visuals": ["A 3 by 3 kernel moving across a pixel grid"]
+}
+```
+
+The agent receives the complete compact index, but only the full reading for the
+current or specifically retrieved slides.
+
+## Professor Agent
+
+The model should make decisions at meaningful teaching boundaries, not once per
+sentence or drawing stroke. One agent turn produces a short **teaching beat**:
+
+```json
+{
+  "narration": "Imagine this grid is a small patch of the image.",
+  "actions": [
+    {"tool": "draw_grid", "args": {"rows": 3, "cols": 3}, "at": 0.4}
+  ],
+  "next": "continue"
+}
+```
+
+The orchestrator executes the actions and speech. After the beat completes, it asks
+the agent what to do next.
+
+### Agent Context
+
+Each decision receives:
+
+- Complete compact deck index
+- Current slide number and full cached slide reading
+- Current whiteboard state
+- Recent conversation and teaching beats
+- Saved lecture position
+- Trigger: continue lecture or student question
+
+### Tools
+
+- `goto_slide(index)` - move to the best supporting slide
+- `next_slide()` and `prev_slide()` - ordinary navigation
+- `look_closer(question)` - ask MiniCPM-V to inspect the current slide for a
+  specific visual detail; wire this after the core loop
+- `write_latex(expression, position)` - place a typeset equation
+- `draw_diagram(spec)` - render structured Excalidraw-style primitives
+- `clear_whiteboard()` - reset the board when the visual context changes
+- `highlight_region(bbox)` - optional later enhancement
+
+Tool calls and their results should be logged as a publishable teaching-session
+trace.
+
+## Student Interruption
+
+The lecture is controlled by an explicit state machine:
+
+```text
+NARRATING
+  -> student begins speaking
+INTERRUPTING
+  -> stop TTS and cancel current generation
+LISTENING
+  -> capture speech until push-to-talk release or VAD pause
+THINKING
+  -> transcribe, search deck index, choose slide and visual support
+ANSWERING
+  -> navigate or draw when useful, then speak the answer
+RESUMING
+  -> continue from the saved teaching position
+```
+
+When a student asks a question, the agent first decides whether the current slide
+is sufficient. It may:
+
+- Answer on the current slide
+- Navigate to a more relevant earlier or later slide
+- Inspect a slide more closely
+- Draw an explanation on the whiteboard
+- Combine navigation and drawing
+
+The agent decides whether to return to the previous slide afterward. Automatically
+returning every time would make the lecture feel mechanical.
+
+The orchestrator saves the interrupted teaching beat and sentence position. For the
+first implementation, resuming at the beginning of that beat is acceptable and much
+simpler than resuming at an exact audio sample.
+
+## Speech
+
+- **TTS:** VoxCPM for professor narration.
+- **STT:** faster-whisper or Moonshine for short student questions.
+- **Transport and turn detection:** FastRTC, initially push-to-talk and later VAD
+  barge-in.
+
+Narration should be synthesized in short sentence or beat-sized chunks. This keeps
+latency low and gives the orchestrator clean cancellation boundaries.
+
+## Whiteboard
+
+Avoid unrestricted free-form drawing as the primary path. It requires too many model
+calls and is difficult to synchronize or reproduce.
+
+Use structured operations:
+
+- LaTeX for equations
+- Excalidraw-style primitives for boxes, arrows, labels, grids, and highlights
+- Optional Mermaid for diagrams where automatic layout is useful
+- Manim only for prepared showcase animations, not the live agent loop
+
+The model emits one structured drawing plan per teaching beat. The frontend animates
+the resulting primitives locally, so drawing does not require another inference call
+for every stroke.
+
+### Speech and Drawing Synchronization
+
+Each action may include an approximate offset relative to the narration:
+
+```json
+{
+  "narration": "The center pixel is replaced using all nine neighbors.",
+  "actions": [
+    {"tool": "highlight_cell", "args": {"row": 1, "column": 1}, "at": 0.2},
+    {"tool": "write_latex", "args": {"expression": "1/9 sum pixels"}, "at": 2.1}
+  ]
+}
+```
+
+The orchestrator:
+
+1. Starts TTS for the teaching beat.
+2. Executes visual actions at their approximate offsets.
+3. Waits for speech and drawing to finish.
+4. Requests the next teaching beat.
+
+This produces the feeling of talking while drawing without making an agent call per
+stroke.
+
+## Demo Path
+
+The public Space should offer two entry points:
+
+1. **Try the prepared lecture:** loads an already processed, polished deck from
+   Hugging Face storage immediately.
+2. **Upload your own lecture:** preprocesses the complete deck, displays progress,
+   then starts the professor.
+
+The prepared lecture is the reliable judging and demo-video path. User uploads prove
+the system generalizes without making the first experience depend on a long vision
+preprocessing wait.
+
+## Implementation Order
+
+1. Complete-deck manifest and compact index
+2. Prepared demo deck loading from Hugging Face storage
+3. Professor teaching-beat schema and tool executor
+4. Slide navigation tools
+5. Structured whiteboard tools and local animation
+6. VoxCPM narration with cancellable chunks
+7. Push-to-talk interruption and STT
+8. Automatic slide retrieval during questions
+9. VAD barge-in
+10. Targeted `look_closer` vision calls
+

IDEATION.md

@@ -0,0 +1,168 @@
+# Build Small Hackathon — Ideation & Decisions
+
+> Working notes from ideation. Project: **AI Prof** (primary submission).
+> Hackathon: https://huggingface.co/build-small-hackathon · Field guide: https://build-small-hackathon-field-guide.hf.space/
+
+## Hackathon constraints (the rules that shape everything)
+- **Model size: ≤32B parameters PER MODEL** (not aggregate). You can freely combine multiple
+  small models as long as each one individually stays under the cap. ("not just active params")
+- **No sponsor exclusivity** — a sponsor's model just has to be *a core part of the experience*;
+  you may mix in other providers' models.
+- **Platform:** must be built in **Gradio**, hosted as a **Hugging Face Space**.
+- **Deliverables:** working Space + **demo video** + **social media post**.
+- **Timeline:** hack window June 5–15, 2026.
+- **Credits:** OpenAI $100 is for **Codex (their coding agent)**, *not* API/inference. Modal $250, HF $20.
+
+## Tracks
+- **Backyard AI** — solve a genuine problem for *someone you personally know*. Judged on specificity,
+  **actual user adoption**, and appropriate model fit. ← our track.
+- **Thousand Token Wood (TTW)** — delightful/unconventional, AI must be load-bearing (game, story, art).
+
+## Prize surface (it stacks — one app can hit many)
+Main tracks ($18k): 1st–4th per track + Community Choice ($2k).
+Sponsor awards:
+- **OpenBMB $10k** — build with MiniCPM (incl. vision MiniCPM-V / omni MiniCPM-o).
+- **OpenAI $10k** — won via **Codex-attributed commits** in the repo/Space (build-tool track, model-agnostic).
+- **NVIDIA** — two RTX 5080 GPUs; build with **Nemotron**. One for "best space", one for community engagement (likes).
+- **Modal $20k credits** — use Modal for dev/runtime, note in README.
+Special awards ($8k): Bonus Quest Champion $2k · Off-Brand (best custom UI via `gr.Server`) $1.5k ·
+Tiny Titan (best app on ≤4B model) $1.5k · Best Demo $1k · Best Agent $1k · Judges' Wildcard $1k.
+Six merit badges: Off the Grid (no cloud APIs) · Well-Tuned (publish fine-tune) · Off-Brand (custom UI) ·
+Llama Champion (llama.cpp runtime) · Sharing is Caring (publish agent trace) · Field Notes (build blog).
+
+## Decision: build the AI Prof (Backyard AI)
+**Problem (real):** a specific classmate finds that having the slides isn't enough — they're static and
+lack the in-class explanation. Test with **real, anonymized lecture slides** from our class.
+
+**Core loop:** upload lecture PDF → model reads each slide *as an image* → explains it like a TA,
+streamed in real time → classmate can **interject** with a question at any moment.
+
+### Two-model architecture (stacks OpenBMB + NVIDIA, cleanly justified)
+- **MiniCPM-V (~4.1B, GGUF)** = *the eyes.* Reads each slide as an image (diagrams, equations, layout —
+  not just scraped text). Core → satisfies **OpenBMB**. Runs via **llama.cpp** → Llama Champion badge.
+- **Nemotron 3 Nano (9B, or 30B-A3B MoE = only ~3.6B active → fast decode)** = *the brain.* Turns the
+  slide reading into the spoken explanation and answers interjections. Reasoning/agentic → **NVIDIA** fit.
+- Per-model cap means no param-budget tension between the two. Division of labor (see/explain) survives
+  the "core part of the experience" test → not sponsor-stuffing.
+
+### Prize map for this one app
+Backyard placement · OpenBMB $10k · NVIDIA RTX 5080 · OpenAI $10k (Codex commits) · Modal $20k (self-host) ·
+Off-Brand $1.5k+badge (custom whiteboard UI) · Llama Champion · Off the Grid · Sharing is Caring (publish a
+teaching-session trace) · Field Notes (blog) · Best Demo. → realistically 6+ surfaces.
+
+### Scope discipline (vertical slice order)
+1. Slide upload → MiniCPM-V reads → Nemotron explains, **streamed** in a simple custom UI. *(Submittable alone.)*
+2. Text interjection (pause, ask, resume).
+3. Whiteboard — model emits **Mermaid / Excalidraw JSON** (structured; small models do this reliably).
+   Avoid freeform tldraw generation — that's the day-eating trap.
+4. Voice / TTS — only with slack.
+
+## Real-time architecture
+Goal: feel like a *live* lecture — explanation streams as if the prof is talking through each slide,
+smooth slide-to-slide, interruptible mid-sentence.
+- **Token streaming:** stream Nemotron output to the UI (Gradio generators) so it appears as it's generated.
+- **Complete index before teaching:** process the full deck before starting the lecture. The professor needs a
+  global map to choose slides intelligently and answer interjections. Show preparation progress in Gradio.
+- **Two-stage per slide, cached:** (a) MiniCPM-V → a structured "slide reading" (text + diagram desc + equations),
+  computed once and cached; (b) Nemotron → the explanation. Interjections reuse cached (a), never re-run vision.
+- **Interjection = interrupt + branch:** need a *cancellable* generation (async cancel token / threading.Event).
+  On input: stop current stream, answer using the cached slide reading + history as context, then resume.
+- **Streaming TTS (optional):** chunk explanation into sentences, TTS each as generated, play sequentially.
+  Barge-in (interrupt-on-speech) is hard mode — defer.
+- **Session state:** { slides[], current_index, cached_slide_readings{}, conversation_history }.
+- **Hosting tradeoff for low latency:** free HF Space hardware likely too slow for 2 models in real time.
+  Lean: **Modal GPU as inference backend** (uses the Modal track) + Gradio Space as frontend. Self-hosting
+  (not an external inference API) still supports the *Off the Grid* badge.
+
+## Voice (STT + TTS) — makes it feel like *class*, but it's the deepest rabbit hole
+Two more tiny, on-HF models (free under the ≤32B-*per-model* cap; FAQ uses "a 7B speech model" as its example).
+Keep both **open + self-hosted** (not ElevenLabs/Deepgram/OpenAI *API*) → protects the **Off the Grid** badge.
+
+**Pipeline:** TTS (out) speaks Nemotron's streamed explanation; STT (in) transcribes the spoken interjection
+into text for Nemotron; **VAD** is the referee that detects *when* the classmate starts talking → triggers barge-in.
+
+- **STT (interjections) — pick: Whisper, or Moonshine for speed.**
+  - `faster-whisper` / `distil-whisper` (large-v3 ~1.5B, or base/small for latency) — accurate, OpenAI open weights.
+  - **Moonshine** (~tiny) — built for real-time/on-device, faster time-to-text on short clips.
+  - Interjections are short → small model is fine; *time-to-text*, not accuracy, is the bottleneck. Start with
+    `faster-whisper-base`, switch to Moonshine only if laggy.
+- **TTS (narration) — pick: Kokoro, fallback Piper.**
+  - **Kokoro-82M** (`hexgrad/Kokoro-82M`) — 82M, good quality, fast time-to-first-audio, streamable. Sweet spot.
+  - **Piper** — even lower latency, CPU-friendly, slightly more robotic. Use if speech layer isn't on GPU.
+  - Stream sentence-by-sentence: synthesize + play each sentence as Nemotron emits it (audio ~1 sentence behind gen).
+- **Barge-in / turn-taking — use FastRTC** (`fastrtc`, the Gradio/HF WebRTC stack). Gives low-latency mic+playback
+  over WebRTC, built-in **VAD turn detection** (`ReplyOnPause`), and the hook to cancel playback + generation the
+  instant the user speaks. Avoids hand-rolling silence detection; reuses our cancellable-generation design.
+  - Loop: narrate (TTS) → Silero VAD hears speech → kill TTS + cancel Nemotron → buffer to end-of-speech → STT →
+    answer using the **cached slide reading** as context → TTS answer → resume narration.
+- **Latency budget (what "feels live" needs):** minimize time-to-first-audio. STT small (~100–300ms) + Nemotron
+  MoE fast prefill + Kokoro first chunk (~tens of ms). Run STT/TTS/VAD on the **same Modal GPU** as Nemotron
+  (or CPU for Piper+Silero) to avoid network hops.
+- **Scope ladder (don't let voice eat the week):**
+  1. **TTS narration only** — Prof talks, classmate *types* to interject. Low risk, already feels real-time.
+  2. **Push-to-talk interject** — hold key / tap to ask aloud → STT. No VAD/barge-in. ~90% of magic, ~30% of work.
+  3. **Full-duplex barge-in** via FastRTC + VAD — only once 1–2 are solid. The 2→3 jump is where time goes.
+- **Model count after voice:** MiniCPM-V (vision) + Nemotron (brain) + Whisper/Moonshine (STT) + Kokoro/Piper (TTS)
+  + Silero VAD. Multi-model is explicitly blessed; more "appropriate model fit" surface, but more integration.
+
+## Agent loop, tools & slide grounding
+The brain (Nemotron) runs as a **tool-using agent** driving the lecture — professor-esque: decides which slide
+to be on, when to draw vs. just talk, when to jump back. (Strengthens the **Best Agent** award; the tool-call
+sequence *is* the publishable agent trace → **Sharing is Caring** badge.)
+
+**Tools (mutate UI / session state):**
+- `goto_slide(i)` / `next_slide()` / `prev_slide()` — navigation; lets it jump back to a referenced slide.
+- `look_closer(question)` — on-demand **real-time** MiniCPM-V call on the *current slide image* for detail.
+- `draw(mermaid | excalidraw_json)` — render on the whiteboard surface.
+- `clear_whiteboard()`
+- `highlight_region(bbox)` — optional, later.
+- narration = the free-text part of the response → streamed to TTS.
+
+**Control flow:** orchestrator loop. Each turn the agent gets `{ current slide reading, deck outline, recent
+history, trigger }` where trigger = "continue lecture" | "user asked: <q>". It returns narration + optional
+tool calls; the orchestrator executes tools (swap displayed slide, render whiteboard) and streams narration.
+**Reserve heavy reasoning for decision points (between slides), not during narration**, or latency balloons.
+
+**Grounding — preprocess (breadth) + real-time (depth), both:**
+- **On upload (once; complete before lecture begins):**
+  - render each slide → image (serves *both* display and vision),
+  - **MiniCPM-V** → cached structured slide reading (title, bullets, equations, diagram desc, key concepts),
+  - **PDF text-layer** extraction (exact text ground-truth; vision misreads text) to complement vision,
+  - build a **deck outline / index** (slide → title/concepts) — this is what lets the agent *plan* and *pick* a
+    slide; it can't navigate to a slide it has never seen.
+- **During lecture (`look_closer`):** targeted MiniCPM-V look at the actual slide *with the specific question in
+  context* — handles the long tail (specific visual questions, detail the cached summary glossed over).
+- Why both: preprocess = global map + fast/cheap narration + navigation; real-time = accuracy on specific asks.
+
+**UI: show the REAL slide, never a summary.**
+- Main surface = the actual rendered slide **image / PDF page**, synced to the agent's `current_slide`.
+- **Whiteboard = a separate adjacent canvas** (Mermaid/Excalidraw render) so drawings read as the prof's
+  annotations, not edits to the original slide. (Region-highlight overlay on the slide can come later.)
+- Plus: live caption / transcript + mic / interject control.
+
+The detailed deployment, deck-cache, teaching-beat, interruption, speech, and whiteboard decisions are in
+[`ARCHITECTURE.md`](ARCHITECTURE.md).
+
+## Fine-tuning (after core pipeline — not before)
+Chosen direction: **teaching-style SFT / guided learning** — tune the brain (Nemotron) to explain like a good
+TA: analogies, checks for understanding, concise, no preamble. Bootstrap dataset = (slide reading → ideal
+explanation) pairs generated with a strong model + a little hand-curation; **QLoRA via TRL / Unsloth on Modal**;
+publish the adapter to HF → **Well-Tuned** badge (+ feeds Bonus Quest Champion). Put a before/after note in the
+model card. Strictly a step-2 enhancement — only once the live pipeline works and there's a clear objective.
+
+## Backburner ideas (TTW — only if a 2nd submission is feasible)
+- **Emotion-driven TRPG / choose-your-own-adventure:** free text → LLM updates structured NPC emotional
+  state (trust/fear/affection…) → state steers the story. Best Agent + Sharing is Caring (emotion-delta traces).
+- **AI Garlic Phone:** message/drawing passed down a chain of model personas, mutating; needs a reveal payoff.
+  If a drawing is passed, pulls in vision = OpenBMB again.
+- **Sketch-to-story adventure:** you draw your action, MiniCPM-V interprets the doodle, the world reacts.
+  Fuses vision + emotion mechanic. Strong demo, only-AI-can-do-this.
+- **Model-vs-model battle → traces for training** (browser-brawl-style, different domain): most technically
+  impressive (Best Agent + Well-Tuned + Sharing is Caring) but research-shaped; the training loop can eat the week.
+
+## Key model references
+- MiniCPM-V-4 (4.1B, multimodal): https://huggingface.co/openbmb/MiniCPM-V-4
+- MiniCPM-V-4 GGUF (llama.cpp): https://huggingface.co/openbmb/MiniCPM-V-4-gguf
+- Nemotron 3 Nano collection: https://huggingface.co/collections/nvidia/nvidia-nemotron-v3
+- Nemotron 3 Nano 4B: https://huggingface.co/blog/nvidia/nemotron-3-nano-4b
+- Nemotron 3 Nano 30B-A3B: https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16

README.md

@@ -1,13 +1,153 @@
 ---
-title: Ai Prof
-emoji: 📈
-colorFrom: gray
-colorTo: gray
+title: AI Prof
+emoji: 🎓
+colorFrom: indigo
+colorTo: purple
 sdk: gradio
-sdk_version: 6.18.0
-python_version: '3.13'
+sdk_version: 5.50.0
 app_file: app.py
-pinned: false
+python_version: "3.12"
+tags:
+  - track:backyard
+  - sponsor:openbmb
+  - sponsor:openai
+  - sponsor:nvidia
+  - sponsor:modal
+  - achievement:offbrand
+  - achievement:llama
+  - achievement:fieldnotes
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# AI Prof
+
+A real-time AI teaching assistant for the [Build Small Hackathon](https://huggingface.co/build-small-hackathon)
+(**Backyard AI** track).
+
+Upload your real lecture slides and AI Prof walks through them one by one — reading each slide *as an image*
+(diagrams, equations, layout, not just scraped text) and explaining it like a TA would in class. Ask a
+question at any time and it pauses, answers, and picks back up. Built for a classmate who has the slides but
+misses the in-class explanation.
+
+## Demo
+
+- **Live Space:** **TODO before submission — add the public Space URL here**
+- **Demo video:** **TODO before submission — add the public video URL here**
+- **Social post:** **TODO before submission — add the public post URL here**
+- **Team:** [@pranavkarthik10](https://huggingface.co/pranavkarthik10)
+
+### Judge-friendly walkthrough
+
+1. Upload a lecture PDF.
+2. Watch AI Prof index every slide before teaching, giving the agent a map of the complete lecture.
+3. Start the lecture and see the professor explain slides aloud while adding notes or equations to the whiteboard.
+4. Interrupt with a typed or spoken question. The professor can answer, navigate to a supporting slide, and continue.
+
+## How it works
+- **Eyes — [MiniCPM-V](https://huggingface.co/openbmb/MiniCPM-V-4) (~4.1B, GGUF / llama.cpp):** reads each slide image.
+- **Brain — [Nemotron 3 Nano](https://huggingface.co/collections/nvidia/nvidia-nemotron-v3):** turns the slide
+  reading into a teaching plan, selects tools, explains concepts, and answers interjections.
+- **Voice:** VoxCPM2 speaks the professor's response; distil-Whisper transcribes student questions.
+- **Runtime:** Modal hosts the four OpenAI-compatible inference services and scales them down when idle.
+- **Frontend:** a custom Gradio classroom UI hosted as a Hugging Face Space.
+
+Every model stays under the hackathon's ≤32B-per-model cap.
+
+## Why it is agentic
+
+AI Prof does more than summarize a PDF. Nemotron receives a compact index of the
+entire deck, the current slide reading, recent conversation, and whiteboard state.
+For every teaching beat it returns validated narration and actions. The orchestrator
+can navigate slides, write a note, typeset an equation, clear the board, or continue
+the lecture. Student questions cancel the active lecture turn and trigger a new plan
+grounded in the full deck.
+
+Current tools: `goto_slide`, `next_slide`, `prev_slide`, `write_note`,
+`write_latex`, and `clear_whiteboard`.
+
+## Small-model stack
+
+| Role | Model | Size / serving |
+| --- | --- | --- |
+| Slide vision | MiniCPM-V-4 | ~4.1B, Q4_K_M GGUF with llama.cpp |
+| Professor agent | NVIDIA Nemotron 3 Nano | 30B total / 3B active, vLLM |
+| Speech synthesis | VoxCPM2 | vLLM-Omni |
+| Speech recognition | distil-Whisper large-v3 | faster-whisper |
+
+## Run it
+```bash
+uv venv --python 3.12 && uv pip install -r requirements.txt
+.venv/bin/python app.py        # opens the Gradio UI
+```
+Out of the box it runs in **mock mode** (no weights) so you can drive the full UX —
+upload a PDF, watch it stream explanations, ask questions. To plug in the real models,
+copy `.env.example` to `.env` and point `VISION_BASE_URL` / `BRAIN_BASE_URL` at your
+OpenAI-compatible endpoints (llama.cpp `llama-server` for MiniCPM-V, vLLM/llama.cpp for Nemotron).
+
+MiniCPM-V is included as a Modal service:
+
+```bash
+modal run modal_app_vision.py::download_model
+modal run modal_app_vision.py::warm
+modal deploy modal_app_vision.py
+```
+
+Set `VISION_BASE_URL` to the deployed `serve` URL. The service uses MiniCPM-V-4
+Q4_K_M plus its F16 vision projector on one L4 and scales down after five idle
+minutes.
+
+## Pre-indexed deck cache
+
+AI Prof hashes each uploaded PDF and caches its rendered slide images, extracted
+text, MiniCPM-V readings, and complete deck index. Re-uploading the same deck with
+the same vision model and DPI skips indexing entirely.
+
+The cache is local by default. To share a prepared demo deck across local testing
+and the public Space, create a Hugging Face dataset repo and configure:
+
+```bash
+HF_DECK_CACHE_REPO=pranavkarthik10/ai-prof-decks
+HF_TOKEN=hf_...
+HF_DECK_CACHE_WRITE=true
+```
+
+Upload the demo PDF once while writes are enabled. The processed deck is stored
+under `decks/<content-key>/` in the dataset. For the public Space, set
+`HF_DECK_CACHE_REPO` but leave `HF_DECK_CACHE_WRITE=false`; judges can then load
+that exact deck from the **Prepared lectures** picker without uploading the PDF
+or granting the app write access.
+
+Do not enable remote writes for arbitrary uploads to a public dataset. Processed
+slides can contain the original lecture material. Use a private dataset for personal
+testing or only prewarm material you have permission to publish.
+
+## Status
+**Professor agent loop working:** upload a lecture PDF → render and index the complete
+deck → Nemotron plans short teaching beats → the orchestrator executes validated slide
+navigation and whiteboard tools → VoxCPM speaks the explanation. Student questions
+cancel active narration, can send the professor to a more relevant slide, and can add
+supporting notes or equations to the whiteboard.
+
+Targeted vision via `look_closer` and richer Excalidraw diagrams remain follow-up work.
+
+Layout: [`app.py`](app.py) (Gradio UI) · [`ai_prof/pdf_utils.py`](ai_prof/pdf_utils.py) (PDF→slides) ·
+[`ai_prof/vision.py`](ai_prof/vision.py) (eyes) · [`ai_prof/brain.py`](ai_prof/brain.py) (brain) ·
+[`ai_prof/config.py`](ai_prof/config.py) (endpoints / mock fallback).
+
+The inference services run on Modal: MiniCPM-V through llama.cpp, Nemotron
+through vLLM, VoxCPM2 through vLLM-Omni, and distil-Whisper through a small
+OpenAI-compatible FastAPI server.
+The Gradio frontend remains deployable as a Hugging Face Space.
+
+Run the focused test suite with:
+
+```bash
+.venv/bin/python -m unittest discover -s tests -v
+```
+
+Next up (see [IDEATION.md](IDEATION.md)): prepared-deck storage on Hugging Face,
+animated structured diagrams, stronger cancellation during browser audio playback,
+and targeted `look_closer` vision.
+
+The current target design is documented in [ARCHITECTURE.md](ARCHITECTURE.md), including complete-deck
+indexing, the professor tool loop, a preprocessed demo lecture, synchronized speech and whiteboard actions,
+and interruption/resume behavior.

ai_prof/__init__.py

@@ -0,0 +1,3 @@
+"""AI Prof — a real-time AI teaching assistant that reads your lecture slides and explains them."""
+
+__version__ = "0.1.0"

ai_prof/agent.py

@@ -0,0 +1,281 @@
+"""Professor agent planning.
+
+The model plans one short teaching beat at a time. The Gradio orchestrator owns
+all state mutation and validates every requested tool before executing it.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import dataclass, field
+from typing import Any, Literal
+
+from openai import OpenAI
+
+from .config import CONFIG
+
+Trigger = Literal["continue", "question"]
+ActionName = Literal[
+    "goto_slide",
+    "next_slide",
+    "prev_slide",
+    "write_note",
+    "write_latex",
+    "clear_whiteboard",
+]
+
+_PLANNER_SYSTEM = """You are the planning brain of AI Prof, a live professor.
+Plan exactly one short teaching beat at a time.
+
+You receive a compact index of the entire lecture, the full reading of the current
+slide, recent conversation, and the current whiteboard. Decide what the student
+should hear next and whether a slide or whiteboard action would help.
+
+Return JSON only:
+{
+  "narration": "Natural spoken explanation, usually 1-3 short paragraphs.",
+  "actions": [
+    {"tool": "goto_slide", "args": {"index": 4}},
+    {"tool": "write_note", "args": {"title": "Kernel", "body": "A small matrix of weights."}},
+    {"tool": "write_latex", "args": {"expression": "g(x,y)=sum_i sum_j h(i,j)f(x-i,y-j)"}},
+    {"tool": "clear_whiteboard", "args": {}}
+  ],
+  "continue_lecture": true
+}
+
+Rules:
+- Slide indices are 1-based and must exist in the supplied deck index.
+- Use at most one navigation action and at most two whiteboard actions.
+- Stay on the current slide unless another indexed slide is clearly more useful.
+- For a student question, answer it directly. Navigate only when another slide
+  materially improves the answer.
+- Use the whiteboard only when a compact note or equation improves understanding.
+- Clear it when old material would be confusing.
+- Do not call a tool just to demonstrate agency.
+- Keep narration conversational and suitable for text-to-speech.
+- Use no markdown headings and do not mention tool names."""
+
+
+@dataclass(frozen=True)
+class AgentAction:
+    tool: ActionName
+    args: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass(frozen=True)
+class TeachingBeat:
+    narration: str
+    actions: tuple[AgentAction, ...] = ()
+    continue_lecture: bool = True
+
+
+def _client() -> OpenAI:
+    return OpenAI(
+        base_url=CONFIG.brain.openai_base_url,
+        api_key=CONFIG.brain.api_key,
+    )
+
+
+def _extract_json(text: str) -> dict[str, Any]:
+    text = text.strip()
+    if text.startswith("```"):
+        text = re.sub(r"^```(?:json)?\s*", "", text)
+        text = re.sub(r"\s*```$", "", text)
+    try:
+        value = json.loads(text)
+    except json.JSONDecodeError:
+        start = text.find("{")
+        end = text.rfind("}")
+        if start < 0 or end <= start:
+            raise
+        value = json.loads(text[start : end + 1])
+    if not isinstance(value, dict):
+        raise ValueError("agent response must be a JSON object")
+    return value
+
+
+def _repair_json(text: str) -> str | None:
+    """Ask the brain to repair malformed planner JSON without re-planning."""
+    if not CONFIG.brain.is_live:
+        return None
+    try:
+        response = _client().chat.completions.create(
+            model=CONFIG.brain.model,
+            messages=[
+                {
+                    "role": "system",
+                    "content": (
+                        "Repair the supplied malformed JSON. Preserve its intended values, "
+                        "return one valid JSON object only, and add no commentary."
+                    ),
+                },
+                {"role": "user", "content": text},
+            ],
+            temperature=0,
+            max_tokens=700,
+            response_format={"type": "json_object"},
+        )
+        return response.choices[0].message.content
+    except Exception as exc:
+        print(f"[agent] JSON repair error: {exc}")
+        return None
+
+
+def _validate_actions(raw_actions: Any, total_slides: int) -> tuple[AgentAction, ...]:
+    if not isinstance(raw_actions, list):
+        return ()
+
+    valid: list[AgentAction] = []
+    navigation_count = 0
+    whiteboard_count = 0
+    for raw in raw_actions:
+        if not isinstance(raw, dict):
+            continue
+        tool = raw.get("tool")
+        args = raw.get("args") if isinstance(raw.get("args"), dict) else {}
+
+        if tool in {"goto_slide", "next_slide", "prev_slide"}:
+            if navigation_count:
+                continue
+            if tool == "goto_slide":
+                index = args.get("index")
+                if not isinstance(index, int) or not 1 <= index <= total_slides:
+                    continue
+                args = {"index": index}
+            else:
+                args = {}
+            navigation_count += 1
+        elif tool == "write_note":
+            if whiteboard_count >= 2:
+                continue
+            title = str(args.get("title", "")).strip()[:80]
+            body = str(args.get("body", "")).strip()[:500]
+            if not title and not body:
+                continue
+            args = {"title": title, "body": body}
+            whiteboard_count += 1
+        elif tool == "write_latex":
+            if whiteboard_count >= 2:
+                continue
+            expression = str(args.get("expression", "")).strip()[:500]
+            if not expression:
+                continue
+            args = {"expression": expression}
+            whiteboard_count += 1
+        elif tool == "clear_whiteboard":
+            if whiteboard_count >= 2:
+                continue
+            args = {}
+            whiteboard_count += 1
+        else:
+            continue
+        valid.append(AgentAction(tool=tool, args=args))
+    return tuple(valid)
+
+
+def _fallback_beat(
+    *,
+    trigger: Trigger,
+    current_slide: int,
+    total_slides: int,
+    current_reading: str,
+    question: str | None,
+) -> TeachingBeat:
+    title = next(
+        (
+            line.split(":", 1)[1].strip()
+            for line in current_reading.splitlines()
+            if line.upper().startswith("TITLE:")
+        ),
+        f"slide {current_slide}",
+    )
+    if trigger == "question":
+        narration = (
+            f"Let’s connect that question to {title}. {question or ''} "
+            "The important idea is how the details on this slide support the concept."
+        )
+        return TeachingBeat(narration=narration.strip(), continue_lecture=False)
+    return TeachingBeat(
+        narration=f"Let’s work through {title} and focus on the main idea.",
+        continue_lecture=current_slide < total_slides,
+    )
+
+
+def plan_teaching_beat(
+    *,
+    trigger: Trigger,
+    deck_index: str,
+    current_slide: int,
+    total_slides: int,
+    current_reading: str,
+    whiteboard_state: list[dict[str, str]] | None = None,
+    history: list[dict] | None = None,
+    question: str | None = None,
+) -> TeachingBeat:
+    """Return one validated teaching beat from the professor agent."""
+    if not CONFIG.brain.is_live:
+        return _fallback_beat(
+            trigger=trigger,
+            current_slide=current_slide,
+            total_slides=total_slides,
+            current_reading=current_reading,
+            question=question,
+        )
+
+    messages: list[dict[str, str]] = [
+        {"role": "system", "content": _PLANNER_SYSTEM}
+    ]
+    for turn in (history or [])[-6:]:
+        role = turn.get("role")
+        content = turn.get("content")
+        if role in {"user", "assistant"} and isinstance(content, str) and content:
+            messages.append({"role": role, "content": content})
+    messages.append(
+        {
+            "role": "user",
+            "content": (
+                f"Trigger: {trigger}\n"
+                f"Student question: {question or '(none)'}\n"
+                f"Current slide: {current_slide} of {total_slides}\n\n"
+                f"Complete deck index:\n{deck_index}\n\n"
+                f"Current slide reading:\n{current_reading}\n\n"
+                "Current whiteboard JSON:\n"
+                f"{json.dumps(whiteboard_state or [], ensure_ascii=True)}"
+            ),
+        }
+    )
+
+    try:
+        response = _client().chat.completions.create(
+            model=CONFIG.brain.model,
+            messages=messages,
+            temperature=0.25,
+            max_tokens=700,
+            response_format={"type": "json_object"},
+        )
+        raw = response.choices[0].message.content or ""
+        try:
+            data = _extract_json(raw)
+        except (json.JSONDecodeError, ValueError):
+            repaired = _repair_json(raw)
+            if not repaired:
+                raise
+            data = _extract_json(repaired)
+        narration = str(data.get("narration", "")).strip()
+        if not narration:
+            raise ValueError("agent returned empty narration")
+        return TeachingBeat(
+            narration=narration,
+            actions=_validate_actions(data.get("actions"), total_slides),
+            continue_lecture=bool(data.get("continue_lecture", trigger == "continue")),
+        )
+    except Exception as exc:
+        print(f"[agent] planning error: {exc}")
+        return _fallback_beat(
+            trigger=trigger,
+            current_slide=current_slide,
+            total_slides=total_slides,
+            current_reading=current_reading,
+            question=question,
+        )

ai_prof/brain.py

@@ -0,0 +1,127 @@
+"""The brain: Nemotron turns a slide reading into a streamed, TA-style explanation
+and answers interjected questions using the cached reading as context.
+"""
+
+from __future__ import annotations
+
+import time
+from collections.abc import Iterator
+from functools import lru_cache
+
+from openai import OpenAI
+
+from .config import CONFIG
+
+_SYSTEM = """You are AI Prof, teaching one student through a lecture deck in real time.
+
+Sound like a professor speaking naturally, not a study guide generating notes.
+- Open conversationally and get to the point. On the first slide, a simple opening
+  like "Hey, welcome back. Today we're looking at image filtering" is ideal.
+- Teach one or two ideas at a time in short spoken paragraphs. Connect them to the
+  previous slide when useful.
+- Explain jargon in plain language and use a compact example or analogy only when it
+  genuinely makes the idea easier to understand.
+- Guide the student toward the idea instead of exhaustively reciting every item.
+- Treat course codes, lecture numbers, citations, and credits as background metadata
+  unless they matter to the concept being taught.
+- End with a brief transition or a useful content question when one fits naturally.
+
+Never use canned headings such as "Key ideas on this slide" or "Quick check". Do not
+announce "Slide 1", narrate the slide layout, repeat all visible text, or ask whether
+the student is ready to begin or move on. Do not say "this slide shows". Keep the
+response concise enough to sound good aloud."""
+
+
+@lru_cache(maxsize=1)
+def _client() -> OpenAI:
+    return OpenAI(base_url=CONFIG.brain.openai_base_url, api_key=CONFIG.brain.api_key)
+
+
+def _stream_chat(messages: list[dict]) -> Iterator[str]:
+    if not CONFIG.brain.is_live:
+        yield from _mock_stream(messages)
+        return
+    stream = _client().chat.completions.create(
+        model=CONFIG.brain.model,
+        messages=messages,
+        temperature=0.6,
+        max_tokens=700,
+        stream=True,
+    )
+    for chunk in stream:
+        delta = chunk.choices[0].delta
+        if delta.content:
+            yield delta.content
+
+
+def _mock_stream(messages: list[dict]) -> Iterator[str]:
+    """Token-by-token mock so the streaming UX is visible without a brain endpoint."""
+    user = messages[-1]["content"]
+    text = (
+        "(mock brain — set BRAIN_BASE_URL for real Nemotron output) "
+        "Here's the gist of this slide: "
+        + " ".join(user.split())[:300]
+        + " ... The key idea to hold onto is how these pieces connect. "
+        "Does that part make sense so far?"
+    )
+    for tok in text.split(" "):
+        yield tok + " "
+        time.sleep(0.02)
+
+
+def explain_slide(
+    reading: str,
+    *,
+    slide_no: int,
+    total: int,
+    outline: str,
+    history: list[dict] | None = None,
+) -> Iterator[str]:
+    """Stream a TA-style explanation of the current slide."""
+    messages = [{"role": "system", "content": _SYSTEM}]
+    for turn in (history or [])[-6:]:
+        if turn.get("content"):
+            messages.append(turn)
+    transition = (
+        "This is the opening slide. Welcome the student briefly and introduce today's topic."
+        if slide_no == 1
+        else "Continue naturally from the previous explanation without greeting the student again."
+    )
+    messages.append(
+        {
+            "role": "user",
+            "content": (
+                f"Lecture deck outline:\n{outline}\n\n"
+                f"Current position: slide {slide_no} of {total}.\n"
+                f"Grounded slide reading:\n{reading}\n\n"
+                f"{transition} Teach the important idea conversationally. Do not turn the "
+                "response into a structured slide summary."
+            ),
+        }
+    )
+    yield from _stream_chat(messages)
+
+
+def answer_question(
+    question: str,
+    *,
+    reading: str,
+    slide_no: int,
+    history: list[dict] | None = None,
+) -> Iterator[str]:
+    """Stream an answer to an interjection, grounded in the current slide reading."""
+    messages = [{"role": "system", "content": _SYSTEM}]
+    for turn in (history or [])[-6:]:
+        messages.append(turn)
+    messages.append(
+        {
+            "role": "user",
+            "content": (
+                f"We're on slide {slide_no}. Its reading:\n{reading}\n\n"
+                f"The student asks: {question}\n\n"
+                "Answer directly and conversationally. Relate the answer to the current "
+                "topic, then smoothly hand control back to the lecture without a canned offer."
+            ),
+        }
+    )
+    yield from _stream_chat(messages)

ai_prof/config.py

@@ -0,0 +1,101 @@
+"""Runtime configuration for AI Prof.
+
+Both models are reached over OpenAI-compatible HTTP, which is what the common
+self-hosted runtimes expose:
+  - MiniCPM-V (eyes)  -> llama.cpp ``llama-server`` with ``--mmproj``
+  - Nemotron (brain)  -> vLLM / llama.cpp chat endpoint
+
+If a base URL is left blank, that model falls back to MOCK mode so the whole
+app still runs end-to-end with no weights — useful for UI work and demos.
+"""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+
+from dotenv import load_dotenv
+
+load_dotenv()
+
+
+def _clean(value: str | None) -> str | None:
+    """Treat blank / placeholder env values as unset."""
+    if value is None:
+        return None
+    value = value.strip()
+    return value or None
+
+
+@dataclass(frozen=True)
+class ModelConfig:
+    base_url: str | None
+    api_key: str
+    model: str
+
+    @property
+    def is_live(self) -> bool:
+        return self.base_url is not None
+
+    @property
+    def openai_base_url(self) -> str | None:
+        """Return an OpenAI-compatible base URL ending in /v1."""
+        if self.base_url is None:
+            return None
+        base = self.base_url.rstrip("/")
+        return base if base.endswith("/v1") else f"{base}/v1"
+
+
+@dataclass(frozen=True)
+class Config:
+    vision: ModelConfig
+    brain: ModelConfig
+    tts: ModelConfig
+    stt: ModelConfig
+    slide_dpi: int
+    tts_voice: str
+    deck_cache_dir: str
+    hf_deck_cache_repo: str | None
+    hf_token: str | None
+    hf_deck_cache_write: bool
+
+    @property
+    def fully_mocked(self) -> bool:
+        return not self.vision.is_live and not self.brain.is_live
+
+
+def load_config() -> Config:
+    return Config(
+        vision=ModelConfig(
+            base_url=_clean(os.getenv("VISION_BASE_URL")),
+            api_key=_clean(os.getenv("VISION_API_KEY")) or "sk-no-key-required",
+            model=_clean(os.getenv("VISION_MODEL")) or "minicpm-v",
+        ),
+        brain=ModelConfig(
+            base_url=_clean(os.getenv("BRAIN_BASE_URL")),
+            api_key=_clean(os.getenv("BRAIN_API_KEY")) or "sk-no-key-required",
+            model=_clean(os.getenv("BRAIN_MODEL")) or "nemotron-3-nano",
+        ),
+        tts=ModelConfig(
+            base_url=_clean(os.getenv("TTS_BASE_URL")),
+            api_key=_clean(os.getenv("TTS_API_KEY")) or "sk-no-key-required",
+            model=_clean(os.getenv("TTS_MODEL")) or "tts-1",
+        ),
+        stt=ModelConfig(
+            base_url=_clean(os.getenv("STT_BASE_URL")),
+            api_key=_clean(os.getenv("STT_API_KEY")) or "sk-no-key-required",
+            model=_clean(os.getenv("STT_MODEL")) or "whisper-1",
+        ),
+        slide_dpi=int(_clean(os.getenv("SLIDE_DPI")) or "150"),
+        tts_voice=_clean(os.getenv("TTS_VOICE")) or "default",
+        deck_cache_dir=_clean(os.getenv("DECK_CACHE_DIR")) or ".cache/ai-prof/decks",
+        hf_deck_cache_repo=_clean(os.getenv("HF_DECK_CACHE_REPO")),
+        hf_token=_clean(os.getenv("HF_TOKEN")),
+        hf_deck_cache_write=(
+            (_clean(os.getenv("HF_DECK_CACHE_WRITE")) or "").lower()
+            in {"1", "true", "yes", "on"}
+        ),
+    )
+
+
+CONFIG = load_config()

ai_prof/deck_cache.py

@@ -0,0 +1,219 @@
+"""Content-addressed cache for rendered and vision-indexed lecture decks."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import shutil
+from dataclasses import dataclass
+from pathlib import Path
+
+from .pdf_utils import Deck, Slide
+
+SCHEMA_VERSION = 1
+
+
+@dataclass(frozen=True)
+class CachedDeck:
+    deck: Deck
+    readings: dict[int, str]
+    deck_index: str
+
+
+@dataclass(frozen=True)
+class DeckSummary:
+    key: str
+    title: str
+    slide_count: int
+
+
+def pdf_sha256(pdf_path: str) -> str:
+    digest = hashlib.sha256()
+    with open(pdf_path, "rb") as source:
+        for chunk in iter(lambda: source.read(1024 * 1024), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+class DeckCache:
+    def __init__(
+        self,
+        *,
+        root: str,
+        repo_id: str | None = None,
+        token: str | None = None,
+        write_remote: bool = False,
+    ) -> None:
+        self.root = Path(root).expanduser()
+        self.repo_id = repo_id
+        self.token = token
+        self.write_remote = write_remote
+
+    def key(self, pdf_path: str, *, dpi: int, vision_model: str) -> str:
+        identity = f"{pdf_sha256(pdf_path)}:{dpi}:{vision_model}:v{SCHEMA_VERSION}"
+        return hashlib.sha256(identity.encode("utf-8")).hexdigest()
+
+    def load(self, key: str) -> CachedDeck | None:
+        cached = self._load_local(key)
+        if cached is not None or not self.repo_id:
+            return cached
+        if self._download_remote(key):
+            return self._load_local(key)
+        return None
+
+    def list_decks(self) -> list[DeckSummary]:
+        manifests: dict[str, Path] = {}
+        if self.root.is_dir():
+            for path in self.root.glob("*/manifest.json"):
+                manifests[path.parent.name] = path
+
+        if self.repo_id:
+            try:
+                from huggingface_hub import HfApi, hf_hub_download
+
+                for filename in HfApi(token=self.token).list_repo_files(
+                    self.repo_id,
+                    repo_type="dataset",
+                ):
+                    parts = Path(filename).parts
+                    if (
+                        len(parts) == 3
+                        and parts[0] == "decks"
+                        and parts[2] == "manifest.json"
+                    ):
+                        key = parts[1]
+                        if key not in manifests:
+                            manifests[key] = Path(
+                                hf_hub_download(
+                                    repo_id=self.repo_id,
+                                    repo_type="dataset",
+                                    filename=filename,
+                                    token=self.token,
+                                )
+                            )
+            except Exception as exc:
+                print(f"[deck-cache] Hub catalog skipped: {exc}")
+
+        summaries = []
+        for key, path in manifests.items():
+            try:
+                manifest = json.loads(path.read_text(encoding="utf-8"))
+                metadata = manifest.get("metadata") or {}
+                slides = manifest.get("slides") or []
+                title = str(metadata.get("title") or f"Prepared lecture {key[:8]}")
+                summaries.append(
+                    DeckSummary(key=key, title=title, slide_count=len(slides))
+                )
+            except (OSError, TypeError, ValueError, json.JSONDecodeError):
+                continue
+        return sorted(summaries, key=lambda item: item.title.lower())
+
+    def save(
+        self,
+        key: str,
+        *,
+        deck: Deck,
+        readings: dict[int, str],
+        deck_index: str,
+        metadata: dict[str, object] | None = None,
+    ) -> None:
+        target = self.root / key
+        slides_dir = target / "slides"
+        if target.exists():
+            shutil.rmtree(target)
+        slides_dir.mkdir(parents=True, exist_ok=True)
+
+        slides = []
+        for slide in deck.slides:
+            filename = f"{slide.index:03d}.png"
+            shutil.copy2(slide.image_path, slides_dir / filename)
+            slides.append(
+                {
+                    "index": slide.index,
+                    "image": f"slides/{filename}",
+                    "text": slide.text,
+                    "reading": readings.get(slide.index, ""),
+                }
+            )
+
+        manifest = {
+            "schema_version": SCHEMA_VERSION,
+            "deck_index": deck_index,
+            "metadata": metadata or {},
+            "slides": slides,
+        }
+        (target / "manifest.json").write_text(
+            json.dumps(manifest, ensure_ascii=True, indent=2),
+            encoding="utf-8",
+        )
+
+        if self.repo_id and self.write_remote:
+            self._upload_remote(key, target)
+
+    def _load_local(self, key: str) -> CachedDeck | None:
+        target = self.root / key
+        manifest_path = target / "manifest.json"
+        if not manifest_path.is_file():
+            return None
+        try:
+            manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
+            if manifest.get("schema_version") != SCHEMA_VERSION:
+                return None
+            slides = []
+            readings = {}
+            for raw in manifest["slides"]:
+                image_path = target / raw["image"]
+                if not image_path.is_file():
+                    return None
+                index = int(raw["index"])
+                slides.append(
+                    Slide(
+                        index=index,
+                        image_path=str(image_path),
+                        text=str(raw.get("text", "")),
+                    )
+                )
+                readings[index] = str(raw.get("reading", ""))
+            return CachedDeck(
+                deck=Deck(slides=slides),
+                readings=readings,
+                deck_index=str(manifest.get("deck_index", "")),
+            )
+        except (KeyError, TypeError, ValueError, json.JSONDecodeError):
+            return None
+
+    def _download_remote(self, key: str) -> bool:
+        try:
+            from huggingface_hub import snapshot_download
+
+            snapshot_download(
+                repo_id=self.repo_id,
+                repo_type="dataset",
+                token=self.token,
+                allow_patterns=[f"decks/{key}/**"],
+                local_dir=self.root,
+            )
+            remote_dir = self.root / "decks" / key
+            local_dir = self.root / key
+            if remote_dir.is_dir():
+                if local_dir.exists():
+                    shutil.rmtree(local_dir)
+                shutil.move(str(remote_dir), str(local_dir))
+            return (local_dir / "manifest.json").is_file()
+        except Exception as exc:
+            print(f"[deck-cache] Hub download skipped: {exc}")
+            return False
+
+    def _upload_remote(self, key: str, source: Path) -> None:
+        try:
+            from huggingface_hub import HfApi
+
+            HfApi(token=self.token).upload_folder(
+                repo_id=self.repo_id,
+                repo_type="dataset",
+                folder_path=str(source),
+                path_in_repo=f"decks/{key}",
+                commit_message=f"Cache processed AI Prof deck {key[:12]}",
+            )
+        except Exception as exc:
+            print(f"[deck-cache] Hub upload skipped: {exc}")

ai_prof/pdf_utils.py

@@ -0,0 +1,56 @@
+"""Turn an uploaded lecture PDF into per-slide images + text-layer ground truth.
+
+The rendered image serves double duty: it's what the user sees AND what the
+vision model reads. The extracted text layer is exact (vision misreads small
+text), so it's kept alongside as a complement to the vision reading.
+"""
+
+from __future__ import annotations
+
+import tempfile
+from dataclasses import dataclass, field
+from pathlib import Path
+
+import fitz  # PyMuPDF
+
+
+@dataclass
+class Slide:
+    index: int  # 0-based
+    image_path: str  # rendered PNG on disk
+    text: str  # PDF text-layer extraction (may be empty for image-only decks)
+
+
+@dataclass
+class Deck:
+    slides: list[Slide] = field(default_factory=list)
+
+    def __len__(self) -> int:
+        return len(self.slides)
+
+    def outline(self) -> str:
+        """A compact slide -> first-line index, so the brain can plan/navigate."""
+        lines = []
+        for s in self.slides:
+            head = next((ln.strip() for ln in s.text.splitlines() if ln.strip()), "")
+            head = head[:80] or "(no text layer)"
+            lines.append(f"  {s.index + 1}. {head}")
+        return "\n".join(lines)
+
+
+def render_pdf(pdf_path: str, dpi: int = 150) -> Deck:
+    """Render every page to a PNG and pull its text layer."""
+    out_dir = Path(tempfile.mkdtemp(prefix="ai_prof_slides_"))
+    zoom = dpi / 72.0
+    matrix = fitz.Matrix(zoom, zoom)
+
+    deck = Deck()
+    with fitz.open(pdf_path) as doc:
+        for i, page in enumerate(doc):
+            pix = page.get_pixmap(matrix=matrix)
+            img_path = out_dir / f"slide_{i:03d}.png"
+            pix.save(img_path)
+            deck.slides.append(
+                Slide(index=i, image_path=str(img_path), text=page.get_text().strip())
+            )
+    return deck

ai_prof/rtc.py

@@ -0,0 +1,347 @@
+"""Real-time WebRTC voice handler for AI Prof.
+
+Architecture
+------------
+Student mic  →  fastrtc ReplyOnPause  →  stt_transcribe()
+                                       →  brain.answer_question()   (streamed text)
+                                       →  tts_speak()               (streamed PCM)
+                                       →  student speaker
+
+The handler is exposed as ``build_rtc_handler()`` which returns a
+``fastrtc.ReplyOnPause`` instance ready to be wired into a ``gr.WebRTC``
+component:
+
+    from ai_prof.rtc import build_rtc_handler
+    handler = build_rtc_handler(state_getter=lambda: app_state)
+    webrtc = gr.WebRTC(rtc_configuration=handler.rtc_configuration)
+    webrtc.stream(handler, inputs=[webrtc, state], outputs=[webrtc])
+
+If fastrtc is not installed the module still imports cleanly — every public
+symbol is present but raises ``RuntimeError("fastrtc not installed")`` when
+called, so app.py can do a safe conditional import.
+"""
+
+from __future__ import annotations
+
+import io
+import base64
+import threading
+import wave
+from typing import Callable, Generator
+
+import numpy as np
+
+# ---------------------------------------------------------------------------
+# Optional fastrtc import — degrade gracefully when not installed.
+# ---------------------------------------------------------------------------
+try:
+    from fastrtc import ReplyOnPause, SileroVad  # type: ignore
+    _FASTRTC_AVAILABLE = True
+except ImportError:  # pragma: no cover
+    _FASTRTC_AVAILABLE = False
+    ReplyOnPause = None  # type: ignore
+    SileroVad = None  # type: ignore
+
+from ai_prof.brain import answer_question
+from ai_prof.config import CONFIG, ModelConfig
+
+# ---------------------------------------------------------------------------
+# STT helpers
+# ---------------------------------------------------------------------------
+
+_MIN_SPEECH_RMS = 0.005
+
+
+def _has_speech(audio_pcm: np.ndarray) -> bool:
+    if audio_pcm.size == 0:
+        return False
+    pcm = audio_pcm.astype(np.float32)
+    if np.issubdtype(audio_pcm.dtype, np.integer):
+        pcm /= float(np.iinfo(audio_pcm.dtype).max)
+    return float(np.sqrt(np.mean(np.square(pcm)))) >= _MIN_SPEECH_RMS
+
+
+def _stt_transcribe_live(audio_pcm: np.ndarray, sample_rate: int = 16_000) -> str:
+    """Transcribe a NumPy PCM array via OpenAI-compatible /v1/audio/transcriptions.
+
+    Returns the transcript string, or an empty string on failure / mock mode.
+    """
+    stt_cfg: ModelConfig = CONFIG.stt
+
+    if not stt_cfg.is_live:
+        # Mock: return a placeholder so the rest of the pipeline can be tested.
+        return "[STT mock — set STT_BASE_URL to transcribe real audio]"
+    if not _has_speech(audio_pcm):
+        return ""
+
+    try:
+        import openai  # already in requirements
+
+        # Encode PCM → in-memory WAV bytes
+        buf = io.BytesIO()
+        with wave.open(buf, "wb") as wf:
+            wf.setnchannels(1)
+            wf.setsampwidth(2)  # 16-bit
+            wf.setframerate(sample_rate)
+            pcm16 = (audio_pcm * 32767).astype(np.int16)
+            wf.writeframes(pcm16.tobytes())
+        buf.seek(0)
+        buf.name = "audio.wav"  # openai SDK reads .name for MIME sniffing
+
+        client = openai.OpenAI(
+            base_url=stt_cfg.openai_base_url,
+            api_key=stt_cfg.api_key,
+        )
+        transcript = client.audio.transcriptions.create(
+            model=stt_cfg.model,
+            file=buf,
+            response_format="text",
+        )
+        return str(transcript).strip()
+    except Exception as exc:  # pragma: no cover
+        print(f"[rtc] STT error: {exc}")
+        return ""
+
+
+# ---------------------------------------------------------------------------
+# TTS helpers
+# ---------------------------------------------------------------------------
+
+# VoxCPM2 Voice Design: prepend a description in parentheses to steer the
+# synthesised voice without requiring a reference audio clip.
+_PROF_VOICE = "(Warm, articulate academic professor, clear and measured pace)"
+_voice_anchors: dict[str, str] = {}
+_voice_anchor_lock = threading.Lock()
+
+
+def reset_tts_voice(voice_key: str | None) -> None:
+    """Forget the generated reference voice for one lecture session."""
+    if not voice_key:
+        return
+    with _voice_anchor_lock:
+        _voice_anchors.pop(voice_key, None)
+
+
+def _speech_request(text: str, voice_key: str | None = None):
+    tts_cfg: ModelConfig = CONFIG.tts
+    client = __import__("openai").OpenAI(
+        base_url=tts_cfg.openai_base_url,
+        api_key=tts_cfg.api_key,
+    )
+    extra_body = {}
+    if voice_key:
+        with _voice_anchor_lock:
+            ref_audio = _voice_anchors.get(voice_key)
+        if ref_audio:
+            extra_body["ref_audio"] = ref_audio
+    request = {
+        "model": tts_cfg.model,
+        "voice": CONFIG.tts_voice,
+        "input": f"{_PROF_VOICE}{text}" if not extra_body else text,
+        "response_format": "wav",
+    }
+    if extra_body:
+        request["extra_body"] = extra_body
+    return client.audio.speech.create(
+        **request,
+    )
+
+
+def _remember_voice(voice_key: str | None, wav_bytes: bytes) -> None:
+    """Use the first utterance as VoxCPM reference audio for later beats."""
+    if not voice_key:
+        return
+    try:
+        source = io.BytesIO(wav_bytes)
+        clipped = io.BytesIO()
+        with wave.open(source, "rb") as reader:
+            params = reader.getparams()
+            frames = reader.readframes(min(reader.getnframes(), reader.getframerate() * 8))
+        with wave.open(clipped, "wb") as writer:
+            writer.setparams(params)
+            writer.writeframes(frames)
+        wav_bytes = clipped.getvalue()
+    except wave.Error:
+        pass
+    with _voice_anchor_lock:
+        if voice_key in _voice_anchors:
+            return
+        _voice_anchors[voice_key] = (
+            "data:audio/wav;base64," + base64.b64encode(wav_bytes).decode("ascii")
+        )
+
+
+def _tts_speak_stream(text: str, sample_rate: int = 48_000) -> Generator[np.ndarray, None, None]:
+    """Yield a PCM chunk (float32, mono) for *text* via /v1/audio/speech.
+
+    VoxCPM2 returns a 48 kHz WAV file; we decode it and yield one chunk.
+    Falls back to a silent chunk when TTS_BASE_URL is unset.
+    """
+    tts_cfg: ModelConfig = CONFIG.tts
+
+    if not tts_cfg.is_live:
+        yield np.zeros(sample_rate // 2, dtype=np.float32)
+        return
+
+    try:
+        response = _speech_request(text)
+        buf = io.BytesIO(response.content)
+        with wave.open(buf, "rb") as wf:
+            raw = wf.readframes(wf.getnframes())
+        pcm16 = np.frombuffer(raw, dtype=np.int16)
+        yield pcm16.astype(np.float32) / 32768.0
+    except Exception as exc:  # pragma: no cover
+        print(f"[rtc] TTS error: {exc}")
+        yield np.zeros(sample_rate // 2, dtype=np.float32)
+
+
+def tts_speak_full(
+    text: str,
+    *,
+    voice_key: str | None = None,
+) -> tuple[int, np.ndarray] | None:
+    """Call TTS and return (sample_rate, pcm_float32) for gr.Audio, or None in mock mode.
+
+    This is used by the agent loop (on_teach_deck / on_explain) to speak each
+    slide's explanation.  No fastrtc needed — pure HTTP to the Modal endpoint.
+    Returns None when TTS_BASE_URL is unset so callers can skip gracefully.
+    """
+    tts_cfg: ModelConfig = CONFIG.tts
+    if not tts_cfg.is_live:
+        return None
+    try:
+        response = _speech_request(text, voice_key)
+        wav_bytes = response.content
+        _remember_voice(voice_key, wav_bytes)
+        buf = io.BytesIO(wav_bytes)
+        with wave.open(buf, "rb") as wf:
+            sr = wf.getframerate()
+            raw = wf.readframes(wf.getnframes())
+        pcm = np.frombuffer(raw, dtype=np.int16).astype(np.float32) / 32768.0
+        return sr, pcm
+    except Exception as exc:
+        print(f"[rtc] TTS error: {exc}")
+        return None
+
+
+# ---------------------------------------------------------------------------
+# Public: build the ReplyOnPause handler
+# ---------------------------------------------------------------------------
+
+def build_rtc_handler(
+    state_getter: Callable[[], dict],
+    sample_rate: int = 16_000,
+) -> "ReplyOnPause":
+    """Return a ``fastrtc.ReplyOnPause`` handler wired to STT → brain → TTS.
+
+    Parameters
+    ----------
+    state_getter:
+        Zero-argument callable that returns the current Gradio session state
+        dict (same dict used by app.py — must contain ``deck``, ``index``,
+        ``readings`` keys).
+    sample_rate:
+        Sample rate (Hz) of the audio chunks delivered by fastrtc (default
+        16 000 Hz, which matches Silero VAD's expectation).
+
+    Usage in app.py
+    ---------------
+    ::
+
+        import gradio as gr
+        try:
+            from ai_prof.rtc import build_rtc_handler
+            _rtc_available = True
+        except RuntimeError:
+            _rtc_available = False
+
+        if _rtc_available:
+            _rtc_state_ref = {}  # populated in on_upload / on_explain
+
+            handler = build_rtc_handler(state_getter=lambda: _rtc_state_ref["state"])
+
+            with gr.Column(scale=2):
+                # ... existing chat column ...
+                webrtc = gr.WebRTC(
+                    label="Voice interjection (hold to speak)",
+                    rtc_configuration=handler.rtc_configuration,
+                    mode="send-receive",
+                )
+                webrtc.stream(
+                    handler,
+                    inputs=[webrtc, state],
+                    outputs=[webrtc],
+                    time_limit=120,
+                )
+        else:
+            # TODO: wire fastrtc when installed — see ai_prof/rtc.py
+            pass
+    """
+    if not _FASTRTC_AVAILABLE:
+        raise RuntimeError(
+            "fastrtc is not installed. "
+            "Install it with: pip install 'fastrtc[vad]'"
+        )
+
+    def _voice_reply(
+        audio: tuple[int, np.ndarray],
+        gradio_state: dict | None = None,
+    ) -> Generator[tuple[int, np.ndarray], None, None]:
+        """Called by ReplyOnPause once the student stops speaking.
+
+        Receives the buffered audio segment since the last pause, runs the
+        full STT → brain → TTS pipeline, and yields PCM chunks back to the
+        student's speaker.
+
+        Parameters
+        ----------
+        audio:
+            (sample_rate, pcm_array) tuple delivered by fastrtc.
+        gradio_state:
+            The Gradio session state dict (passed as a gr.State input).
+        """
+        in_sr, pcm = audio
+        pcm = pcm.astype(np.float32)
+        if pcm.ndim > 1:
+            pcm = pcm.mean(axis=1)  # stereo → mono
+        # Resample if fastrtc delivers a different rate than we asked for.
+        if in_sr != sample_rate and in_sr > 0:
+            factor = sample_rate / in_sr
+            new_len = int(len(pcm) * factor)
+            pcm = np.interp(
+                np.linspace(0, len(pcm) - 1, new_len),
+                np.arange(len(pcm)),
+                pcm,
+            ).astype(np.float32)
+
+        # 1. Speech-to-text
+        question = _stt_transcribe_live(pcm, sample_rate=sample_rate)
+        if not question:
+            return
+
+        # 2. Brain: stream the text answer
+        state = gradio_state or state_getter()
+        deck = state.get("deck")
+        if deck is None:
+            # No deck loaded yet — skip answering.
+            return
+        idx = state.get("index", 0)
+        reading = state.get("readings", {}).get(idx, "")
+
+        answer_chunks: list[str] = []
+        for tok in answer_question(question, reading=reading, slide_no=idx + 1, history=[]):
+            answer_chunks.append(tok)
+        answer_text = "".join(answer_chunks)
+
+        if not answer_text.strip():
+            return
+
+        # 3. TTS: stream PCM back to the caller
+        tts_sr = 48_000  # VoxCPM2 outputs 48 kHz WAV
+        for pcm_chunk in _tts_speak_stream(answer_text, sample_rate=tts_sr):
+            yield tts_sr, pcm_chunk
+
+    return ReplyOnPause(
+        _voice_reply,
+        vad_parameters={"threshold": 0.5},
+    )

ai_prof/vision.py

@@ -0,0 +1,130 @@
+"""The eyes: MiniCPM-V reads a slide image into a structured 'slide reading'.
+
+Computed once per slide and cached — interjections reuse the reading and never
+re-run the (heavy) vision pass.
+"""
+
+from __future__ import annotations
+
+import base64
+import mimetypes
+from functools import lru_cache
+
+from openai import OpenAI
+
+from .config import CONFIG
+
+_READING_PROMPT = (
+    "You are reading a single lecture slide shown as an image. "
+    "Respond in English only. "
+    "Produce a compact, structured reading another model will use to explain the slide aloud. "
+    "Use exactly these plain-text sections, omitting any that don't apply:\n"
+    "TITLE: <slide title>\n"
+    "BULLETS: <key points, one per line>\n"
+    "EQUATIONS: <any formulas, written out>\n"
+    "DIAGRAM: <what any figure/diagram/chart shows>\n"
+    "CONCEPTS: <the core concepts this slide is about>\n"
+    "Be faithful to what's actually on the slide. Do not invent content. "
+    "Do not use XML or JSON — plain text only."
+)
+
+_RETRY_PROMPT = (
+    "Read this lecture slide image and respond in English only. "
+    "Use plain text with these sections (omit any that don't apply):\n"
+    "TITLE: ...\nBULLETS: ...\nEQUATIONS: ...\nDIAGRAM: ...\nCONCEPTS: ...\n"
+    "No XML, no JSON, no repetition."
+)
+
+
+@lru_cache(maxsize=1)
+def _client() -> OpenAI:
+    return OpenAI(base_url=CONFIG.vision.openai_base_url, api_key=CONFIG.vision.api_key)
+
+
+def _data_uri(image_path: str) -> str:
+    mime = mimetypes.guess_type(image_path)[0] or "image/png"
+    with open(image_path, "rb") as f:
+        b64 = base64.b64encode(f.read()).decode("ascii")
+    return f"data:{mime};base64,{b64}"
+
+
+def _mock_reading(text: str, question: str | None) -> str:
+    head = next((ln.strip() for ln in text.splitlines() if ln.strip()), "Untitled slide")
+    body = " ".join(text.split())[:400]
+    if question:
+        return f"[mock vision] Looking closely for: {question}\nVisible text: {body or '(none)'}"
+    return (
+        f"TITLE: {head}\n"
+        f"BULLETS:\n{text or '(no extractable text)'}\n"
+        "CONCEPTS: (mock reading — set VISION_BASE_URL for a real MiniCPM-V pass)"
+    )
+
+
+def _is_degenerate(text: str) -> bool:
+    """Detect infinite-loop or language-drift outputs."""
+    if not text:
+        return True
+    lines = [ln for ln in text.splitlines() if ln.strip()]
+    if not lines:
+        return True
+    # Flag if >40% of non-empty lines are duplicates (repetition loop)
+    if len(set(lines)) / len(lines) < 0.6:
+        return True
+    # Flag if majority of characters are non-ASCII (language drift)
+    non_ascii = sum(1 for c in text if ord(c) > 127)
+    if non_ascii / max(len(text), 1) > 0.3:
+        return True
+    return False
+
+
+def _call_vision(instruction: str, image_path: str) -> str:
+    resp = _client().chat.completions.create(
+        model=CONFIG.vision.model,
+        messages=[
+            {
+                "role": "user",
+                "content": [
+                    {"type": "text", "text": instruction},
+                    {"type": "image_url", "image_url": {"url": _data_uri(image_path)}},
+                ],
+            }
+        ],
+        temperature=0.1,
+        max_tokens=512,
+    )
+    return (resp.choices[0].message.content or "").strip()
+
+
+def read_slide(
+    image_path: str,
+    text_layer: str = "",
+    question: str | None = None,
+    prior_reading: str | None = None,
+) -> str:
+    """Return a structured reading of the slide image.
+
+    ``question`` switches to a targeted 'look closer' read for a specific ask.
+    ``prior_reading`` is the reading of the previous slide; passed when slides
+    are part of an animation sequence so the model has context on what changed.
+    Falls back to a mock reading derived from the PDF text layer if no endpoint
+    is configured.
+    """
+    if not CONFIG.vision.is_live:
+        return _mock_reading(text_layer, question)
+
+    if question:
+        instruction = f"Look closely at this slide and answer in English: {question}"
+    else:
+        instruction = _READING_PROMPT
+        if prior_reading:
+            instruction = (
+                f"The previous slide reading was:\n{prior_reading}\n\n"
+                + instruction
+            )
+
+    result = _call_vision(instruction, image_path)
+    if _is_degenerate(result):
+        result = _call_vision(_RETRY_PROMPT, image_path)
+    if _is_degenerate(result):
+        return _mock_reading(text_layer, question)
+    return result

app.py

@@ -0,0 +1,1051 @@
+"""AI Prof — Gradio app.
+
+Vertical slice #1 + text interjection: upload a lecture PDF, AI Prof reads each
+slide as an image (MiniCPM-V) and explains it like a TA (Nemotron, streamed).
+Ask a question at any time; it answers using the cached slide reading, then you
+continue the walkthrough.
+"""
+
+from __future__ import annotations
+
+import io
+import html
+from pathlib import Path
+import threading
+import time
+import uuid
+import wave
+
+import numpy as np
+import gradio as gr
+from openai import OpenAI
+
+from ai_prof.agent import AgentAction, plan_teaching_beat
+from ai_prof.brain import explain_slide
+from ai_prof.config import CONFIG
+from ai_prof.deck_cache import DeckCache
+from ai_prof.pdf_utils import Deck, render_pdf
+from ai_prof.vision import read_slide
+
+# ---------------------------------------------------------------------------
+# Optional WebRTC real-time voice layer (requires: pip install "fastrtc[vad]")
+# When fastrtc is installed, replace the push-to-talk gr.Audio mic below with
+# a gr.WebRTC component and wire it through build_rtc_handler — see
+# ai_prof/rtc.py for full wiring instructions.
+# ---------------------------------------------------------------------------
+try:
+    from ai_prof.rtc import (
+        _has_speech,
+        build_rtc_handler as _build_rtc_handler,
+        reset_tts_voice,
+        tts_speak_full,
+    )
+    _RTC_AVAILABLE = True
+except Exception:
+    try:
+        from ai_prof.rtc import _has_speech, reset_tts_voice, tts_speak_full
+    except Exception:
+        tts_speak_full = lambda _text, **_kwargs: None  # type: ignore
+        reset_tts_voice = lambda _key: None  # type: ignore
+        _has_speech = lambda _audio: True  # type: ignore
+    _RTC_AVAILABLE = False
+
+# module-level pre-read cache: {session_id: {slide_idx: reading_str}}
+_preread: dict[str, dict[int, str]] = {}
+_preread_lock = threading.Lock()
+_deck_cache = DeckCache(
+    root=CONFIG.deck_cache_dir,
+    repo_id=CONFIG.hf_deck_cache_repo,
+    token=CONFIG.hf_token,
+    write_remote=CONFIG.hf_deck_cache_write,
+)
+
+
+def _prepared_deck_choices() -> list[tuple[str, str]]:
+    return [
+        (f"{item.title} ({item.slide_count} slides)", item.key)
+        for item in _deck_cache.list_decks()
+    ]
+
+
+def _new_state() -> dict:
+    return {
+        "deck": None,
+        "index": 0,
+        "readings": {},
+        "deck_index": "",
+        "whiteboard": [],
+        "session_id": str(uuid.uuid4()),
+    }
+
+
+def _ensure_reading(state: dict, idx: int) -> str:
+    """Read + cache the slide once; reused by both explanation and Q&A."""
+    sid = state["session_id"]
+    with _preread_lock:
+        if sid in _preread and idx in _preread[sid]:
+            result = _preread[sid][idx]
+            state["readings"][idx] = result
+            return result
+    cache = state["readings"]
+    if idx not in cache:
+        slide = state["deck"].slides[idx]
+        prior = cache.get(idx - 1) if idx > 0 else None
+        cache[idx] = read_slide(slide.image_path, text_layer=slide.text, prior_reading=prior)
+    return cache[idx]
+
+
+def _build_deck_index(state: dict) -> str:
+    deck: Deck | None = state["deck"]
+    if not deck:
+        return ""
+    lines = []
+    for idx, slide in enumerate(deck.slides):
+        reading = state["readings"].get(idx, "")
+        title = _reading_field(reading, "TITLE")
+        if not title:
+            title = next(
+                (line.strip() for line in slide.text.splitlines() if line.strip()),
+                f"Slide {idx + 1}",
+            )
+        concepts = _reading_field(reading, "CONCEPTS")
+        summary = concepts or " ".join(slide.text.split())[:220] or "(visual slide)"
+        lines.append(f"{idx + 1}. {title} — {summary}")
+    return "\n".join(lines)
+
+
+def _slide_view(state: dict):
+    deck: Deck | None = state["deck"]
+    if not deck:
+        return None, "No deck loaded."
+    idx = state["index"]
+    slide = deck.slides[idx]
+    return slide.image_path, f"Slide {idx + 1} / {len(deck)}"
+
+
+def _index_choices(state: dict) -> list[tuple[str, int]]:
+    deck: Deck | None = state["deck"]
+    if not deck:
+        return []
+    choices = []
+    for idx, slide in enumerate(deck.slides):
+        reading = state["readings"].get(idx, "")
+        title = _reading_field(reading, "TITLE")
+        if not title:
+            title = next(
+                (line.strip() for line in slide.text.splitlines() if line.strip()),
+                f"Slide {idx + 1}",
+            )
+        choices.append((f"{idx + 1}. {title[:90]}", idx))
+    return choices
+
+
+def _reading_field(reading: str, name: str) -> str:
+    prefix = f"{name}:"
+    for line in reading.splitlines():
+        if line.upper().startswith(prefix):
+            return line[len(prefix):].strip()
+    return ""
+
+
+def _whiteboard_view(state: dict, reading: str | None = None) -> str:
+    deck: Deck | None = state["deck"]
+    if not deck:
+        return (
+            '<div class="whiteboard-empty">'
+            "<strong>Professor's whiteboard</strong>"
+            "<span>Key ideas and worked notes will appear here.</span>"
+            "</div>"
+        )
+    board = state.get("whiteboard", [])
+    if board:
+        items = []
+        for item in board:
+            if item.get("type") == "latex":
+                expression = item.get("expression", "").replace("$$", "")
+                items.append(
+                    '<div class="whiteboard-equation">'
+                    f"\n\n$$\n{expression}\n$$\n\n"
+                    "</div>"
+                )
+            else:
+                title = html.escape(item.get("title", ""))
+                body = html.escape(item.get("body", ""))
+                items.append(
+                    '<div class="whiteboard-note">'
+                    f"<strong>{title}</strong><p>{body}</p>"
+                    "</div>"
+                )
+        return (
+            '<div class="whiteboard-sheet">'
+            '<div class="whiteboard-kicker">Professor notes</div>'
+            + "".join(items)
+            + "</div>"
+        )
+
+    idx = state["index"]
+    reading = reading or state["readings"].get(idx, "")
+    title = _reading_field(reading, "TITLE") or f"Slide {idx + 1}"
+    concepts = _reading_field(reading, "CONCEPTS")
+    if not concepts:
+        concepts = "Listening for the central idea..."
+    return (
+        '<div class="whiteboard-sheet">'
+        f'<div class="whiteboard-kicker">Working notes · {idx + 1}/{len(deck)}</div>'
+        f"<h3>{html.escape(title)}</h3>"
+        f"<p>{html.escape(concepts)}</p>"
+        '<div class="whiteboard-line"></div>'
+        '<span class="whiteboard-hint">The professor can draw here as the lecture develops.</span>'
+        "</div>"
+    )
+
+
+def _execute_actions(
+    state: dict,
+    actions: tuple[AgentAction, ...],
+    *,
+    allow_navigation: bool = True,
+) -> bool:
+    """Apply validated agent actions. Return whether navigation occurred."""
+    deck: Deck | None = state["deck"]
+    if not deck:
+        return False
+    navigated = False
+    for action in actions:
+        if action.tool in {"goto_slide", "next_slide", "prev_slide"} and not allow_navigation:
+            continue
+        if action.tool == "goto_slide":
+            state["index"] = action.args["index"] - 1
+            navigated = True
+        elif action.tool == "next_slide":
+            state["index"] = min(len(deck) - 1, state["index"] + 1)
+            navigated = True
+        elif action.tool == "prev_slide":
+            state["index"] = max(0, state["index"] - 1)
+            navigated = True
+        elif action.tool == "clear_whiteboard":
+            state["whiteboard"] = []
+        elif action.tool == "write_note":
+            state["whiteboard"].append(
+                {
+                    "type": "note",
+                    "title": action.args.get("title", ""),
+                    "body": action.args.get("body", ""),
+                }
+            )
+        elif action.tool == "write_latex":
+            state["whiteboard"].append(
+                {
+                    "type": "latex",
+                    "expression": action.args.get("expression", ""),
+                }
+            )
+        state["whiteboard"] = state["whiteboard"][-4:]
+    return navigated
+
+
+# ----------------------------------------------------------------------------- handlers
+
+
+def on_upload(pdf_file, state):
+    old_sid = state.get("session_id")
+    state = _new_state()
+    sid = state["session_id"]
+
+    if old_sid:
+        reset_tts_voice(old_sid)
+        with _preread_lock:
+            _preread.pop(old_sid, None)
+
+    if pdf_file is None:
+        img, caption = _slide_view(state)
+        yield state, img, caption, [], _whiteboard_view(state), _STATUS_IDLE, gr.update(choices=[], value=None)
+        return
+
+    cache_key = _deck_cache.key(
+        pdf_file,
+        dpi=CONFIG.slide_dpi,
+        vision_model=CONFIG.vision.model,
+    )
+    cached = _deck_cache.load(cache_key)
+    if cached is not None:
+        state["deck"] = cached.deck
+        state["readings"] = cached.readings
+        state["deck_index"] = cached.deck_index or _build_deck_index(state)
+        with _preread_lock:
+            _preread[sid] = dict(cached.readings)
+        img, caption = _slide_view(state)
+        yield (
+            state,
+            img,
+            caption,
+            [],
+            _whiteboard_view(state, state["readings"].get(0, "")),
+            _STATUS_CACHE_HIT,
+            gr.update(choices=_index_choices(state), value=0),
+        )
+        return
+
+    deck = render_pdf(pdf_file, dpi=CONFIG.slide_dpi)
+    state["deck"] = deck
+    img, caption = _slide_view(state)
+    yield (
+        state,
+        img,
+        caption,
+        [],
+        _whiteboard_view(state),
+        _status_indexing(0, len(deck)),
+        gr.update(choices=_index_choices(state), value=0),
+    )
+
+    with _preread_lock:
+        _preread[sid] = {}
+
+    for idx, slide in enumerate(deck.slides):
+        prior = state["readings"].get(idx - 1) if idx > 0 else None
+        reading = read_slide(slide.image_path, text_layer=slide.text, prior_reading=prior)
+        state["readings"][idx] = reading
+        with _preread_lock:
+            _preread[sid][idx] = reading
+        yield (
+            state,
+            img,
+            caption,
+            [],
+            _whiteboard_view(state, reading if idx == 0 else None),
+            _status_indexing(idx + 1, len(deck)),
+            gr.update(choices=_index_choices(state), value=0),
+        )
+    state["deck_index"] = _build_deck_index(state)
+    _deck_cache.save(
+        cache_key,
+        deck=deck,
+        readings=state["readings"],
+        deck_index=state["deck_index"],
+        metadata={
+            "title": Path(pdf_file).stem,
+            "dpi": CONFIG.slide_dpi,
+            "vision_model": CONFIG.vision.model,
+        },
+    )
+
+    img, caption = _slide_view(state)
+    yield (
+        state,
+        img,
+        caption,
+        [],
+        _whiteboard_view(state, state["readings"][0]),
+        _STATUS_IDLE,
+        gr.update(choices=_index_choices(state), value=0),
+    )
+
+
+def on_load_prepared(cache_key, state):
+    old_sid = state.get("session_id")
+    state = _new_state()
+    sid = state["session_id"]
+    if old_sid:
+        reset_tts_voice(old_sid)
+        with _preread_lock:
+            _preread.pop(old_sid, None)
+
+    cached = _deck_cache.load(str(cache_key or ""))
+    if cached is None:
+        gr.Warning("That prepared lecture could not be loaded.")
+        yield (
+            state,
+            *_slide_view(state),
+            [],
+            _whiteboard_view(state),
+            _STATUS_IDLE,
+            gr.update(choices=[], value=None),
+        )
+        return
+
+    state["deck"] = cached.deck
+    state["readings"] = cached.readings
+    state["deck_index"] = cached.deck_index or _build_deck_index(state)
+    with _preread_lock:
+        _preread[sid] = dict(cached.readings)
+    yield (
+        state,
+        *_slide_view(state),
+        [],
+        _whiteboard_view(state, state["readings"].get(0, "")),
+        _STATUS_CACHE_HIT,
+        gr.update(choices=_index_choices(state), value=0),
+    )
+
+
+_STATUS_READING = (
+    '<div style="background:#f0f9ff;border-left:3px solid #3b82f6;padding:6px 12px;'
+    'font-size:0.85rem;color:#1e40af;border-radius:0 4px 4px 0">📖 Reading slide…</div>'
+)
+_STATUS_EXPLAINING = (
+    '<div style="background:#f0fdf4;border-left:3px solid #22c55e;padding:6px 12px;'
+    'font-size:0.85rem;color:#166534;border-radius:0 4px 4px 0">💬 Explaining…</div>'
+)
+_STATUS_SPEAKING = (
+    '<div style="background:#fdf4ff;border-left:3px solid #a855f7;padding:6px 12px;'
+    'font-size:0.85rem;color:#6b21a8;border-radius:0 4px 4px 0">🔊 Professor speaking…</div>'
+)
+_STATUS_THINKING = (
+    '<div style="background:#fff7ed;border-left:3px solid #f97316;padding:6px 12px;'
+    'font-size:0.85rem;color:#9a3412;border-radius:0 4px 4px 0">Thinking…</div>'
+)
+_STATUS_IDLE = ""
+_STATUS_CACHE_HIT = (
+    '<div style="background:#ecfdf5;border-left:3px solid #10b981;padding:6px 12px;'
+    'font-size:0.85rem;color:#065f46;border-radius:0 4px 4px 0">'
+    "Loaded pre-indexed lecture from cache"
+    "</div>"
+)
+
+
+def _status_indexing(done: int, total: int) -> str:
+    return (
+        '<div style="background:#f5f3ff;border-left:3px solid #625ce7;padding:6px 12px;'
+        'font-size:0.85rem;color:#4c46a8;border-radius:0 4px 4px 0">'
+        f"Indexing lecture… {done} / {total} slides"
+        "</div>"
+    )
+
+
+def on_explain(state, chat):
+    deck: Deck | None = state["deck"]
+    if not deck:
+        gr.Warning("Upload a lecture PDF first.")
+        yield chat, _STATUS_IDLE, None
+        return
+    idx = state["index"]
+    yield chat, _STATUS_READING, None
+    reading = _ensure_reading(state, idx)
+    yield chat, _STATUS_EXPLAINING, None
+    chat = chat + [{"role": "assistant", "content": ""}]
+    acc = ""
+    for tok in explain_slide(
+        reading,
+        slide_no=idx + 1,
+        total=len(deck),
+        outline=deck.outline(),
+        history=chat,
+    ):
+        acc += tok
+        chat[-1]["content"] = acc
+        yield chat, _STATUS_EXPLAINING, None
+    audio = tts_speak_full(acc, voice_key=state["session_id"])
+    if audio is not None:
+        yield chat, _STATUS_SPEAKING, gr.update(value=audio, visible=True)
+        time.sleep(len(audio[1]) / audio[0])
+    yield chat, _STATUS_IDLE, gr.update(value=None, visible=False)
+
+
+def on_teach_deck(state, chat):
+    """Run professor-planned teaching beats with navigation, board tools, and TTS."""
+    deck: Deck | None = state["deck"]
+    if not deck:
+        gr.Warning("Upload a lecture PDF first.")
+        img, caption = _slide_view(state)
+        yield state, img, caption, chat, _STATUS_IDLE, _whiteboard_view(state), None
+        return
+
+    max_beats = max(1, len(deck) * 2)
+    for _ in range(max_beats):
+        idx = state["index"]
+        img, caption = _slide_view(state)
+        yield state, img, caption, chat, _STATUS_READING, _whiteboard_view(state), None
+
+        reading = _ensure_reading(state, idx)
+        beat = plan_teaching_beat(
+            trigger="continue",
+            deck_index=state["deck_index"],
+            current_slide=idx + 1,
+            total_slides=len(deck),
+            current_reading=reading,
+            whiteboard_state=state["whiteboard"],
+            history=chat,
+        )
+        _execute_actions(state, beat.actions, allow_navigation=False)
+        img, caption = _slide_view(state)
+        board = _whiteboard_view(state)
+        chat = chat + [{"role": "assistant", "content": beat.narration}]
+        yield state, img, caption, chat, _STATUS_EXPLAINING, board, None
+
+        audio = tts_speak_full(beat.narration, voice_key=state["session_id"])
+        if audio is not None:
+            sr, pcm = audio
+            yield state, img, caption, chat, _STATUS_SPEAKING, board, gr.update(value=audio, visible=True)
+            time.sleep(len(pcm) / sr)
+
+        if not beat.continue_lecture:
+            break
+        if state["index"] >= len(deck) - 1:
+            break
+        state["index"] += 1
+
+    yield state, *_slide_view(state), chat, _STATUS_IDLE, _whiteboard_view(state), gr.update(value=None, visible=False)
+
+
+def on_ask(question, state, chat):
+    deck: Deck | None = state["deck"]
+    if not deck:
+        gr.Warning("Upload a lecture PDF first.")
+        yield state, *_slide_view(state), chat, "", _whiteboard_view(state), None, ""
+        return
+    question = (question or "").strip()
+    if not question:
+        yield state, *_slide_view(state), chat, "", _whiteboard_view(state), None, ""
+        return
+    idx = state["index"]
+    reading = _ensure_reading(state, idx)
+    history = chat + [{"role": "user", "content": question}]
+    img, caption = _slide_view(state)
+    yield (
+        state,
+        img,
+        caption,
+        history,
+        _STATUS_THINKING,
+        _whiteboard_view(state),
+        None,
+        "",
+    )
+    beat = plan_teaching_beat(
+        trigger="question",
+        deck_index=state["deck_index"],
+        current_slide=idx + 1,
+        total_slides=len(deck),
+        current_reading=reading,
+        whiteboard_state=state["whiteboard"],
+        history=history,
+        question=question,
+    )
+    _execute_actions(state, beat.actions)
+    chat = history + [{"role": "assistant", "content": beat.narration}]
+    img, caption = _slide_view(state)
+    board = _whiteboard_view(state)
+    yield state, img, caption, chat, _STATUS_EXPLAINING, board, None, ""
+    audio = tts_speak_full(beat.narration, voice_key=state["session_id"])
+    if audio is not None:
+        sr, pcm = audio
+        yield state, img, caption, chat, _STATUS_SPEAKING, board, gr.update(value=audio, visible=True), ""
+        time.sleep(len(pcm) / sr)
+    yield state, img, caption, chat, _STATUS_IDLE, board, gr.update(value=None, visible=False), ""
+
+
+def on_nav(delta, state):
+    deck: Deck | None = state["deck"]
+    if deck:
+        state["index"] = max(0, min(len(deck) - 1, state["index"] + delta))
+    img, caption = _slide_view(state)
+    return state, img, caption, _whiteboard_view(state), state["index"] if deck else None
+
+
+def on_index_select(index, state):
+    deck: Deck | None = state["deck"]
+    if deck and index is not None:
+        state["index"] = max(0, min(len(deck) - 1, int(index)))
+    img, caption = _slide_view(state)
+    return state, img, caption, _whiteboard_view(state)
+
+
+def on_transcribe(audio):
+    if audio is None:
+        return ""
+    if not CONFIG.stt.is_live:
+        return "[voice input]"
+    sr, data = audio
+    if data is None or len(data) == 0:
+        return ""
+    if not _has_speech(data):
+        return ""
+    buf = io.BytesIO()
+    if data.dtype != np.int16:
+        data = (data * 32767).astype(np.int16)
+    if data.ndim > 1:
+        data = data[:, 0]
+    with wave.open(buf, "wb") as wf:
+        wf.setnchannels(1)
+        wf.setsampwidth(2)
+        wf.setframerate(sr)
+        wf.writeframes(data.tobytes())
+    buf.seek(0)
+    buf.name = "audio.wav"
+    client = OpenAI(base_url=CONFIG.stt.openai_base_url, api_key=CONFIG.stt.api_key)
+    transcript = client.audio.transcriptions.create(model=CONFIG.stt.model, file=buf)
+    return transcript.text
+
+
+# ----------------------------------------------------------------------------- UI
+
+_BANNER = (
+    "⚠️ Running in **mock mode** — set `VISION_BASE_URL` / `BRAIN_BASE_URL` (see `.env.example`) "
+    "to plug in real MiniCPM-V + Nemotron."
+    if CONFIG.fully_mocked
+    else None
+)
+
+_CSS = """
+.gradio-container {
+    max-width: 1320px !important;
+    margin: 0 auto !important;
+    padding-inline: 24px !important;
+}
+.app-title {
+    max-width: 1280px;
+    margin: 0 auto 18px;
+    padding: 4px 2px 8px;
+}
+.app-title h1 {
+    margin: 0 0 4px !important;
+    color: #24283b;
+    font-size: 2.15rem !important;
+    font-weight: 800 !important;
+    letter-spacing: -.035em;
+}
+.app-title p {
+    margin: 0 !important;
+    color: #697386;
+    font-size: 1rem;
+}
+.workspace-row {
+    width: 100%;
+    max-width: 1280px;
+    margin-inline: auto;
+    align-items: stretch !important;
+}
+.panel-card {
+    min-width: 0 !important;
+    overflow: hidden;
+    gap: 0 !important;
+    border: 1px solid #e1e5eb;
+    border-radius: 14px;
+    background: #fff;
+}
+.panel-title {
+    flex: 0 0 46px !important;
+    min-height: 46px !important;
+    height: 46px !important;
+    display: flex !important;
+    align-items: center !important;
+    margin: 0 !important;
+    padding: 0 14px !important;
+    background: #ecebff;
+    border-bottom: 1px solid #dedcff;
+    color: #5b55c7;
+}
+.panel-title p {
+    margin: 0 !important;
+    font-size: .86rem;
+    font-weight: 700;
+    line-height: 1.35;
+}
+.panel-body {
+    padding: 12px !important;
+}
+.teaching-panel { min-height: 655px; }
+.slide-frame {
+    flex: 0 0 550px !important;
+    height: 550px !important;
+    min-height: 550px !important;
+    border: 0 !important;
+    border-radius: 0 !important;
+}
+.slide-frame img {
+    height: 550px !important;
+    object-fit: contain !important;
+    background: #f8f9fb;
+}
+.slide-footer {
+    padding: 0 12px 12px !important;
+}
+.slide-caption {
+    min-height: 24px;
+    margin: 0 !important;
+    color: #667085;
+}
+.slide-index {
+    margin: 2px 0 8px !important;
+}
+.slide-controls {
+    gap: 10px !important;
+}
+.slide-controls button {
+    min-height: 42px !important;
+    border-radius: 10px !important;
+    font-weight: 700 !important;
+}
+.nav-button button {
+    color: #5b55c7 !important;
+    border: 1px solid #d8d6ff !important;
+    background: #f7f6ff !important;
+}
+.explain-button button {
+    color: #fff !important;
+    border-color: #625ce7 !important;
+    background: #625ce7 !important;
+    box-shadow: 0 5px 14px rgb(98 92 231 / 20%) !important;
+}
+.whiteboard {
+    flex: 1 1 auto !important;
+    min-height: 550px;
+    border: 0;
+    border-radius: 0;
+    overflow: hidden;
+    background:
+        linear-gradient(#e8edf3 1px, transparent 1px),
+        linear-gradient(90deg, #e8edf3 1px, transparent 1px),
+        #fbfcfe;
+    background-size: 28px 28px;
+}
+.whiteboard-empty, .whiteboard-sheet {
+    min-height: 550px;
+    padding: 28px 32px;
+    color: #172033;
+}
+.whiteboard-empty {
+    display: flex;
+    flex-direction: column;
+    align-items: center;
+    justify-content: center;
+    gap: 8px;
+    color: #667085;
+}
+.whiteboard-sheet h3 { font-size: 1.8rem; margin: 24px 0 18px; }
+.whiteboard-sheet p { font-size: 1.2rem; line-height: 1.7; max-width: 90%; }
+.whiteboard-note {
+    margin-top: 22px;
+    padding: 16px 18px;
+    border-left: 4px solid #625ce7;
+    border-radius: 0 10px 10px 0;
+    background: rgb(255 255 255 / 82%);
+}
+.whiteboard-note strong { font-size: 1.08rem; }
+.whiteboard-note p { margin: 6px 0 0; font-size: 1rem; line-height: 1.55; }
+.whiteboard-equation {
+    margin-top: 22px;
+    padding: 18px;
+    border-radius: 10px;
+    background: rgb(255 255 255 / 86%);
+    text-align: center;
+}
+.whiteboard-equation code {
+    color: #24283b;
+    font-size: 1.15rem;
+    white-space: normal;
+}
+.whiteboard-kicker {
+    color: #3157a4;
+    font-size: .78rem;
+    font-weight: 700;
+    letter-spacing: .08em;
+    text-transform: uppercase;
+}
+.whiteboard-line { width: 72px; height: 4px; margin: 30px 0 14px; background: #f4b740; }
+.whiteboard-hint { color: #7a8496; font-size: .82rem; }
+.bottom-panel {
+    min-height: 382px;
+}
+.transcript-panel {
+    flex: 1 1 auto !important;
+    height: 334px !important;
+    min-height: 334px !important;
+    border: 0 !important;
+    border-radius: 0 !important;
+}
+.transcript-panel .placeholder {
+    height: 100% !important;
+    display: flex !important;
+    align-items: center !important;
+    justify-content: center !important;
+    padding: 24px !important;
+    color: #8a94a6 !important;
+    text-align: center !important;
+}
+.question-body, .upload-body {
+    flex: 1 1 auto !important;
+    min-height: 334px;
+    padding: 18px !important;
+}
+.question-body {
+    display: flex !important;
+    flex-direction: column !important;
+    gap: 12px !important;
+}
+.question-row {
+    align-items: stretch !important;
+    gap: 10px !important;
+}
+.question-input textarea {
+    min-height: 58px !important;
+}
+.question-input {
+    flex: 1 1 auto !important;
+}
+.ask-button {
+    min-width: 96px !important;
+    max-width: 110px !important;
+}
+.ask-button button {
+    height: 100% !important;
+    min-height: 58px !important;
+    font-weight: 750 !important;
+}
+.mic-label {
+    margin: 2px 0 -4px !important;
+    color: #697386;
+    font-size: .82rem;
+    font-weight: 650;
+}
+.mic-control {
+    min-height: 108px !important;
+    max-height: 122px !important;
+    overflow: hidden !important;
+}
+.mic-control button[aria-label="Record"],
+.mic-control button.record {
+    min-width: 150px !important;
+    min-height: 58px !important;
+    border-radius: 999px !important;
+    color: #fff !important;
+    background: #625ce7 !important;
+    border-color: #625ce7 !important;
+    font-size: 1rem !important;
+    font-weight: 750 !important;
+}
+.teach-button {
+    margin-top: auto !important;
+}
+.upload-body {
+    display: flex !important;
+    flex-direction: column !important;
+    gap: 12px !important;
+}
+.upload-control {
+    min-height: 210px !important;
+}
+.upload-copy {
+    margin: 0 !important;
+    color: #667085;
+    font-size: .88rem;
+}
+@media (max-width: 900px) {
+    .gradio-container { padding-inline: 12px !important; }
+    .teaching-panel, .bottom-panel { min-width: 100% !important; }
+}
+"""
+
+with gr.Blocks(title="AI Prof", theme=gr.themes.Soft(), css=_CSS) as demo:
+    state = gr.State(_new_state())
+
+    gr.Markdown(
+        "# AI Prof\nA live, guided walkthrough of your lecture.",
+        elem_classes=["app-title"],
+    )
+    if _BANNER:
+        gr.Markdown(_BANNER)
+
+    with gr.Row(equal_height=True, elem_classes=["workspace-row"]):
+        with gr.Column(scale=1, elem_classes=["panel-card", "teaching-panel"]):
+            gr.Markdown("Lecture slides", elem_classes=["panel-title"])
+            slide_img = gr.Image(
+                show_label=False,
+                height=470,
+                elem_classes=["slide-frame"],
+            )
+            with gr.Column(elem_classes=["slide-footer"]):
+                caption = gr.Markdown("No deck loaded.", elem_classes=["slide-caption"])
+                slide_index = gr.Dropdown(
+                    label="Lecture index",
+                    choices=[],
+                    value=None,
+                    interactive=True,
+                    elem_classes=["slide-index"],
+                )
+                with gr.Row(elem_classes=["slide-controls"]):
+                    prev_btn = gr.Button("Previous", elem_classes=["nav-button"])
+                    explain_btn = gr.Button(
+                        "Explain slide",
+                        variant="primary",
+                        elem_classes=["explain-button"],
+                    )
+                    next_btn = gr.Button("Next", elem_classes=["nav-button"])
+
+        with gr.Column(scale=1, elem_classes=["panel-card", "teaching-panel"]):
+            gr.Markdown("Whiteboard", elem_classes=["panel-title"])
+            whiteboard = gr.Markdown(
+                value=_whiteboard_view(_new_state()),
+                elem_classes=["whiteboard"],
+            )
+
+    with gr.Row(equal_height=True, elem_classes=["workspace-row"]):
+        with gr.Column(scale=5, elem_classes=["panel-card", "bottom-panel"]):
+            gr.Markdown("Lecture transcript", elem_classes=["panel-title"])
+            status_strip = gr.HTML(value=_STATUS_IDLE)
+            prof_audio = gr.Audio(
+                autoplay=True,
+                show_label=False,
+                visible=False,  # hidden; plays automatically via autoplay
+                interactive=False,
+            )
+            chat = gr.Chatbot(
+                show_label=False,
+                height=320,
+                type="messages",
+                layout="panel",
+                placeholder=(
+                    "Upload a lecture to begin. The professor's explanation "
+                    "will appear here as it is spoken."
+                ),
+                elem_classes=["transcript-panel"],
+            )
+
+        with gr.Column(scale=3, elem_classes=["panel-card", "bottom-panel"]):
+            gr.Markdown("Ask a question", elem_classes=["panel-title"])
+            with gr.Column(elem_classes=["question-body"]):
+                with gr.Row(equal_height=True, elem_classes=["question-row"]):
+                    question = gr.Textbox(
+                        placeholder="Type a question...",
+                        show_label=False,
+                        lines=1,
+                        elem_classes=["question-input"],
+                        scale=5,
+                    )
+                    ask_btn = gr.Button(
+                        "Ask",
+                        variant="primary",
+                        elem_classes=["ask-button"],
+                        scale=1,
+                    )
+                gr.Markdown("Or ask out loud", elem_classes=["mic-label"])
+                mic = gr.Audio(
+                    sources=["microphone"],
+                    type="numpy",
+                    streaming=False,
+                    show_label=False,
+                    elem_classes=["mic-control"],
+                )
+                teach_btn = gr.Button(
+                    "Teach from current slide",
+                    variant="secondary",
+                    elem_classes=["teach-button"],
+                )
+            # TODO: wire fastrtc when installed — replace `mic` above with:
+            #
+            #   if _RTC_AVAILABLE:
+            #       _rtc_handler = _build_rtc_handler(state_getter=lambda: state.value)
+            #       webrtc = gr.WebRTC(
+            #           label="Live voice (real-time)",
+            #           rtc_configuration=_rtc_handler.rtc_configuration,
+            #           mode="send-receive",
+            #       )
+            #       webrtc.stream(
+            #           _rtc_handler,
+            #           inputs=[webrtc, state],
+            #           outputs=[webrtc],
+            #           time_limit=120,
+            #       )
+            #
+            # See ai_prof/rtc.py for the full pipeline:
+            #   student mic → STT (/v1/audio/transcriptions)
+            #               → brain.answer_question (streamed text)
+            #               → TTS (/v1/audio/speech, PCM chunks)
+            #               → student speaker (sub-second latency)
+
+        with gr.Column(scale=2, elem_classes=["panel-card", "bottom-panel"]):
+            gr.Markdown("Choose a lecture", elem_classes=["panel-title"])
+            with gr.Column(elem_classes=["upload-body"]):
+                prepared_deck = gr.Dropdown(
+                    label="Prepared lectures",
+                    choices=_prepared_deck_choices(),
+                    value=None,
+                    interactive=True,
+                )
+                load_prepared_btn = gr.Button(
+                    "Load prepared lecture",
+                    variant="primary",
+                )
+                gr.Markdown("Or upload your own PDF", elem_classes=["mic-label"])
+                pdf = gr.File(
+                    label="Drop a PDF to begin",
+                    file_types=[".pdf"],
+                    type="filepath",
+                    height=130,
+                    elem_classes=["upload-control"],
+                )
+                gr.Markdown(
+                    "The professor starts at slide 1 and advances automatically. "
+                    "Use the slide controls to revisit anything.",
+                    elem_classes=["upload-copy"],
+                )
+
+    lecture_outputs = [state, slide_img, caption, chat, status_strip, whiteboard, prof_audio]
+    question_outputs = [
+        state,
+        slide_img,
+        caption,
+        chat,
+        status_strip,
+        whiteboard,
+        prof_audio,
+        question,
+    ]
+    upload_event = pdf.change(
+        on_upload,
+        [pdf, state],
+        [state, slide_img, caption, chat, whiteboard, status_strip, slide_index],
+    ).then(
+        on_teach_deck,
+        [state, chat],
+        lecture_outputs,
+    )
+    prepared_event = load_prepared_btn.click(
+        on_load_prepared,
+        [prepared_deck, state],
+        [state, slide_img, caption, chat, whiteboard, status_strip, slide_index],
+    ).then(
+        on_teach_deck,
+        [state, chat],
+        lecture_outputs,
+    )
+
+    explain_event = explain_btn.click(
+        on_explain,
+        [state, chat],
+        [chat, status_strip, prof_audio],
+    )
+    teach_event = teach_btn.click(on_teach_deck, [state, chat], lecture_outputs)
+    question.submit(
+        on_ask,
+        [question, state, chat],
+        question_outputs,
+        cancels=[upload_event, prepared_event, explain_event, teach_event],
+    )
+    ask_btn.click(
+        on_ask,
+        [question, state, chat],
+        question_outputs,
+        cancels=[upload_event, prepared_event, explain_event, teach_event],
+    )
+    prev_btn.click(
+        on_nav,
+        [gr.State(-1), state],
+        [state, slide_img, caption, whiteboard, slide_index],
+    )
+    next_btn.click(
+        on_nav,
+        [gr.State(1), state],
+        [state, slide_img, caption, whiteboard, slide_index],
+    )
+    slide_index.change(
+        on_index_select,
+        [slide_index, state],
+        [state, slide_img, caption, whiteboard],
+    )
+    mic.stop_recording(on_transcribe, inputs=[mic], outputs=[question])
+
+
+if __name__ == "__main__":
+    demo.launch()

modal_app.py

@@ -0,0 +1,150 @@
+"""Modal backend for AI Prof — the brain (Nemotron 3 Nano) served over vLLM.
+
+OpenAI-compatible endpoint that the Gradio app points ``BRAIN_BASE_URL`` at. The
+GPU container scales to zero when idle.
+
+Nemotron 3 Nano is a hybrid Mamba-2 + MoE *reasoning* model. On first run vLLM
+JIT-compiles CUDA kernels — notably FlashInfer's CUTLASS fused-MoE kernel (slow,
+minutes). We persist those compile caches to Volumes and warm ONCE, so every
+later cold start reuses them (~tens of seconds) instead of recompiling on an
+expensive GPU. Cost discipline: ``max_containers=1`` so a request burst can never
+fan out into multiple GPUs, and warming is a single controlled ``modal run`` —
+never a curl storm against the live endpoint.
+
+Bring-up:
+    modal run   modal_app.py::download_model   # 1. pull weights to a Volume (CPU, cheap)
+    modal run   modal_app.py::warm             # 2. ONE GPU run: compile + cache kernels
+    modal deploy modal_app.py                  # 3. serve; cold starts now reuse the cache
+    # the printed *.modal.run URL + "/v1" is BRAIN_BASE_URL
+"""
+
+from __future__ import annotations
+
+import json
+import subprocess
+import time
+import urllib.request
+
+import modal
+
+# --- what to serve -----------------------------------------------------------
+MODEL_NAME = "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16"
+SERVED_NAME = "nemotron-3-nano"  # must match BRAIN_MODEL in the app's .env
+GPU = "H100"
+MAX_MODEL_LEN = 16384  # ample for slide reading + outline + history; small KV cache
+VLLM_PORT = 8000
+MINUTES = 60
+
+app = modal.App("ai-prof-brain")
+
+# Persistent caches: weights, vLLM torch.compile artifacts, and FlashInfer's JIT
+# kernels. Mounting the FlashInfer cache is what stops the CUTLASS MoE kernel from
+# recompiling on every cold start.
+hf_cache = modal.Volume.from_name("ai-prof-hf-cache", create_if_missing=True)
+vllm_cache = modal.Volume.from_name("ai-prof-vllm-cache", create_if_missing=True)
+flashinfer_cache = modal.Volume.from_name("ai-prof-flashinfer-cache", create_if_missing=True)
+
+triton_cache = modal.Volume.from_name("ai-prof-triton-cache", create_if_missing=True)
+
+VOLUMES = {
+    "/root/.cache/huggingface": hf_cache,
+    "/root/.cache/vllm": vllm_cache,
+    "/root/.cache/flashinfer": flashinfer_cache,
+    "/root/.triton": triton_cache,  # Mamba2 SSD Triton kernel cache (~11 min compile)
+}
+
+# CUDA *devel* base ships nvcc — Nemotron-H's Mamba-2 kernels JIT-compile at init.
+vllm_image = (
+    modal.Image.from_registry("nvidia/cuda:12.8.1-devel-ubuntu22.04", add_python="3.12")
+    .entrypoint([])
+    .pip_install("vllm>=0.12.0", "huggingface_hub[hf_transfer]>=0.27")
+    .env({"HF_HUB_ENABLE_HF_TRANSFER": "1"})
+)
+
+
+def _vllm_cmd() -> list[str]:
+    # No --enforce-eager: we WANT torch.compile + CUDA-graph capture so the
+    # artifacts get cached to the vLLM volume (FAST_BOOT=False pattern).
+    return [
+        "vllm", "serve", MODEL_NAME,
+        "--served-model-name", SERVED_NAME,
+        "--host", "0.0.0.0", "--port", str(VLLM_PORT),
+        "--max-model-len", str(MAX_MODEL_LEN),
+        "--max-num-seqs", "8",
+        "--tensor-parallel-size", "1",
+        "--trust-remote-code",
+        "--reasoning-parser", "nemotron_v3",
+    ]
+
+
+def _wait_healthy(timeout_s: int = 25 * MINUTES) -> None:
+    base = f"http://127.0.0.1:{VLLM_PORT}"
+    deadline = time.time() + timeout_s
+    while time.time() < deadline:
+        try:
+            urllib.request.urlopen(f"{base}/health", timeout=5)
+            return
+        except Exception:
+            time.sleep(5)
+    raise TimeoutError("vLLM did not become healthy in time")
+
+
+@app.function(
+    image=vllm_image,
+    volumes={"/root/.cache/huggingface": hf_cache},
+    timeout=60 * MINUTES,
+)
+def download_model(model_name: str = MODEL_NAME) -> None:
+    """Pull weights to the Volume on CPU (no GPU billed during the big download)."""
+    from huggingface_hub import snapshot_download
+
+    print(f"Downloading {model_name} -> /root/.cache/huggingface ...")
+    snapshot_download(model_name, ignore_patterns=["*.pt", "*.pth"])
+    hf_cache.commit()
+    print("Done.")
+
+
+@app.function(image=vllm_image, gpu=GPU, volumes=VOLUMES, timeout=60 * MINUTES)
+def warm() -> None:
+    """One controlled GPU run: boot vLLM, fire a single request to trigger every
+    kernel compile, then commit the compile caches. Run with ``modal run``."""
+    proc = subprocess.Popen(_vllm_cmd())
+    try:
+        print("Waiting for vLLM to compile + become healthy (first time is slow)...")
+        _wait_healthy()
+        req = urllib.request.Request(
+            f"http://127.0.0.1:{VLLM_PORT}/v1/chat/completions",
+            data=json.dumps(
+                {
+                    "model": SERVED_NAME,
+                    "messages": [{"role": "user", "content": "Say hello."}],
+                    "max_tokens": 32,
+                }
+            ).encode(),
+            headers={"Content-Type": "application/json"},
+        )
+        print("Response:", urllib.request.urlopen(req, timeout=120).read().decode()[:400])
+    finally:
+        proc.terminate()
+        try:
+            proc.wait(timeout=30)
+        except Exception:
+            proc.kill()
+    vllm_cache.commit()
+    flashinfer_cache.commit()
+    print("Warm complete — compile caches committed. Cold starts will now be fast.")
+
+
+@app.function(
+    image=vllm_image,
+    gpu=GPU,
+    volumes=VOLUMES,
+    scaledown_window=20 * MINUTES,  # keep warm between requests; cold start is ~2 min
+    timeout=60 * MINUTES,
+    max_containers=1,  # cost guard: a request burst can never fan out into many GPUs
+)
+@modal.concurrent(max_inputs=8)  # vLLM batches concurrent requests on the one replica
+@modal.web_server(port=VLLM_PORT, startup_timeout=30 * MINUTES)
+def serve() -> None:
+    print("Launching:", " ".join(_vllm_cmd()))
+    subprocess.Popen(_vllm_cmd())

modal_app_vision.py

@@ -0,0 +1,261 @@
+"""Modal backend for AI Prof's eyes: MiniCPM-V-4 served by llama.cpp.
+
+The endpoint is OpenAI-compatible and accepts the image_url content produced by
+``ai_prof/vision.py``. Point ``VISION_BASE_URL`` at the deployed URL; the app
+adds ``/v1`` when needed.
+
+The GGUF route is intentional: it gives this small, bursty vision service quick
+cold starts and keeps the project on llama.cpp for the hackathon's Llama
+Champion track. An L4 comfortably holds the Q4_K_M language model, F16 vision
+projector, and an 8K context.
+
+Bring-up:
+    modal run   modal_app_vision.py::download_model
+    modal run   modal_app_vision.py::warm
+    modal deploy modal_app_vision.py
+    # .env: VISION_BASE_URL=<serve URL>  VISION_MODEL=minicpm-v
+"""
+
+from __future__ import annotations
+
+import base64
+import binascii
+import contextlib
+import json
+import struct
+import subprocess
+import time
+import urllib.request
+import zlib
+
+import modal
+
+MODEL_REPO = "openbmb/MiniCPM-V-4-gguf"
+MODEL_REVISION = "a5a17436782fb15dff8df61ab0ec3126c3564695"
+MODEL_FILE = "ggml-model-Q4_K_M.gguf"
+MMPROJ_FILE = "mmproj-model-f16.gguf"
+SERVED_NAME = "minicpm-v"
+
+# Official llama.cpp CUDA server image b9570, pinned to its amd64 manifest so an
+# upstream image update cannot silently alter multimodal behavior during judging.
+LLAMA_CPP_IMAGE = (
+    "ghcr.io/ggml-org/llama.cpp:server-cuda"
+    "@sha256:3f167c81f5f281f642be62d1d9750b609fa38e7aa7be9b9ea2017a7f43a0d5eb"
+)
+LLAMA_SERVER = "/app/llama-server"
+MODEL_DIR = "/models/minicpm-v-4"
+
+GPU = "L4"
+LLAMA_PORT = 8081
+MAX_MODEL_LEN = 8192
+MINUTES = 60
+
+app = modal.App("ai-prof-vision")
+model_volume = modal.Volume.from_name("ai-prof-vision-models", create_if_missing=True)
+
+llama_image = (
+    modal.Image.from_registry(LLAMA_CPP_IMAGE, add_python="3.12")
+    .entrypoint([])
+    .pip_install("fastapi[standard]", "httpx", "huggingface_hub>=1.0")
+    .env({"HF_XET_HIGH_PERFORMANCE": "1"})
+)
+
+
+def _server_cmd() -> list[str]:
+    return [
+        LLAMA_SERVER,
+        "--model",
+        f"{MODEL_DIR}/{MODEL_FILE}",
+        "--mmproj",
+        f"{MODEL_DIR}/{MMPROJ_FILE}",
+        "--alias",
+        SERVED_NAME,
+        "--host",
+        "127.0.0.1",
+        "--port",
+        str(LLAMA_PORT),
+        "--ctx-size",
+        str(MAX_MODEL_LEN),
+        "--n-gpu-layers",
+        "99",
+        "--parallel",
+        "1",
+    ]
+
+
+def _wait_healthy(timeout_s: int = 10 * MINUTES) -> None:
+    deadline = time.time() + timeout_s
+    while time.time() < deadline:
+        try:
+            with urllib.request.urlopen(
+                f"http://127.0.0.1:{LLAMA_PORT}/health", timeout=5
+            ) as response:
+                if response.status == 200:
+                    return
+        except Exception:
+            time.sleep(2)
+    raise TimeoutError("llama-server did not become healthy in time")
+
+
+def _png_data_uri(width: int = 64, height: int = 64) -> str:
+    """Create a tiny valid RGB test image without adding Pillow to the image."""
+
+    def chunk(kind: bytes, data: bytes) -> bytes:
+        body = kind + data
+        return struct.pack(">I", len(data)) + body + struct.pack(">I", binascii.crc32(body))
+
+    # A white image with one blue horizontal stripe gives the vision encoder
+    # real, non-uniform pixels while keeping the warm-up payload tiny.
+    rows = []
+    for y in range(height):
+        pixel = b"\x20\x70\xd0" if height // 3 <= y < 2 * height // 3 else b"\xf8\xf8\xf8"
+        rows.append(b"\x00" + pixel * width)
+    png = (
+        b"\x89PNG\r\n\x1a\n"
+        + chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0))
+        + chunk(b"IDAT", zlib.compress(b"".join(rows)))
+        + chunk(b"IEND", b"")
+    )
+    return "data:image/png;base64," + base64.b64encode(png).decode("ascii")
+
+
+@app.function(
+    image=llama_image,
+    volumes={"/models": model_volume},
+    timeout=30 * MINUTES,
+)
+def download_model() -> None:
+    """Download only the two GGUF files required by llama-server on CPU."""
+    from huggingface_hub import hf_hub_download
+
+    for filename in (MODEL_FILE, MMPROJ_FILE):
+        print(f"Downloading {MODEL_REPO}/{filename} ...")
+        hf_hub_download(
+            repo_id=MODEL_REPO,
+            filename=filename,
+            revision=MODEL_REVISION,
+            local_dir=MODEL_DIR,
+        )
+    model_volume.commit()
+    print("MiniCPM-V model and projector downloaded.")
+
+
+@app.function(
+    image=llama_image,
+    gpu=GPU,
+    volumes={"/models": model_volume},
+    timeout=20 * MINUTES,
+)
+def warm() -> None:
+    """Smoke-test model loading and the complete multimodal request path."""
+    proc = subprocess.Popen(_server_cmd())
+    try:
+        print("Waiting for llama-server to load MiniCPM-V...")
+        _wait_healthy()
+        payload = {
+            "model": SERVED_NAME,
+            "messages": [
+                {
+                    "role": "user",
+                    "content": [
+                        {"type": "text", "text": "What color is the stripe? Answer briefly."},
+                        {"type": "image_url", "image_url": {"url": _png_data_uri()}},
+                    ],
+                }
+            ],
+            "temperature": 0,
+            "max_tokens": 16,
+        }
+        request = urllib.request.Request(
+            f"http://127.0.0.1:{LLAMA_PORT}/v1/chat/completions",
+            data=json.dumps(payload).encode(),
+            headers={"Content-Type": "application/json"},
+        )
+        response = urllib.request.urlopen(request, timeout=180).read().decode()
+        print("Multimodal warm-up response:", response[:800])
+    finally:
+        proc.terminate()
+        try:
+            proc.wait(timeout=30)
+        except subprocess.TimeoutExpired:
+            proc.kill()
+
+
+@app.function(
+    image=llama_image,
+    gpu=GPU,
+    volumes={"/models": model_volume},
+    scaledown_window=5 * MINUTES,
+    timeout=30 * MINUTES,
+    max_containers=1,
+)
+@modal.concurrent(max_inputs=4)
+@modal.asgi_app()
+def serve():
+    """Expose llama.cpp only after the model has finished loading."""
+    import httpx
+    from fastapi import FastAPI, Request
+    from fastapi.responses import StreamingResponse
+
+    # ``from __future__ import annotations`` makes the route annotation a
+    # string; FastAPI resolves it through module globals.
+    globals()["_ProxyRequest"] = Request
+
+    @contextlib.asynccontextmanager
+    async def lifespan(_app: FastAPI):
+        print("Launching:", " ".join(_server_cmd()))
+        proc = subprocess.Popen(_server_cmd())
+        try:
+            # Modal does not route traffic to the ASGI app until startup exits.
+            # This avoids llama-server's temporary 503 while weights are loading.
+            await __import__("asyncio").to_thread(_wait_healthy)
+            print("MiniCPM-V is ready.")
+            yield
+        finally:
+            proc.terminate()
+            try:
+                proc.wait(timeout=30)
+            except subprocess.TimeoutExpired:
+                proc.kill()
+
+    proxy = FastAPI(lifespan=lifespan)
+    upstream = f"http://127.0.0.1:{LLAMA_PORT}"
+
+    @proxy.api_route(
+        "/{path:path}",
+        methods=["GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "HEAD"],
+    )
+    async def forward(path: str, request: _ProxyRequest):
+        client = httpx.AsyncClient(timeout=None)
+        upstream_request = client.build_request(
+            request.method,
+            f"{upstream}/{path}",
+            params=request.query_params,
+            headers={
+                key: value
+                for key, value in request.headers.items()
+                if key.lower() not in {"host", "content-length"}
+            },
+            content=await request.body(),
+        )
+        response = await client.send(upstream_request, stream=True)
+
+        async def body():
+            try:
+                async for chunk in response.aiter_raw():
+                    yield chunk
+            finally:
+                await response.aclose()
+                await client.aclose()
+
+        return StreamingResponse(
+            body(),
+            status_code=response.status_code,
+            headers={
+                key: value
+                for key, value in response.headers.items()
+                if key.lower() not in {"content-length", "transfer-encoding", "connection"}
+            },
+        )
+
+    return proxy

modal_app_vox.py

@@ -0,0 +1,262 @@
+"""Modal backend for AI Prof — VoxCPM2 TTS + distil-Whisper STT.
+
+Two endpoints in one Modal app:
+  serve()      → /v1/audio/speech        (TTS — VoxCPM2 via vLLM-Omni)
+  transcribe() → /v1/audio/transcriptions (STT — distil-whisper-large-v3)
+
+OpenAI-compatible /v1/audio/speech endpoint; point TTS_BASE_URL at the
+printed *.modal.run URL (no /v1 suffix — the client appends it).
+
+VoxCPM2 (2B, MiniCPM-4 backbone) outputs 48 kHz WAV and supports Voice
+Design via a natural-language prefix prepended to the synthesis input — used
+in ai_prof/rtc.py to give AI Prof a consistent professor voice.
+
+vLLM-Omni layers omni-modal (TTS/STT) support on top of vLLM's scheduler
+and exposes a drop-in /v1/audio/speech endpoint with PagedAttention and
+continuous batching.
+
+Bring-up:
+    modal run   modal_app_vox.py::download_model   # 1. pull VoxCPM2 weights (CPU, cheap)
+    modal run   modal_app_vox.py::download_stt     # 2. pull Whisper weights  (CPU, cheap)
+    modal run   modal_app_vox.py::warm             # 3. ONE GPU run: compile CUDA kernels
+    modal deploy modal_app_vox.py                  # 4. serve both endpoints
+    # .env: TTS_BASE_URL=<serve URL>   TTS_MODEL=voxcpm2
+    #       STT_BASE_URL=<transcribe URL>  STT_MODEL=distil-whisper-large-v3
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import subprocess
+import time
+import urllib.request
+
+import modal
+
+# --- TTS: VoxCPM2 ------------------------------------------------------------
+MODEL_NAME  = "openbmb/VoxCPM2"
+SERVED_NAME = "voxcpm2"   # must match TTS_MODEL in the app's .env
+GPU         = "A10G"      # 24 GB VRAM; VoxCPM2 needs ~8 GB
+MAX_MODEL_LEN = 2048      # generous for any TTS input chunk
+VLLM_PORT   = 8000
+MINUTES     = 60
+
+# --- STT: distil-Whisper -----------------------------------------------------
+# Use the full CTranslate2 HF path; faster-whisper loads these natively.
+STT_MODEL = "Systran/faster-distil-whisper-large-v3"
+STT_PORT  = 8001
+
+# Minimal OpenAI-compatible /v1/audio/transcriptions endpoint.
+# Written to /tmp/stt_app.py inside the container at startup.
+_STT_SERVER = """\
+import io, os, tempfile
+from fastapi import FastAPI, File, Form, UploadFile
+from faster_whisper import WhisperModel
+
+_MODEL_ID = os.environ.get("WHISPER_MODEL", "Systran/faster-distil-whisper-large-v3")
+_model = WhisperModel(_MODEL_ID, device="cuda", compute_type="float16")
+
+app = FastAPI()
+
+@app.post("/v1/audio/transcriptions")
+async def transcribe(
+    file: UploadFile = File(...),
+    model: str = Form(default=_MODEL_ID),
+    language: str = Form(default=None),
+    response_format: str = Form(default="json"),
+):
+    data = await file.read()
+    with tempfile.NamedTemporaryFile(suffix=os.path.splitext(file.filename or ".wav")[1] or ".wav", delete=False) as tmp:
+        tmp.write(data)
+        tmp_path = tmp.name
+    try:
+        segments, _ = _model.transcribe(tmp_path, language=language or None)
+        text = " ".join(s.text for s in segments).strip()
+    finally:
+        os.unlink(tmp_path)
+    return {"text": text}
+"""
+
+app = modal.App("ai-prof-vox")
+
+# Persistent caches — same pattern as modal_app.py (brain).
+# FlashInfer cache is kept even though VoxCPM2 is transformer-only:
+# vLLM itself may emit FlashInfer kernels depending on the backend.
+hf_cache         = modal.Volume.from_name("ai-prof-vox-hf-cache",      create_if_missing=True)
+vllm_cache       = modal.Volume.from_name("ai-prof-vox-vllm-cache",    create_if_missing=True)
+flashinfer_cache = modal.Volume.from_name("ai-prof-vox-fi-cache",      create_if_missing=True)
+triton_cache     = modal.Volume.from_name("ai-prof-vox-triton-cache",  create_if_missing=True)
+
+VOLUMES = {
+    "/root/.cache/huggingface": hf_cache,
+    "/root/.cache/vllm":        vllm_cache,
+    "/root/.cache/flashinfer":  flashinfer_cache,
+    "/root/.triton":            triton_cache,
+}
+
+# vLLM-Omni installation recipe from the official README:
+#   1. pip install vllm==0.19.0
+#   2. git clone vllm-omni && pip install -e .
+# We use uv for faster resolution and to avoid pip's slower solver on large graphs.
+vox_image = (
+    modal.Image.from_registry("nvidia/cuda:12.8.1-devel-ubuntu22.04", add_python="3.12")
+    .entrypoint([])
+    .apt_install("git")
+    .run_commands(
+        "pip install uv 'huggingface_hub[hf_transfer]>=0.27'",
+        "uv pip install --system 'vllm==0.22.1'",
+        "pip install hf_transfer",  # vllm's uv install drops hf_transfer; reinstall after
+        "git clone --depth 1 https://github.com/vllm-project/vllm-omni.git /opt/vllm-omni",
+        "cd /opt/vllm-omni && uv pip install --system -e .",
+        "pip install 'voxcpm>=2.0'",  # vllm-omni requires this for VoxCPM2; it's a separate PyPI package
+    )
+    .env({"HF_HUB_ENABLE_HF_TRANSFER": "1"})
+)
+
+
+def _vllm_omni_cmd() -> list[str]:
+    # Use `vllm-omni` CLI (installed by pip install -e vllm-omni).
+    # With vllm 0.22.1 the plugin also patches `vllm serve` to accept --omni,
+    # but the explicit CLI is the documented/safe path.
+    return [
+        "vllm-omni", "serve", MODEL_NAME,
+        "--omni",
+        "--served-model-name", SERVED_NAME,
+        "--host", "0.0.0.0", "--port", str(VLLM_PORT),
+        "--max-model-len", str(MAX_MODEL_LEN),
+        "--max-num-seqs", "4",
+        "--tensor-parallel-size", "1",
+        "--trust-remote-code",
+    ]
+
+
+def _wait_healthy(timeout_s: int = 20 * MINUTES) -> None:
+    base = f"http://127.0.0.1:{VLLM_PORT}"
+    deadline = time.time() + timeout_s
+    while time.time() < deadline:
+        try:
+            urllib.request.urlopen(f"{base}/health", timeout=5)
+            return
+        except Exception:
+            time.sleep(5)
+    raise TimeoutError("vLLM-Omni did not become healthy in time")
+
+
+@app.function(
+    image=vox_image,
+    volumes={"/root/.cache/huggingface": hf_cache},
+    timeout=60 * MINUTES,
+)
+def download_model(model_name: str = MODEL_NAME) -> None:
+    """Pull VoxCPM2 weights to the Volume on CPU (no GPU billed)."""
+    from huggingface_hub import snapshot_download
+
+    print(f"Downloading {model_name} ...")
+    snapshot_download(model_name, ignore_patterns=["*.pt", "*.pth"])
+    hf_cache.commit()
+    print("Done.")
+
+
+@app.function(image=vox_image, gpu=GPU, volumes=VOLUMES, timeout=60 * MINUTES)
+def warm() -> None:
+    """One controlled GPU run: boot vLLM-Omni, fire a warm-up TTS request to
+    trigger every kernel compile, then commit the caches."""
+    proc = subprocess.Popen(_vllm_omni_cmd())
+    try:
+        print("Waiting for vLLM-Omni to compile + become healthy (first run is slow)...")
+        _wait_healthy()
+        req_data = json.dumps({
+            "model": SERVED_NAME,
+            "input": "Warm-up synthesis complete.",
+            "voice": "default",
+            "response_format": "wav",
+        }).encode()
+        req = urllib.request.Request(
+            f"http://127.0.0.1:{VLLM_PORT}/v1/audio/speech",
+            data=req_data,
+            headers={"Content-Type": "application/json"},
+        )
+        wav_bytes = urllib.request.urlopen(req, timeout=120).read()
+        print(f"Warm-up TTS response: {len(wav_bytes):,} bytes of WAV audio.")
+    finally:
+        proc.terminate()
+        try:
+            proc.wait(timeout=30)
+        except Exception:
+            proc.kill()
+    vllm_cache.commit()
+    flashinfer_cache.commit()
+    triton_cache.commit()
+    print("Warm complete — caches committed.")
+
+
+@app.function(
+    image=vox_image,
+    gpu=GPU,
+    volumes=VOLUMES,
+    scaledown_window=10 * MINUTES,
+    timeout=60 * MINUTES,
+    max_containers=1,
+)
+@modal.concurrent(max_inputs=4)
+@modal.web_server(port=VLLM_PORT, startup_timeout=25 * MINUTES)
+def serve() -> None:
+    print("Launching:", " ".join(_vllm_omni_cmd()))
+    subprocess.Popen(_vllm_omni_cmd())
+
+
+# =============================================================================
+# STT — distil-whisper-large-v3 via faster-whisper-server
+# Exposes OpenAI-compatible POST /v1/audio/transcriptions.
+# Point STT_BASE_URL at the printed *.modal.run URL; set STT_MODEL=distil-whisper-large-v3.
+# =============================================================================
+
+stt_image = (
+    modal.Image.from_registry(
+        "nvidia/cuda:12.1.0-cudnn8-runtime-ubuntu22.04", add_python="3.12"
+    )
+    .entrypoint([])
+    .pip_install(
+        "faster-whisper",
+        "fastapi",
+        "uvicorn[standard]",
+        "python-multipart",
+        "hf_transfer",
+        "huggingface_hub",
+    )
+    .env({"HF_HUB_ENABLE_HF_TRANSFER": "1"})
+)
+
+
+@app.function(
+    image=stt_image,
+    volumes={"/root/.cache/huggingface": hf_cache},
+    timeout=30 * MINUTES,
+)
+def download_stt(model_name: str = STT_MODEL) -> None:
+    """Pull distil-Whisper weights to the shared HF cache Volume (CPU, no GPU billed)."""
+    from huggingface_hub import snapshot_download
+
+    print(f"Downloading {model_name} ...")
+    snapshot_download(model_name)
+    hf_cache.commit()
+    print("Done.")
+
+
+@app.function(
+    image=stt_image,
+    gpu="T4",
+    volumes={"/root/.cache/huggingface": hf_cache},
+    scaledown_window=5 * MINUTES,
+    timeout=30 * MINUTES,
+    max_containers=1,
+)
+@modal.concurrent(max_inputs=4)
+@modal.web_server(port=STT_PORT, startup_timeout=5 * MINUTES)
+def transcribe() -> None:
+    with open("/tmp/stt_app.py", "w") as f:
+        f.write(_STT_SERVER)
+    cmd = ["uvicorn", "stt_app:app", "--host", "0.0.0.0", "--port", str(STT_PORT)]
+    print("Launching STT server (model:", STT_MODEL, ")")
+    subprocess.Popen(cmd, cwd="/tmp", env={**os.environ, "WHISPER_MODEL": STT_MODEL})

requirements.txt

@@ -0,0 +1,7 @@
+gradio>=4.44
+fastrtc>=0.0.18        # WebRTC real-time voice layer; install with: pip install "fastrtc[vad]"
+pymupdf>=1.24
+openai>=1.40
+pillow>=10.0
+python-dotenv>=1.0
+huggingface_hub>=0.27

tests/test_agent.py

@@ -0,0 +1,48 @@
+import unittest
+
+from ai_prof.agent import _extract_json, _validate_actions
+
+
+class AgentValidationTests(unittest.TestCase):
+    def test_extract_json_accepts_fenced_output(self):
+        self.assertEqual(
+            _extract_json('```json\n{"narration": "Hello"}\n```'),
+            {"narration": "Hello"},
+        )
+
+    def test_validate_actions_bounds_navigation_and_whiteboard(self):
+        actions = _validate_actions(
+            [
+                {"tool": "goto_slide", "args": {"index": 3}},
+                {"tool": "next_slide", "args": {}},
+                {"tool": "goto_slide", "args": {"index": 99}},
+                {
+                    "tool": "write_note",
+                    "args": {"title": "Kernel", "body": "Weights"},
+                },
+                {"tool": "write_latex", "args": {"expression": "a+b"}},
+                {"tool": "clear_whiteboard", "args": {}},
+                {"tool": "unknown", "args": {}},
+            ],
+            total_slides=8,
+        )
+
+        self.assertEqual(
+            [action.tool for action in actions],
+            ["goto_slide", "write_note", "write_latex"],
+        )
+        self.assertEqual(actions[0].args, {"index": 3})
+
+    def test_validate_actions_rejects_empty_whiteboard_content(self):
+        actions = _validate_actions(
+            [
+                {"tool": "write_note", "args": {"title": "", "body": ""}},
+                {"tool": "write_latex", "args": {"expression": ""}},
+            ],
+            total_slides=2,
+        )
+        self.assertEqual(actions, ())
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_deck_cache.py

@@ -0,0 +1,58 @@
+import tempfile
+import unittest
+from pathlib import Path
+
+from ai_prof.deck_cache import DeckCache
+from ai_prof.pdf_utils import Deck, Slide
+
+
+class DeckCacheTests(unittest.TestCase):
+    def test_save_and_load_round_trip(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            source = Path(tmp) / "source.png"
+            source.write_bytes(b"png")
+            cache = DeckCache(root=str(Path(tmp) / "cache"))
+            deck = Deck(
+                slides=[
+                    Slide(index=0, image_path=str(source), text="First slide"),
+                ]
+            )
+
+            cache.save(
+                "deck-key",
+                deck=deck,
+                readings={0: "TITLE: First\nCONCEPTS: caching"},
+                deck_index="1. First - caching",
+                metadata={"title": "Caching 101", "dpi": 150},
+            )
+            loaded = cache.load("deck-key")
+
+            self.assertIsNotNone(loaded)
+            self.assertEqual(loaded.deck.slides[0].text, "First slide")
+            self.assertEqual(
+                loaded.readings[0],
+                "TITLE: First\nCONCEPTS: caching",
+            )
+            self.assertEqual(loaded.deck_index, "1. First - caching")
+            self.assertTrue(Path(loaded.deck.slides[0].image_path).is_file())
+            self.assertEqual(
+                cache.list_decks()[0].title,
+                "Caching 101",
+            )
+
+    def test_key_changes_with_processing_settings(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            pdf = Path(tmp) / "lecture.pdf"
+            pdf.write_bytes(b"same lecture")
+            cache = DeckCache(root=str(Path(tmp) / "cache"))
+
+            first = cache.key(str(pdf), dpi=150, vision_model="minicpm-v")
+            second = cache.key(str(pdf), dpi=200, vision_model="minicpm-v")
+            third = cache.key(str(pdf), dpi=150, vision_model="other-model")
+
+            self.assertNotEqual(first, second)
+            self.assertNotEqual(first, third)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_orchestrator.py

@@ -0,0 +1,110 @@
+import unittest
+from unittest.mock import patch
+
+from ai_prof.agent import AgentAction, TeachingBeat
+from ai_prof.pdf_utils import Deck, Slide
+import app
+
+
+def _state() -> dict:
+    state = app._new_state()
+    state["deck"] = Deck(
+        slides=[
+            Slide(index=0, image_path="/tmp/slide-1.png", text="One"),
+            Slide(index=1, image_path="/tmp/slide-2.png", text="Two"),
+            Slide(index=2, image_path="/tmp/slide-3.png", text="Three"),
+        ]
+    )
+    state["readings"] = {
+        0: "TITLE: One\nCONCEPTS: first",
+        1: "TITLE: Two\nCONCEPTS: second",
+        2: "TITLE: Three\nCONCEPTS: third",
+    }
+    state["deck_index"] = "1. One\n2. Two\n3. Three"
+    return state
+
+
+class OrchestratorTests(unittest.TestCase):
+    def test_execute_actions_navigates_and_updates_whiteboard(self):
+        state = _state()
+        navigated = app._execute_actions(
+            state,
+            (
+                AgentAction("goto_slide", {"index": 3}),
+                AgentAction(
+                    "write_note",
+                    {"title": "Remember", "body": "Use the third slide."},
+                ),
+            ),
+        )
+
+        self.assertTrue(navigated)
+        self.assertEqual(state["index"], 2)
+        self.assertEqual(state["whiteboard"][0]["title"], "Remember")
+
+    @patch.object(app, "tts_speak_full", return_value=None)
+    @patch.object(app, "plan_teaching_beat")
+    def test_question_can_move_to_supporting_slide(self, plan, _tts):
+        plan.return_value = TeachingBeat(
+            narration="Let’s revisit the definition.",
+            actions=(AgentAction("goto_slide", {"index": 2}),),
+            continue_lecture=False,
+        )
+        outputs = list(app.on_ask("What was the definition?", _state(), []))
+        final = outputs[-1]
+
+        self.assertEqual(final[0]["index"], 1)
+        self.assertEqual(final[2], "Slide 2 / 3")
+        self.assertEqual(final[3][-1]["content"], "Let’s revisit the definition.")
+
+    @patch.object(app, "tts_speak_full", return_value=None)
+    @patch.object(app, "plan_teaching_beat")
+    def test_lecture_advances_when_agent_does_not_navigate(self, plan, _tts):
+        plan.side_effect = [
+            TeachingBeat("First beat.", continue_lecture=True),
+            TeachingBeat("Second beat.", continue_lecture=False),
+        ]
+        outputs = list(app.on_teach_deck(_state(), []))
+        final = outputs[-1]
+
+        self.assertEqual(final[0]["index"], 1)
+        self.assertEqual(
+            [message["content"] for message in final[3]],
+            ["First beat.", "Second beat."],
+        )
+
+    @patch.object(app, "tts_speak_full", return_value=None)
+    @patch.object(app, "plan_teaching_beat")
+    def test_lecture_ignores_agent_navigation_and_stays_sequential(self, plan, _tts):
+        plan.side_effect = [
+            TeachingBeat(
+                "First beat.",
+                actions=(AgentAction("goto_slide", {"index": 3}),),
+                continue_lecture=True,
+            ),
+            TeachingBeat("Second beat.", continue_lecture=False),
+        ]
+
+        outputs = list(app.on_teach_deck(_state(), []))
+        final = outputs[-1]
+
+        self.assertEqual(final[0]["index"], 1)
+        self.assertEqual(final[2], "Slide 2 / 3")
+
+    def test_index_select_moves_to_requested_slide(self):
+        state, _img, caption, _board = app.on_index_select(2, _state())
+
+        self.assertEqual(state["index"], 2)
+        self.assertEqual(caption, "Slide 3 / 3")
+
+    def test_whiteboard_emits_math_block_for_latex(self):
+        state = _state()
+        state["whiteboard"] = [{"type": "latex", "expression": r"x^2 + y^2"}]
+
+        board = app._whiteboard_view(state)
+
+        self.assertIn("$$\nx^2 + y^2\n$$", board)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_voice.py

@@ -0,0 +1,40 @@
+import unittest
+from unittest.mock import MagicMock, patch
+
+import numpy as np
+
+from ai_prof import rtc
+from ai_prof.rtc import _has_speech
+
+
+class VoiceGateTests(unittest.TestCase):
+    def test_silence_is_rejected(self):
+        self.assertFalse(_has_speech(np.zeros(16_000, dtype=np.float32)))
+
+    def test_audible_signal_is_accepted(self):
+        t = np.arange(16_000, dtype=np.float32) / 16_000
+        signal = 0.05 * np.sin(2 * np.pi * 220 * t)
+        self.assertTrue(_has_speech(signal))
+
+    def test_int16_audio_is_normalized(self):
+        signal = np.full(2_000, 1_000, dtype=np.int16)
+        self.assertTrue(_has_speech(signal))
+
+    @patch.object(rtc, "_voice_anchors", {"lecture": "data:audio/wav;base64,abc"})
+    @patch("openai.OpenAI")
+    def test_tts_reuses_session_voice_reference(self, openai_client):
+        speech = MagicMock()
+        openai_client.return_value.audio.speech = speech
+
+        rtc._speech_request("Next slide.", voice_key="lecture")
+
+        kwargs = speech.create.call_args.kwargs
+        self.assertEqual(
+            kwargs["extra_body"]["ref_audio"],
+            "data:audio/wav;base64,abc",
+        )
+        self.assertEqual(kwargs["input"], "Next slide.")
+
+
+if __name__ == "__main__":
+    unittest.main()