Upload folder using huggingface_hub

.gitattributes

@@ -33,39 +33,3 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
-app/deploy/book1_data/book1/forster1/B1_P_012.jpg filter=lfs diff=lfs merge=lfs -text
-app/deploy/book1_data/book1/forster1/B1_P_014.jpg filter=lfs diff=lfs merge=lfs -text
-app/deploy/book1_data/book1/forster1/B1_P_015.jpg filter=lfs diff=lfs merge=lfs -text
-app/deploy/book1_data/book1/forster1/B1_P_016.jpg filter=lfs diff=lfs merge=lfs -text
-app/deploy/book1_data/book1/forster1/B1_P_017.jpg filter=lfs diff=lfs merge=lfs -text
-app/deploy/book1_data/book1/forster1/B1_P_020.jpg filter=lfs diff=lfs merge=lfs -text
-app/deploy/book1_data/book1/forster1/B1_P_021.jpg filter=lfs diff=lfs merge=lfs -text
-app/deploy/book1_data/book1/forster1/B1_P_024.jpg filter=lfs diff=lfs merge=lfs -text
-app/deploy/book1_data/book1/forster1/B1_P_025.jpg filter=lfs diff=lfs merge=lfs -text
-app/deploy/book1_data/book1/forster1/B1_P_028.jpg filter=lfs diff=lfs merge=lfs -text
-app/deploy/book1_data/book1/forster1/B1_P_029.jpg filter=lfs diff=lfs merge=lfs -text
-app/deploy/book1_data/book1/forster1/B1_P_030.jpg filter=lfs diff=lfs merge=lfs -text
-app/deploy/book1_data/book1/forster1/B1_P_031.jpg filter=lfs diff=lfs merge=lfs -text
-app/deploy/book1_data/book1/forster1/B1_P_034.jpg filter=lfs diff=lfs merge=lfs -text
-app/deploy/book1_data/book1/forster1/B1_P_035.jpg filter=lfs diff=lfs merge=lfs -text
-app/deploy/book1_data/book1/forster1/B1_P_038.jpg filter=lfs diff=lfs merge=lfs -text
-app/deploy/book1_data/book1/forster1/B1_P_039.jpg filter=lfs diff=lfs merge=lfs -text
-app/deploy/book1_data/book1/forster1/B1_P_042.jpg filter=lfs diff=lfs merge=lfs -text
-app/deploy/book1_data/book1/forster1/B1_P_043.jpg filter=lfs diff=lfs merge=lfs -text
-app/deploy/book1_data/book1/forster1/B1_P_046.jpg filter=lfs diff=lfs merge=lfs -text
-app/deploy/book1_data/book1/forster1/B1_P_047.jpg filter=lfs diff=lfs merge=lfs -text
-app/deploy/book1_data/book1/forster1/B1_P_050.jpg filter=lfs diff=lfs merge=lfs -text
-app/deploy/book1_data/book1/forster1/B1_P_051.jpg filter=lfs diff=lfs merge=lfs -text
-app/deploy/book1_data/book1/forster1/B1_P_052.jpg filter=lfs diff=lfs merge=lfs -text
-app/deploy/book1_data/book1/forster1/B1_P_053.jpg filter=lfs diff=lfs merge=lfs -text
-app/deploy/book1_data/book1/forster1/B1_P_056.jpg filter=lfs diff=lfs merge=lfs -text
-app/deploy/book1_data/book1/forster1/B1_P_057.jpg filter=lfs diff=lfs merge=lfs -text
-app/deploy/book1_data/book1/forster1/B1_P_060.jpg filter=lfs diff=lfs merge=lfs -text
-app/deploy/book1_data/book1/forster1/B1_P_061.jpg filter=lfs diff=lfs merge=lfs -text
-app/deploy/book1_data/book1/forster1/B1_P_064.jpg filter=lfs diff=lfs merge=lfs -text
-app/deploy/book1_data/book1/forster1/B1_P_065.jpg filter=lfs diff=lfs merge=lfs -text
-app/deploy/book1_data/book1/forster1/B1_P_068.jpg filter=lfs diff=lfs merge=lfs -text
-app/deploy/book1_data/book1/forster1/B1_P_069.jpg filter=lfs diff=lfs merge=lfs -text
-app/deploy/book1_data/book1/forster1/B1_P_072.jpg filter=lfs diff=lfs merge=lfs -text
-app/deploy/book1_data/book1/forster1/B1_P_073.jpg filter=lfs diff=lfs merge=lfs -text
-app/deploy/book1_data/book1/forster1/B1_P_074.jpg filter=lfs diff=lfs merge=lfs -text

Dockerfile

@@ -36,15 +36,19 @@ ENV INKFERENCE_DATA_ROOT=/data \
     TROCR_MODEL_ID=microsoft/trocr-base-handwritten \
     HTR_MAX_LONG_EDGE=1600 \
     HTR_NUM_BEAMS=1 \
+    INKFERENCE_LOG_LEVEL=INFO \
     # --- post-correction via Groq (GROQ_API_KEY is a Space secret) ---
     CORRECTION_ENABLED=true \
     CORRECTION_BACKEND=api \
     CORRECTION_API_BASE=https://api.groq.com/openai/v1 \
     CORRECTION_API_MODEL=qwen/qwen3-32b \
-    # --- Ask-the-Archive via Gemini (GEMINI_API_KEY is a Space secret) ---
-    LLM_PROVIDER=gemini \
-    LLM_MODEL=gemini-2.5-flash
-# Set as Space secrets (Settings -> Variables and secrets): GEMINI_API_KEY, GROQ_API_KEY.
+    # --- Ask-the-Archive: Groq gpt-oss-120b -> Gemini fallback -> extractive ---
+    LLM_PROVIDER=groq \
+    LLM_MODEL=openai/gpt-oss-120b \
+    LLM_FALLBACK=gemini:gemini-2.5-flash-lite
+# Set as Space secrets (Settings -> Variables and secrets):
+#   GROQ_API_KEY   -> post-correction AND primary RAG answers
+#   GEMINI_API_KEY -> RAG fallback when Groq is rate-limited/unavailable
 # To use your fine-tuned recognizer, push it to the Hub and set TROCR_MODEL_ID=<user>/inkference-trocr.
 
 USER user

README.md

@@ -12,8 +12,9 @@ pinned: false
 
 Reader · Ask the Archive · Upload. One FastAPI container serves the Inkference UI +
 API: the 36-page **Book 1** corpus is baked in and seeded on boot, and **live upload**
-runs the full pipeline (Kraken → TrOCR → confidence → Groq correction). "Ask the
-Archive" answers over the corrected text with Gemini, citing source pages.
+runs the full pipeline (Kraken → TrOCR → confidence → Qwen correction). "Ask the
+Archive" answers over the corrected text (with page citations), and can also answer
+**in character as the author** (Forster).
 
 ## What the Space runs
 - **Frontend + API**: same URL (the app serves `frontend/` at `/`).
@@ -22,34 +23,53 @@ Archive" answers over the corrected text with Gemini, citing source pages.
 - **Live upload**: Kraken segmentation + TrOCR recognition + per-word confidence +
   Qwen post-correction (Groq). Works on the free 16 GB Space, but CPU-slow
   (~minutes/page) and uploaded pages are lost on restart (ephemeral `/data`).
-- **Ask the Archive**: MiniLM + FAISS retrieval → Gemini answer + page citations,
-  with an extractive fallback if the LLM is unavailable.
+- **Ask the Archive**: MiniLM + FAISS retrieval → LLM answer + page citations.
+  Answer generation uses a **fallback chain**: primary **Groq `openai/gpt-oss-120b`**
+  → **Gemini `gemini-2.5-flash-lite`** (when Groq is rate-limited/unavailable) →
+  **extractive** passage (always works, cited, $0). The **"Answer as Author"** button
+  answers in first person as Forster with an *IN CHARACTER* tag.
 
 ## Deploy steps
 
 1. **Log in to Hugging Face**: `hf auth login` (token from
-   https://huggingface.co/settings/tokens), or use the web UI.
+   https://huggingface.co/settings/tokens); confirm with `hf auth whoami`.
 
 2. *(Recommended)* **Push the fine-tuned recognizer to the Hub** so uploads get good OCR
    (otherwise the base model is used):
    ```bash
-   hf upload <user>/inkference-trocr models/trocr_best_from_bentham
+   hf upload <user>/inkference-trocr models/trocr_best_from_bentham --repo-type model
    ```
    Then set the Space variable `TROCR_MODEL_ID=<user>/inkference-trocr`.
 
-3. **Create a Space** → SDK **Docker** (free CPU, 16 GB).
+3. **Create a Space** (Docker SDK, free CPU):
+   ```bash
+   hf repo create inkference --repo-type space --space-sdk docker
+   ```
 
-4. **Populate the Space repo** with ONLY what the image needs (do NOT push `data/`,
-   `models/`, or `notebooks/` — they're huge). From a clean checkout:
+4. **Populate the Space repo** with ONLY what the image needs (never `data/`, `models/`,
+   or `notebooks/`). Clone the Space and copy the required files in:
+   ```bash
+   REPO=$(pwd)                        # this project's root
+   git clone https://huggingface.co/spaces/<user>/inkference ~/hf-inkference
+   cd ~/hf-inkference
+   cp "$REPO/app/deploy/Dockerfile" Dockerfile     # HF builds the ROOT Dockerfile
+   cp "$REPO/app/deploy/README.md"  README.md      # HF frontmatter (sdk: docker, app_port)
+   mkdir -p app/deploy
+   cp    "$REPO/app/pyproject.toml"                app/
+   cp -r "$REPO/app/src"                           app/
+   cp -r "$REPO/app/frontend"                      app/
+   cp    "$REPO/app/deploy/requirements-space.txt" app/deploy/
+   cp -r "$REPO/app/deploy/book1_data"             app/deploy/
+   ```
+   Push with **`hf upload`** (uses your login token — avoids the git-credential prompt
+   that makes `git push` hang):
    ```bash
-   cp app/deploy/Dockerfile Dockerfile          # HF builds the ROOT Dockerfile
-   # keep: Dockerfile, app/ (src, frontend, pyproject.toml, deploy/), app/deploy/book1_data/
-   git add Dockerfile app/ && git commit -m "Inkference Space" && git push <space-remote> main
+   hf upload <user>/inkference . --repo-type space --exclude ".git/*"
    ```
 
 5. **Secrets** (Space → Settings → Variables and secrets):
-   - `GEMINI_API_KEY` — Ask-the-Archive answers
-   - `GROQ_API_KEY` — post-correction
+   - `GROQ_API_KEY` — post-correction **and** primary Ask-the-Archive answers
+   - `GEMINI_API_KEY` — Ask-the-Archive fallback (used when Groq is rate-limited)
    - *(optional)* `TROCR_MODEL_ID` — your Hub recognizer
 
 6. HF builds the image (~4–5 GB; a few minutes) and boots: it seeds Book 1, then serves.
@@ -64,27 +84,40 @@ Without the keys the app still runs — correction and answers degrade to their
 | `TROCR_MODEL_ID` | `microsoft/trocr-base-handwritten` | recognizer (set to your Hub model) |
 | `HTR_MAX_LONG_EDGE` | `1600` | downscale cap (speed vs accuracy) |
 | `CORRECTION_ENABLED` / `CORRECTION_BACKEND` | `true` / `api` | Qwen correction via Groq |
-| `CORRECTION_API_MODEL` | `qwen/qwen3-32b` | Groq model |
-| `GROQ_API_KEY` | – (secret) | correction key |
-| `LLM_PROVIDER` / `LLM_MODEL` | `gemini` / `gemini-2.5-flash` | Ask-the-Archive |
-| `GEMINI_API_KEY` | – (secret) | RAG answer key |
+| `CORRECTION_API_MODEL` | `qwen/qwen3-32b` | Groq correction model |
+| `LLM_PROVIDER` / `LLM_MODEL` | `groq` / `openai/gpt-oss-120b` | primary Ask-the-Archive model |
+| `LLM_FALLBACK` | `gemini:gemini-2.5-flash-lite` | ordered `provider:model` fallback chain |
+| `GROQ_API_KEY` | – (secret) | correction + primary RAG |
+| `GEMINI_API_KEY` / `GOOGLE_API_KEY` | – (secret) | RAG fallback |
+| `INKFERENCE_LOG_LEVEL` | `INFO` | `DEBUG` for per-page/stage + provider logs |
 | `INKFERENCE_DATA_ROOT` | `/data` | ephemeral corpus store |
+| `CORS_ORIGINS` | `*` | allowed frontend origins |
+
+Key resolution is provider-aware: `LLM_PROVIDER=groq` uses `GROQ_API_KEY`, `=gemini`
+uses `GEMINI_API_KEY`/`GOOGLE_API_KEY` — so switching the provider "just works".
+
+## Logs
+All app logs use the `inkference.*` loggers (`inkference.api`, `.rag`, `.ingest`,
+`.correction`) and print to the container console (visible in the Space **Logs** tab).
+API keys are **redacted** from logs, and provider errors are logged server-side only —
+never returned to the client. Set `INKFERENCE_LOG_LEVEL=DEBUG` for verbose detail.
 
 ## Caveats (free tier)
 - **Ephemeral storage**: `/data` resets on restart → Book 1 re-seeds automatically, but
-  uploaded pages are lost. For persistence, attach paid persistent storage or move the
-  store to a managed DB.
+  uploaded pages are lost. For persistence, attach paid persistent storage or a managed DB.
 - **CPU speed**: live upload is minutes/page (design assumed a GPU). For production, run
   HTR on a serverless GPU (Modal/Replicate) via a `remote` executor.
 - **Sleep**: free Spaces sleep on inactivity (cold start ~30–60 s).
+- **LLM free-tier limits**: Groq gpt-oss-120b ≈ 8k tokens/min (plenty for a demo);
+  Gemini free tier is stingy — the fallback chain + extractive default keep answers flowing.
 
 ## Local run
 
 ```bash
 pip install -r requirements.txt && pip install -e ./app
 python -m inkference.store.seed_book1 --alex ~/Downloads/AlexFiles   # or store.seed for the demo
-uvicorn inkference.api.main:app --port 8000
+uvicorn inkference.api.main:app --port 8000 --log-level info
 ```
 
 See [../projectNotes/running_and_seeds.md](../projectNotes/running_and_seeds.md) for seeds/data-roots
-and [../projectNotes/inkference_platform_plan.md](../projectNotes/inkference_platform_plan.md) for the plan.
+and [../projectNotes/inkference_platform_plan.md](../projectNotes/inkference_platform_plan.md) for the plan.

app/frontend/app.js

@@ -144,22 +144,25 @@ $("#view-toggle").addEventListener("click", (e) => {
 });
 
 /* ---------- Ask the Archive ---------- */
-async function ask(question) {
+async function ask(question, persona) {
   if (!question.trim() || !state.doc) return;
+  const cook = persona === "cook";
   const thread = $("#thread");
   const q = el("div", "bubble-q"); q.textContent = question; thread.appendChild(q);
 
   const ans = el("div", "answer");
+  const tag = cook ? '<span class="in-character">in character</span>' : "";
+  const loading = cook ? "Consulting the journal…" : "…thinking…";
   ans.innerHTML = `<div class="answer-head"><div class="answer-mark">I</div>
-    <span class="answer-who">Inkference</span></div>
-    <div class="answer-body">…thinking…</div>`;
+    <span class="answer-who">${cook ? "Author" : "Inkference"}</span>${tag}</div>
+    <div class="answer-body">${loading}</div>`;
   thread.appendChild(ans);
   thread.scrollTop = thread.scrollHeight;
 
   try {
     const res = await api(`/documents/${state.doc.id}/ask`, {
       method: "POST", headers: { "content-type": "application/json" },
-      body: JSON.stringify({ question }),
+      body: JSON.stringify({ question, persona: persona || null }),
     });
     ans.querySelector(".answer-body").textContent = res.answer;
     if (res.source_pages && res.source_pages.length) {
@@ -177,8 +180,10 @@ async function ask(question) {
   }
   thread.scrollTop = thread.scrollHeight;
 }
-$("#ask-send").addEventListener("click", () => { const i = $("#ask-input"); ask(i.value); i.value = ""; });
-$("#ask-input").addEventListener("keydown", (e) => { if (e.key === "Enter") { ask(e.target.value); e.target.value = ""; } });
+function submitAsk(persona) { const i = $("#ask-input"); ask(i.value, persona); i.value = ""; }
+$("#ask-send").addEventListener("click", () => submitAsk());
+$("#ask-cook").addEventListener("click", () => submitAsk("cook"));
+$("#ask-input").addEventListener("keydown", (e) => { if (e.key === "Enter") submitAsk(); });
 $("#suggestions").addEventListener("click", (e) => { if (e.target.dataset.q) ask(e.target.dataset.q); });
 
 /* ---------- Upload & Process ---------- */

app/frontend/index.html

@@ -91,7 +91,11 @@
         </div>
         <div class="dock">
           <input id="ask-input" placeholder="Ask anything about this document…" />
-          <button id="ask-send" class="send-btn">↑</button>
+          <button id="ask-cook" class="cook-btn" title="Answer in the voice of the author">
+            <svg width="15" height="15" viewBox="0 0 24 24" fill="none"><circle cx="12" cy="12" r="9" stroke="currentColor" stroke-width="1.7"></circle><path d="M15.5 8.5l-2 5-5 2 2-5z" fill="currentColor"></path></svg>
+            Answer as Author
+          </button>
+          <button id="ask-send" class="send-btn" title="Ask">↑</button>
         </div>
         <div class="disclaimer muted">Inkference can make mistakes. Verify answers against the cited pages.</div>
       </div>

app/frontend/styles.css

@@ -180,12 +180,24 @@ body {
   border: 1px solid #ddd0b8; border-radius: 14px; padding: 8px 8px 8px 18px;
   box-shadow: 0 6px 20px rgba(60,42,18,.07);
 }
-.dock input { flex: 1; border: none; outline: none; font-size: 16px; font-family: var(--sans); background: none; color: var(--text); }
+.dock input { flex: 1; min-width: 0; border: none; outline: none; font-size: 16px; font-family: var(--sans); background: none; color: var(--text); }
+.cook-btn {
+  display: flex; align-items: center; gap: 7px; cursor: pointer; white-space: nowrap;
+  font-family: var(--sans); font-size: 13px; font-weight: 500; color: var(--accent);
+  background: #f6ece0; border: 1px solid rgba(138,58,47,.28); border-radius: 10px; padding: 9px 13px;
+}
+.cook-btn:hover { background: #f0e0d0; }
 .send-btn {
   width: 40px; height: 40px; border: none; border-radius: 11px; background: var(--accent);
-  color: #fff; font-size: 18px; cursor: pointer;
+  color: #fff; font-size: 18px; cursor: pointer; flex: 0 0 auto;
 }
 .disclaimer { text-align: center; font-size: 11.5px; color: #a99a82; margin-top: 10px; }
+/* IN CHARACTER pill on Captain Cook answers */
+.in-character {
+  font-size: 10.5px; letter-spacing: .04em; text-transform: uppercase; color: var(--accent);
+  background: rgba(138,58,47,.09); border: 1px solid rgba(138,58,47,.22);
+  border-radius: 999px; padding: 2px 9px;
+}
 
 /* ===== Upload & Process ===== */
 .dropzone {

app/src/inkference/api/main.py

@@ -13,6 +13,7 @@ Endpoints (see projectNotes/inkference_platform_plan.md):
 """
 from __future__ import annotations
 
+import logging
 import os
 import re
 import unicodedata
@@ -33,6 +34,24 @@ from ..config import (
 from ..rag.answer import answer_question
 from . import services
 
+
+def _setup_logging() -> None:
+    """Route all `inkference.*` loggers to the console at INKFERENCE_LOG_LEVEL
+    (default INFO; set DEBUG for verbose). Independent of uvicorn's own loggers."""
+    level = os.getenv("INKFERENCE_LOG_LEVEL", "INFO").upper()
+    lg = logging.getLogger("inkference")
+    lg.setLevel(level)
+    if not lg.handlers:
+        h = logging.StreamHandler()
+        h.setFormatter(logging.Formatter(
+            "%(asctime)s [%(levelname)s] %(name)s: %(message)s", "%H:%M:%S"))
+        lg.addHandler(h)
+        lg.propagate = False
+
+
+_setup_logging()
+logger = logging.getLogger("inkference.api")
+
 app = FastAPI(title="Inkference", version="0.1.0")
 
 app.add_middleware(
@@ -43,6 +62,16 @@ app.add_middleware(
 )
 
 
+@app.middleware("http")
+async def _no_cache_frontend(request, call_next):
+    """Tell browsers to revalidate the static frontend so edits (js/css/html)
+    always load fresh instead of serving a stale cached bundle."""
+    response = await call_next(request)
+    if not request.url.path.startswith("/api/"):
+        response.headers["Cache-Control"] = "no-cache, no-store, must-revalidate"
+    return response
+
+
 # --------------------------------------------------------------------------- #
 # request models
 # --------------------------------------------------------------------------- #
@@ -55,6 +84,7 @@ class CreateDocument(BaseModel):
 class AskRequest(BaseModel):
     question: str
     top_k: int | None = None
+    persona: str | None = None  # "cook" -> answer in character as Captain Cook
 
 
 def _slugify(text: str) -> str:
@@ -165,8 +195,11 @@ def ask(doc_id: int, body: AskRequest) -> dict:
         raise HTTPException(404, "document not found")
     index = services.get_index()
     if not index.exists(doc_id):
+        logger.info("building RAG index for doc %s", doc_id)
         index.build_from_store(doc_id, store)
-    ans = answer_question(doc_id, body.question, index, top_k=body.top_k)
+    logger.info("ask doc=%s persona=%s q=%r", doc_id, body.persona, body.question[:100])
+    ans = answer_question(doc_id, body.question, index, top_k=body.top_k, persona=body.persona)
+    logger.info("ask doc=%s -> sources=%s", doc_id, ans.source_pages)
     return ans.to_dict()
 
 

app/src/inkference/api/services.py

@@ -6,6 +6,7 @@ free-CPU pages process serially; the frontend polls GET /jobs/{id} for progress.
 """
 from __future__ import annotations
 
+import logging
 import threading
 import traceback
 from concurrent.futures import ThreadPoolExecutor
@@ -16,6 +17,8 @@ from ..rag.index import RagIndex
 from ..schemas import JobStatus, Stage
 from ..store import DocumentStore
 
+logger = logging.getLogger("inkference.ingest")
+
 # Map a pipeline Stage -> the job status shown in the queue UI.
 _STAGE_STATUS = {
     Stage.SEGMENTATION: JobStatus.SEGMENTING,
@@ -62,12 +65,15 @@ def submit_ingest(doc_id: int, page_specs: list[tuple[int, int, str]], job_id: i
 def _run_ingest(doc_id: int, page_specs: list[tuple[int, int, str]], job_id: int) -> None:
     store = get_store()
     total = len(page_specs)
+    logger.info("ingest job %s: doc=%s, %d page(s) queued", job_id, doc_id, total)
     store.update_job(job_id, status=JobStatus.QUEUED, total_pages=total, done_pages=0)
     try:
         pipeline = get_pipeline()
         for done, (page_id, page_number, image_path) in enumerate(page_specs):
-            def progress(stage: Stage, frac: float, msg: str, _done=done) -> None:
+            def progress(stage: Stage, frac: float, msg: str, _done=done, _pn=page_number) -> None:
                 overall = (_done + frac) / total
+                logger.debug("job %s page %s: %s %.0f%% — %s",
+                             job_id, _pn, stage.value, frac * 100, msg)
                 store.update_job(
                     job_id,
                     status=_STAGE_STATUS.get(stage, JobStatus.RECOGNIZING),
@@ -76,17 +82,22 @@ def _run_ingest(doc_id: int, page_specs: list[tuple[int, int, str]], job_id: int
                     message=f"Page {page_number} — {msg}",
                 )
 
+            logger.info("job %s: processing page %s (%d/%d)", job_id, page_number, done + 1, total)
             store.set_page_status(page_id, "processing")
             result = pipeline.process_path(image_path, page_number, progress)
             store.save_page_result(page_id, result)
             store.update_job(job_id, done_pages=done + 1)
+            logger.info("job %s: page %s done — %d lines, avg conf %.2f",
+                        job_id, page_number, len(result.lines), result.avg_confidence)
 
         # Rebuild the retrieval index now that new pages exist.
-        get_index().build_from_store(doc_id, store)
+        n_chunks = get_index().build_from_store(doc_id, store)
         store.update_job(
             job_id, status=JobStatus.COMPLETE, progress=1.0, message="Complete"
         )
+        logger.info("ingest job %s complete; RAG index rebuilt (%s chunks)", job_id, n_chunks)
     except Exception as exc:  # surface failure to the job poller
+        logger.exception("ingest job %s FAILED: %s", job_id, exc)
         store.update_job(
             job_id, status=JobStatus.FAILED,
             error=f"{exc}\n{traceback.format_exc()}", message=str(exc),

app/src/inkference/config.py

@@ -108,10 +108,15 @@ class RAGConfig:
         )
     )
     top_k: int = field(default_factory=lambda: _env_int("RAG_TOP_K", 5))
-    # Provider for the written answer: gemini | groq | claude | openai
+    # Primary provider for the written answer: gemini | groq | claude | openai
     llm_provider: str = field(default_factory=lambda: os.getenv("LLM_PROVIDER", "gemini"))
     llm_model: str = field(default_factory=lambda: os.getenv("LLM_MODEL", ""))
     llm_api_key: str = field(default_factory=lambda: os.getenv("LLM_API_KEY", ""))
+    # Ordered fallback chain tried when the primary errors/rate-limits, as a
+    # comma-separated "provider:model" list. After all fail -> extractive fallback.
+    llm_fallback: str = field(
+        default_factory=lambda: os.getenv("LLM_FALLBACK", "gemini:gemini-2.5-flash-lite")
+    )
 
     _PROVIDER_KEYS = {
         "gemini": ("GEMINI_API_KEY", "GOOGLE_API_KEY"),
@@ -124,10 +129,28 @@ class RAGConfig:
         # If no explicit LLM_API_KEY, pull the key for the SELECTED provider so a
         # provider switch (e.g. gemini -> groq) uses the right key automatically.
         if not self.llm_api_key:
-            for env in self._PROVIDER_KEYS.get((self.llm_provider or "").lower(), ()):
-                if os.getenv(env):
-                    self.llm_api_key = os.getenv(env)
-                    break
+            self.llm_api_key = self.key_for(self.llm_provider)
+
+    def key_for(self, provider: str) -> str:
+        """Resolve the API key for a provider (used per-attempt in the chain)."""
+        provider = (provider or "").lower()
+        if self.llm_api_key and provider == (self.llm_provider or "").lower():
+            return self.llm_api_key
+        for env in self._PROVIDER_KEYS.get(provider, ()):
+            if os.getenv(env):
+                return os.getenv(env)
+        return ""
+
+    def attempts(self) -> list[tuple[str, str]]:
+        """Ordered (provider, model) attempts: primary first, then the fallback chain."""
+        out: list[tuple[str, str]] = [((self.llm_provider or "").lower(), self.llm_model)]
+        for part in self.llm_fallback.split(","):
+            part = part.strip()
+            if not part:
+                continue
+            provider, _, model = part.partition(":")
+            out.append((provider.strip().lower(), model.strip()))
+        return out
 
 
 @dataclass

app/src/inkference/rag/answer.py

@@ -14,6 +14,7 @@ class Answer:
     question: str
     answer: str
     source_pages: list[int] = field(default_factory=list)
+    persona: str | None = None  # e.g. "cook" -> shown with an IN CHARACTER tag
     # retrieved evidence, for transparency / debugging
     contexts: list[dict] = field(default_factory=list)
 
@@ -22,6 +23,8 @@ class Answer:
             "question": self.question,
             "answer": self.answer,
             "source_pages": self.source_pages,
+            "persona": self.persona,
+            "in_character": bool(self.persona),
             "contexts": self.contexts,
         }
 
@@ -32,13 +35,15 @@ def answer_question(
     index: RagIndex,
     cfg: RAGConfig = default_rag,
     top_k: int | None = None,
+    persona: str | None = None,
 ) -> Answer:
     retrieved = index.query(doc_id, question, top_k=top_k)
     if not retrieved:
-        return Answer(question, "No transcribed text is available for this document yet.")
+        return Answer(question, "No transcribed text is available for this document yet.",
+                      persona=persona)
 
     contexts = [(r.page_number, r.text) for r in retrieved]
-    text = generate_answer(question, contexts, cfg)
+    text = generate_answer(question, contexts, cfg, persona=persona)
 
     # Distinct source pages in retrieval order -> the design's "Sources" chips.
     seen: set[int] = set()
@@ -52,6 +57,7 @@ def answer_question(
         question=question,
         answer=text,
         source_pages=source_pages,
+        persona=persona,
         contexts=[{"page_number": r.page_number, "score": round(r.score, 4),
                    "text": r.text} for r in retrieved],
     )

app/src/inkference/rag/llm.py

@@ -7,11 +7,22 @@ the app still works at $0 and offline.
 """
 from __future__ import annotations
 
+import logging
+import re
 import time
 
 from ..config import RAGConfig
 from ..config import rag as default_rag
 
+logger = logging.getLogger("inkference.rag")
+
+# Strip secrets before anything is logged (Gemini puts ?key= in the URL; bearer tokens too).
+_SECRET_RE = re.compile(r"(key=)[\w.\-]+|(AIza[\w\-]{20,})|(gsk_[A-Za-z0-9]{20,})|(Bearer\s+\S+)")
+
+
+def _redact(text: str) -> str:
+    return _SECRET_RE.sub("\\1***", str(text))
+
 
 def _post_retry(url: str, retries: int = 3, **kwargs):
     """POST that retries transient 429/503 (rate limit / overload) with backoff."""
@@ -41,6 +52,23 @@ _SYSTEM = (
     "Do not invent facts. Write 2–4 sentences in a clear, scholarly tone."
 )
 
+# In-character persona: answer AS Captain Cook, still grounded in the excerpts.
+_SYSTEM_COOK = (
+    "You are Johann Reinhold Forster, the naturalist aboard HMS Resolution during "
+    "Captain Cook's second voyage and the author of this journal (Books 1-6, "
+    "transcribed from your own handwriting). Answer the reader's question in the "
+    "first person, as yourself, drawing ONLY on the provided excerpts from your own "
+    "journal as your memory of the voyage — do not break character and do not refer "
+    "to yourself as an AI or assistant. Write in a reflective, learned 18th-century "
+    "voice, but keep the language clear for a modern reader. If your journal "
+    "excerpts do not cover the question, say so honestly as yourself rather than "
+    "inventing facts. Write 2-4 sentences."
+)
+
+
+def _system_for(persona: str | None) -> str:
+    return _SYSTEM_COOK if (persona or "").lower() == "cook" else _SYSTEM
+
 
 def _build_prompt(question: str, contexts: list[tuple[int, str]]) -> str:
     blocks = "\n\n".join(f"[Page {pn}]\n{txt}" for pn, txt in contexts)
@@ -57,43 +85,77 @@ def _extractive_fallback(question: str, contexts: list[tuple[int, str]]) -> str:
     top = contexts[0][1].replace("\n", " ").strip()
     pages = ", ".join(str(pn) for pn, _ in contexts)
     return (
-        f"(No language model configured — showing the most relevant transcribed "
-        f"passage from page{'s' if ',' in pages else ''} {pages}.)\n\n“{top}”"
+        f"Showing the most relevant transcribed passage from "
+        f"page{'s' if ',' in pages else ''} {pages}:\n\n“{top}”"
     )
 
 
+def _dispatch(provider: str, model: str, system: str, prompt: str, key: str) -> str:
+    if provider == "gemini":
+        return _call_gemini(model, system, prompt, key)
+    if provider in ("groq", "openai"):
+        return _call_openai_compatible(provider, model, system, prompt, key)
+    if provider == "claude":
+        return _call_claude(model, system, prompt, key)
+    raise ValueError(f"unknown provider: {provider!r}")
+
+
 def generate_answer(
-    question: str, contexts: list[tuple[int, str]], cfg: RAGConfig = default_rag
+    question: str, contexts: list[tuple[int, str]], cfg: RAGConfig = default_rag,
+    persona: str | None = None,
 ) -> str:
-    """contexts = [(page_number, text), ...] in relevance order."""
-    provider = (cfg.llm_provider or "").lower()
-    if not cfg.llm_api_key or provider not in _DEFAULT_MODELS:
-        return _extractive_fallback(question, contexts)
+    """contexts = [(page_number, text), ...] in relevance order.
 
-    model = cfg.llm_model or _DEFAULT_MODELS[provider]
+    Tries the provider chain (primary -> fallbacks from cfg.attempts()); on a
+    provider error/rate-limit it moves to the next, and if all fail returns the
+    extractive fallback. persona="cook" answers in Captain Cook's voice."""
+    system = _system_for(persona)
     prompt = _build_prompt(question, contexts)
-    try:
-        if provider == "gemini":
-            return _call_gemini(model, prompt, cfg.llm_api_key)
-        if provider in ("groq", "openai"):
-            return _call_openai_compatible(provider, model, prompt, cfg.llm_api_key)
-        if provider == "claude":
-            return _call_claude(model, prompt, cfg.llm_api_key)
-    except Exception as exc:  # network/quota/etc — degrade gracefully
-        return _extractive_fallback(question, contexts) + f"\n\n[generation error: {exc}]"
+
+    seen: set[tuple[str, str]] = set()
+    tried_any = False
+    for provider, model in cfg.attempts():
+        if provider not in _DEFAULT_MODELS:
+            logger.debug("skip unknown provider %r", provider)
+            continue
+        model = model or _DEFAULT_MODELS[provider]
+        if (provider, model) in seen:
+            continue
+        seen.add((provider, model))
+        key = cfg.key_for(provider)
+        if not key:
+            logger.debug("skip %s:%s (no API key configured)", provider, model)
+            continue
+        tried_any = True
+        logger.info("RAG answering via %s:%s%s", provider, model,
+                    f" (persona={persona})" if persona else "")
+        try:
+            answer = _dispatch(provider, model, system, prompt, key)
+            logger.info("RAG answer OK via %s:%s (%d chars)", provider, model, len(answer))
+            return answer
+        except Exception as exc:  # rate limit / network / quota -> try next provider
+            # Redacted: never let the API key (in ?key= / bearer) reach the logs.
+            logger.warning("RAG provider %s:%s failed, falling back: %s",
+                           provider, model, _redact(exc))
+
+    # Client never sees provider errors/keys — only the clean extractive passage.
+    if tried_any:
+        logger.warning("RAG all providers failed; using extractive fallback")
+    else:
+        logger.info("RAG no LLM configured; using extractive fallback")
     return _extractive_fallback(question, contexts)
 
 
 # --------------------------------------------------------------------------- #
 # provider calls
 # --------------------------------------------------------------------------- #
-def _call_gemini(model: str, prompt: str, api_key: str) -> str:
+def _call_gemini(model: str, system: str, prompt: str, api_key: str) -> str:
     url = (
         f"https://generativelanguage.googleapis.com/v1beta/models/{model}:generateContent"
         f"?key={api_key}"
     )
     body = {
-        "system_instruction": {"parts": [{"text": _SYSTEM}]},
+        "system_instruction": {"parts": [{"text": system}]},
         "contents": [{"role": "user", "parts": [{"text": prompt}]}],
     }
     r = _post_retry(url, json=body, timeout=60)
@@ -101,7 +163,7 @@ def _call_gemini(model: str, prompt: str, api_key: str) -> str:
     return r.json()["candidates"][0]["content"]["parts"][0]["text"].strip()
 
 
-def _call_openai_compatible(provider: str, model: str, prompt: str, api_key: str) -> str:
+def _call_openai_compatible(provider: str, model: str, system: str, prompt: str, api_key: str) -> str:
     base = "https://api.groq.com/openai/v1" if provider == "groq" else "https://api.openai.com/v1"
     r = _post_retry(
         f"{base}/chat/completions",
@@ -109,7 +171,7 @@ def _call_openai_compatible(provider: str, model: str, prompt: str, api_key: str
         json={
             "model": model,
             "messages": [
-                {"role": "system", "content": _SYSTEM},
+                {"role": "system", "content": system},
                 {"role": "user", "content": prompt},
             ],
             "temperature": 0.2,
@@ -120,7 +182,7 @@ def _call_openai_compatible(provider: str, model: str, prompt: str, api_key: str
     return r.json()["choices"][0]["message"]["content"].strip()
 
 
-def _call_claude(model: str, prompt: str, api_key: str) -> str:
+def _call_claude(model: str, system: str, prompt: str, api_key: str) -> str:
     import requests
 
     r = requests.post(
@@ -133,7 +195,7 @@ def _call_claude(model: str, prompt: str, api_key: str) -> str:
         json={
             "model": model,
             "max_tokens": 600,
-            "system": _SYSTEM,
+            "system": system,
             "messages": [{"role": "user", "content": prompt}],
         },
         timeout=60,