fix: re-run scribe on cards a failed chunk left blank; never paint an empty (subjectless) prompt

app.py

@@ -448,6 +448,29 @@ def do_conjure(theme, style, custom):
     yield from _run_parallel(chunks, fn=lambda i, ch: _gpu_scribe(deck, ch),
                              on_done=lambda res, ch: merge_fields(res, ch, _SCRIBE_OUT),
                              tick=scribe_tick)
+
+    # RECONCILE: if a scribe chunk failed, its cards still have blank meaning/art.
+    # Re-run the scribe on JUST those (one more GPU pass) so we never paint an empty,
+    # subjectless prompt — which FLUX fills with a generic medieval/religious figure —
+    # and so those cards aren't left meaningless in the reading.
+    def _blank(c):
+        return not (c.get("art_prompt") or "").strip()
+    missing = [c["arcana_number"] for c in deck["cards"] if _blank(c)]
+    for _try in range(2):
+        if not missing:
+            break
+        yield stay("<div class='gen-load'><div class='gen-spin'></div>"
+                   "<h2>✦ Writing the lore</h2><p>finishing the last cards…</p></div>")
+        try:
+            merge_fields(_gpu_scribe(deck, missing), missing, _SCRIBE_OUT)
+        except Exception as e:
+            print(f"[SCRIBE_RECONCILE_ERR] {missing}: {type(e).__name__}: {e}", flush=True)
+        missing = [c["arcana_number"] for c in deck["cards"] if _blank(c)]
+    if missing:  # still blank after retries → at least give FLUX the concept as subject
+        print(f"[SCRIBE_FALLBACK] concept-only art for {missing}", flush=True)
+        for c in deck["cards"]:
+            if _blank(c):
+                c["art_prompt"] = c["concept"]
     t_scribe = time.time() - t_scribe0
 
     # Lore is done → OPEN THE REAL DECK VIEW now (names + full meanings); the art then
@@ -1060,6 +1083,153 @@ footer { display:none !important; }
 """
 
 
+TRACE_ON = os.environ.get("ARCANA_TRACE", "") == "1"  # the JG1310 trace-logging copy sets this
+
+
+def _assemble_trace(steps, task, theme, visual_style, deck_id, models, pipeline, extra=None):
+    import hashlib
+    from datetime import datetime, timezone
+    trace_id = hashlib.sha1(f"{deck_id}:{task}".encode("utf-8")).hexdigest()[:8]
+    obj = {
+        "trace_id": trace_id, "app": "Arcana", "task": task,
+        "theme": theme, "visual_style": visual_style, "deck_id": deck_id,
+        "ts": datetime.now(timezone.utc).isoformat(),
+        "models": models, "pipeline": pipeline, "n_steps": len(steps),
+        "provenance": ("Live capture — every step's prompts, raw model output, and "
+                       "latency were recorded as the pipeline ran (sequential trace "
+                       "harness on the JG1310/arcana-gpu-dev trace-logging copy)."),
+        "steps": steps,
+    }
+    if extra:
+        obj.update(extra)
+    return obj
+
+
+_TR_TEXT = {"name": "Qwen3-14B", "backend": "llama.cpp"}
+_TR_IMG = {"name": "FLUX.2-klein-9B", "backend": "diffusers"}
+
+
+def _heartbeat(target, args):
+    """Run `target(*args, state)` in a worker thread, yielding a heartbeat every ~3s so
+    the SSE stream stays alive through long GPU phases; final yield is the result/ERROR.
+    Splitting the work into SHORT GPU calls (this is used per-phase) also avoids the
+    'GPU task aborted' that a single multi-minute ZeroGPU hold triggers."""
+    import threading, time as _t
+    state = {"done": False, "result": None, "err": None, "msg": "starting…"}
+    th = threading.Thread(target=target, args=(*args, state), daemon=True)
+    th.start()
+    while not state["done"]:
+        _t.sleep(3)
+        yield f"⏳ {state['msg']}"
+    yield ("ERROR\n" + state["err"]) if state["err"] else state["result"]
+
+
+def _run_llm_phase(theme, style, custom, question, state):
+    """designer → scribe (4 chunks) → reader, all through tracing wrappers. Returns the
+    deck + the LLM trace steps (no images here, so it stays a short ~150s GPU call)."""
+    import json as _json, traceback
+    from arcana import tracing, gpu_models as gm
+    from arcana.styles import resolve_style
+    from arcana.designer import design_deck
+    from arcana.loremaster import scribe_deck
+    from arcana.build import deck_dir, slugify, _seed_base
+    from arcana.reader import draw_spread, card_partial, final_synthesis
+    from arcana.llm import get_llm
+    try:
+        theme = (theme or "Greek mythology").strip()
+        question = (question or "").strip()
+        TEXT = {**_TR_TEXT, "repo": gm.MAP_REPO, "file": gm.MAP_FILE}
+        gtr = tracing.Tracer()
+        gllm = tracing.TracingLLM(get_llm(), gtr)
+        state["msg"] = "designer: mapping the 22 arcana…"
+        gtr.ctx("designer", "mapping")
+        deck = design_deck(theme, llm=gllm)
+        style_id, style_suffix = resolve_style(style, custom)
+        deck_id = f"{slugify(theme)}-{style_id}-{int(time.time())}"
+        os.makedirs(deck_dir(deck_id), exist_ok=True)
+        deck.update({"deck_id": deck_id, "theme": theme, "visual_style": style_id,
+                     "style_suffix": style_suffix, "seed_base": _seed_base(theme, style_id)})
+        nums = [c["arcana_number"] for c in deck["cards"]]
+        chunks = [sorted(c) for c in (nums[i::4] for i in range(4)) if c]
+        for wi, ch in enumerate(chunks):
+            state["msg"] = f"scribe: writing lore + art ({wi + 1}/{len(chunks)} batches)…"
+            gtr.ctx("scribe", "meaning+art", worker=wi, cards=ch)
+            scribe_deck(deck, subset=ch, llm=gllm)
+        gen_llm_steps = gtr.steps
+
+        rtr = tracing.Tracer()
+        rllm = tracing.TracingLLM(get_llm(), rtr)
+        drawn = draw_spread(deck, spread="three", reversals=True, seed=7)
+        partials = []
+        for i in range(len(drawn)):
+            state["msg"] = f"reader: interpreting card {i + 1}/{len(drawn)}…"
+            rtr.ctx("reader", "card_reading", position=drawn[i]["position"],
+                    concept=drawn[i]["concept"], orientation=drawn[i]["orientation"])
+            partials.append(card_partial(deck, question, drawn, i, llm=rllm))
+        state["msg"] = "reader: final synthesis…"
+        rtr.ctx("reader", "synthesis")
+        final_synthesis(deck, question, drawn, partials, llm=rllm)
+
+        meta = {"theme": theme, "visual_style": style_id, "deck_id": deck_id,
+                "question": question or "(open reading)", "spread": "three",
+                "text_model": TEXT, "image_model": _TR_IMG,
+                "drawn": [{"position": d["position"], "concept": d["concept"],
+                           "orientation": d["orientation"]} for d in drawn]}
+        state["result"] = _json.dumps({"deck": deck, "gen_llm_steps": gen_llm_steps,
+                                       "read_steps": rtr.steps, "meta": meta},
+                                      ensure_ascii=False)
+    except Exception:
+        state["err"] = traceback.format_exc()
+    finally:
+        state["done"] = True
+
+
+def _run_paint_phase(deck_json, subset_csv, state):
+    """Trace the FLUX image calls for a subset of cards (or 'back'). Image is discarded —
+    the trace records the real call (styled prompt the model receives, seed, latency).
+    Kept to a short subset per call so the GPU hold stays well under the abort threshold."""
+    import json as _json, traceback
+    from arcana import tracing, gpu_models as gm
+    from arcana.imagegen import get_imagegen
+    from arcana.build import _back_prompt
+    try:
+        deck = _json.loads(deck_json)
+        by_n = {c["arcana_number"]: c for c in deck["cards"]}
+        style_suffix = deck.get("style_suffix")
+        seed_base = deck["seed_base"]
+        IMG = {**_TR_IMG, "repo": gm.IMAGE_MODEL}
+        tr = tracing.Tracer()
+        tig = tracing.TracingImageGen(get_imagegen(deck_style=style_suffix), tr, style_suffix, IMG["name"])
+        if subset_csv.strip().lower() == "back":
+            state["msg"] = "painter: deck-back…"
+            tig.generate(_back_prompt(style_suffix), seed=seed_base - 1)
+            tr.steps[-1]["card"] = {"deck_back": True}
+        else:
+            nums = [int(x) for x in subset_csv.split(",") if x.strip() != ""]
+            for i, m in enumerate(nums):
+                state["msg"] = f"painter: FLUX rendering {i + 1}/{len(nums)} (card {m})…"
+                tig.generate(by_n[m].get("art_prompt", ""), seed=seed_base + m)
+                tr.steps[-1]["card"] = {"arcana_number": m, "arcana_name": by_n[m].get("arcana_name"),
+                                        "concept": by_n[m].get("concept")}
+        state["result"] = _json.dumps({"paint_steps": tr.steps}, ensure_ascii=False)
+    except Exception:
+        state["err"] = traceback.format_exc()
+    finally:
+        state["done"] = True
+
+
+@GPU(duration=300)
+def trace_llm(theme, style, custom, question):
+    """LIVE trace (trace-logging copy only): designer + scribe + reader in one short call."""
+    yield from _heartbeat(_run_llm_phase, (theme, style, custom, question))
+
+
+@GPU(duration=300)
+def trace_paint(deck_json, subset_csv):
+    """LIVE trace (trace-logging copy only): FLUX image calls for a subset of cards."""
+    yield from _heartbeat(_run_paint_phase, (deck_json, subset_csv))
+
+
 # ----------------------------------------------------------------- ui
 def build_demo() -> gr.Blocks:
     with gr.Blocks(title="Arcana") as demo:
@@ -1211,6 +1381,21 @@ def build_demo() -> gr.Blocks:
         # on load, collapse to just the landing page.
         demo.load(lambda: show("landing"), None, pages)
 
+        # TRACE-LOGGING COPY ONLY (ARCANA_TRACE=1): a hidden endpoint that runs the full
+        # pipeline through tracing wrappers and returns the live traces. Never registered
+        # on the production Space, so its behaviour is byte-identical there.
+        if TRACE_ON:
+            _llm_in = [gr.Textbox("Greek mythology", visible=False),
+                       gr.Textbox("rider-waite-smith", visible=False),
+                       gr.Textbox("", visible=False), gr.Textbox("", visible=False)]
+            _llm_out = gr.Textbox(visible=False)
+            gr.Button(visible=False).click(trace_llm, _llm_in, _llm_out, api_name="trace_llm")
+            _pt_deck = gr.Textbox("", visible=False)
+            _pt_sub = gr.Textbox("", visible=False)
+            _pt_out = gr.Textbox(visible=False)
+            gr.Button(visible=False).click(trace_paint, [_pt_deck, _pt_sub], _pt_out,
+                                           api_name="trace_paint")
+
     return demo