GPU hack fixes

app.py

@@ -340,10 +340,15 @@ def chat_send(message, quests, adventure, theme, photo, selected_id, images, bro
         raise gr.Error(_import_error_message())
     theme = theme if theme in THEMES else "fantasy"
 
-    intent = route_intent(message, _chat_context(quests, selected_id))
-    kind = intent.get("intent", "unknown")
+    # Skip the (GPU) intent classifier when there's no log yet — the only sensible action is to
+    # forge one. Saves a whole GPU reservation on the most common first message.
+    if not quests:
+        intent, kind = {}, "forge"
+    else:
+        intent = route_intent(message, _chat_context(quests, selected_id))
+        kind = intent.get("intent", "unknown")
 
-    if kind == "forge" or (kind == "add_tasks" and not quests):
+    if kind == "forge":
         q, a, fid, im, sc, de, st, b2 = _do_forge(message, theme, photo, browser)
         return ("", q, a, fid, im, sc, de, st, b2)
 
@@ -527,7 +532,7 @@ with gr.Blocks(title="FrogQuest") as demo:
         )
         send_btn = gr.Button("SEND ▶", elem_classes=["pix-btn", "primary", "fq-chat-send"])
     gr.HTML('<p class="fq-chat-hint">e.g. "Finish the report, reply to emails, book dentist" · '
-            '"I finished the report" · "Couldn\'t do the gym — too tired"</p>')
+            '"I finished the report" · "Couldn\'t do the gym — too tired"</p><p>Where to Start, Where to Break and let Go,<br>Where to change <br>Where to look and to grow </p>')
 
     # ---------- wiring ----------
     photo_image.upload(upload_photo, [photo_image, browser], [photo_state, browser])

images.py

@@ -21,30 +21,53 @@ from diffusers import Flux2KleinPipeline
 from PIL import Image
 
 MODEL_ID = "black-forest-labs/FLUX.2-klein-4B"  # verified: ungated, Apache 2.0
-# Quality config — already running at klein's best:
-#   - precision: full bfloat16 weights (the model's NATIVE dtype, NOT quantized). No Q4/Q8
-#     tradeoff like the GGUF LLM has — this is the highest-quality way to run it.
+# Quality config:
+#   - precision: bf16 on GPUs that support it (ZeroGPU/Blackwell = klein's NATIVE dtype, best
+#     quality). On a Turing GPU like T4 (no bf16) we fall back to fp16 — see _get_pipe.
 #   - steps: 4 is the model card's recommended value because klein is DISTILLED; raising it
 #     does not improve a distilled model (and can hurt).
 #   - there is no "context length" for a diffusion model.
-#   - resolution is the only real speed/quality knob; 768 keeps ZeroGPU generation fast.
+#   - resolution is the only real speed/quality knob; 768 keeps generation fast.
 STEPS = 4
 GUIDANCE = 4.0
 MAX_SIDE = 768  # generated-scene resolution (the one quality/speed knob)
+LOW_VRAM_GB = 24  # at/below this (e.g. T4 16GB) use fp16 + CPU offload so ~13GB of weights fit
+
+# Best-effort: pre-fetch the weights at startup so the first @spaces.GPU call doesn't pay the
+# multi-GB download out of its (metered, on ZeroGPU) duration. No-op offline / on fresh checkout.
+try:
+    from huggingface_hub import snapshot_download
+    snapshot_download(MODEL_ID)
+except Exception:
+    pass
 
 _pipe = None
+_offloaded = False  # True when we used CPU offload (small GPU) instead of a full .to("cuda")
 
 
 def _get_pipe():
-    global _pipe
+    """Construct the pipeline lazily INSIDE the GPU call so we can read the REAL device's caps
+    and adapt — this is what lets ONE codebase run on both ZeroGPU (Blackwell, 48GB, native
+    bf16) and a small standard GPU like T4 (Turing, 16GB, fp16 + CPU offload). Just flip the
+    hardware in the Space settings; no code change needed."""
+    global _pipe, _offloaded
     if _pipe is None:
-        _pipe = Flux2KleinPipeline.from_pretrained(MODEL_ID, torch_dtype=torch.bfloat16)
+        bf16 = torch.cuda.is_available() and torch.cuda.is_bf16_supported()
+        dtype = torch.bfloat16 if bf16 else torch.float16  # T4 (Turing) has no bf16
+        _pipe = Flux2KleinPipeline.from_pretrained(MODEL_ID, torch_dtype=dtype)
+        vram_gb = (torch.cuda.get_device_properties(0).total_memory / 1e9
+                   if torch.cuda.is_available() else 0)
+        if 0 < vram_gb < LOW_VRAM_GB:
+            # Small GPU: stream modules GPU<-CPU per step so the weights fit in ~16GB VRAM.
+            _pipe.enable_model_cpu_offload()
+            _offloaded = True
     return _pipe
 
 
 def _gen(prompt: str, image, seed: int) -> Image.Image:
     pipe = _get_pipe()
-    pipe.to("cuda")
+    if not _offloaded:
+        pipe.to("cuda")  # full-residency path (big GPU); offload manages its own device moves
     generator = torch.Generator("cuda").manual_seed(int(seed))
     result = pipe(
         prompt=prompt,
@@ -58,7 +81,7 @@ def _gen(prompt: str, image, seed: int) -> Image.Image:
     return result.images[0]
 
 
-@spaces.GPU(duration=240)
+@spaces.GPU(duration=60)
 def initial_image(user_photo: Image.Image, art_style: str, scene_prompt: str, seed: int) -> Image.Image:
     """Generate the quest's initial scene with the user as the hero (photo as reference)."""
     prompt = (
@@ -68,7 +91,7 @@ def initial_image(user_photo: Image.Image, art_style: str, scene_prompt: str, se
     return _gen(prompt, image=[user_photo], seed=seed)
 
 
-@spaces.GPU(duration=240)
+@spaces.GPU(duration=60)
 def edit_image(base_image: Image.Image, edit_instruction: str, art_style: str, seed: int) -> Image.Image:
     """NEXT PASS (priority 5): edit the existing image into a success/failure state."""
     prompt = f"{art_style}. {edit_instruction}"

llm.py

@@ -26,10 +26,24 @@ from schema import INTENT_SCHEMA, RESPONSE_SCHEMA  # noqa: E402
 GGUF_REPO = "unsloth/NVIDIA-Nemotron-3-Nano-4B-GGUF"
 GGUF_FILE = "*Q8_0*.gguf"
 
-# 128k context. Nemotron Nano 4B supports up to 131072 tokens; ZeroGPU (~48GB) easily holds
-# the KV cache for a 4B model at this length. Our prompts are tiny, but this leaves huge
-# headroom for long to-do lists and growing quest-log context in the chat router.
-N_CTX = 131072
+# Context length is hardware-adaptive (chosen in _get_llm from the REAL device's VRAM). On a big
+# GPU (ZeroGPU, 48GB) we use the full 128k Nemotron supports; on a small standard GPU (e.g. T4
+# 16GB) a 128k KV cache won't fit alongside FLUX, so we drop to 16k — still vastly more than this
+# app's tiny prompts ever need. This is one half of the ZeroGPU<->T4 portability (see images.py).
+N_CTX = 131072          # big GPUs (ZeroGPU)
+N_CTX_SMALL = 16384     # small GPUs (T4 & similar)
+LOW_VRAM_GB = 24        # at/below this, treat the GPU as "small" (T4 = 16GB)
+
+# Best-effort: warm the HF cache at startup so the FIRST @spaces.GPU call doesn't spend its
+# (metered, on ZeroGPU) duration downloading ~4GB. No-op if offline or on a fresh local checkout
+# — the model still lazy-loads from cache/network inside the GPU call either way.
+try:
+    from huggingface_hub import hf_hub_download, list_repo_files  # noqa: E402
+    _gguf = next((f for f in list_repo_files(GGUF_REPO) if "Q8_0" in f and f.endswith(".gguf")), None)
+    if _gguf:
+        hf_hub_download(GGUF_REPO, _gguf)
+except Exception:
+    pass
 
 SYSTEM_PROMPT = """You are FrogQuest's quest designer. Convert the user's real to-do list into a themed text-adventure quest log and OUTPUT JSON ONLY - no prose, no markdown, no code.
 
@@ -110,17 +124,20 @@ def _get_llm():
         _preload_cuda_libs()
         from llama_cpp import Llama
 
+        vram_gb = (torch.cuda.get_device_properties(0).total_memory / 1e9
+                   if torch.cuda.is_available() else 0)
+        n_ctx = N_CTX if vram_gb >= LOW_VRAM_GB else N_CTX_SMALL
         _llm = Llama.from_pretrained(
             repo_id=GGUF_REPO,
-            filename=GGUF_FILE,   # glob -> resolves the exact Q4_K_M file
-            n_gpu_layers=-1,      # offload all layers to GPU
-            n_ctx=N_CTX,
+            filename=GGUF_FILE,   # glob -> resolves the exact Q8_0 file (warmed at import)
+            n_gpu_layers=-1,      # offload all layers (Q8 4B ~4.3GB fits even on a T4)
+            n_ctx=n_ctx,
             verbose=False,
         )
     return _llm
 
 
-@spaces.GPU(duration=180)
+@spaces.GPU(duration=70)
 def generate_quests_raw(todos: str, theme: str) -> dict:
     """Return the model's raw JSON object (UNVALIDATED - caller must validate_and_clamp)."""
     llm = _get_llm()
@@ -140,7 +157,7 @@ def generate_quests_raw(todos: str, theme: str) -> dict:
     return _extract_json(content)
 
 
-@spaces.GPU(duration=60)
+@spaces.GPU(duration=45)
 def route_intent(message: str, context: str) -> dict:
     """Classify one Frog Master chat message into {intent, target_task?, reason?}.