Fix Space startup crash: drop device_map="auto" for CPU inference

On HF cpu-basic, accelerate's "auto" device_map split Qwen across
devices and marked layers for disk offload. PEFT then failed applying
the LoRA adapter over that offload index:

KeyError: 'base_model.model.model.model.embed_tokens'

which killed the uvicorn import of main.py and put the Space in
RUNTIME_ERROR. Load the model whole and move it to the device instead,
and use float32 off-GPU since fp16 on CPU is slow with unimplemented ops.

Also load the model lazily on first /contentdistillery request, so a
model problem can no longer take down captioning and cbow with it, and
strip a stray merge-conflict marker from README.md.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

README.md

@@ -14,4 +14,5 @@ This Hugging Face Space includes multiple AI tools:
 - 🖼️ ViT image captioning
 - 📝 PPO-based Reddit summarization (coming soon)
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference<<<<<<< HEAD
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+

ppo_logic.py

@@ -5,6 +5,7 @@ from transformers import AutoTokenizer, AutoModelForCausalLM
 from peft import PeftModel
 import argparse
 import os
+import threading
 
 # -------------------------------
 # Config
@@ -15,28 +16,52 @@ CHECKPOINT_DIR = "./qwen_loRA"
 DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 MAX_NEW_TOKENS = 256
 
+# fp16 is a GPU-only win here: on CPU it is slow and some ops are unimplemented.
+DTYPE = torch.float16 if DEVICE.type == "cuda" else torch.float32
+
 # -------------------------------
-# Load tokenizer and model
+# Load tokenizer and model (lazily, on first summary request)
 # -------------------------------
-print("🔄 Loading tokenizer and model...")
-tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME, trust_remote_code=True)
-tokenizer.pad_token = tokenizer.eos_token
+_tokenizer = None
+_model = None
+_load_lock = threading.Lock()
+
+
+def _load():
+    """Load the base model + LoRA adapter once, on first use."""
+    global _tokenizer, _model
 
-base_model = AutoModelForCausalLM.from_pretrained(
-    MODEL_NAME,
-    torch_dtype=torch.float16,
-    device_map="auto",
-    trust_remote_code=True
-)
+    with _load_lock:
+        if _model is not None:
+            return _tokenizer, _model
+
+        print("🔄 Loading tokenizer and model...")
+        tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME, trust_remote_code=True)
+        tokenizer.pad_token = tokenizer.eos_token
+
+        # No device_map="auto": on a CPU-only box accelerate splits the model and
+        # offloads layers, which breaks the PEFT adapter load. Load whole, then move.
+        base_model = AutoModelForCausalLM.from_pretrained(
+            MODEL_NAME,
+            torch_dtype=DTYPE,
+            trust_remote_code=True
+        )
+
+        model = PeftModel.from_pretrained(base_model, CHECKPOINT_DIR)
+        model.eval()
+        model = model.to(DEVICE)
+
+        _tokenizer, _model = tokenizer, model
+        print("✅ Model ready on", DEVICE)
+        return _tokenizer, _model
 
-model = PeftModel.from_pretrained(base_model, CHECKPOINT_DIR)
-model.eval()
-model = model.to(DEVICE)
 
 # -------------------------------
 # Generate Summary
 # -------------------------------
 def generate_summary(post: str) -> str:
+    tokenizer, model = _load()
+
     #prompt = f"Instruction: Summarize the post in one sentence.\n\nPost:\n{post}\n\nSummary:"
     # prompt = f"Please summarize the following Reddit post in 1–2 sentences:\n\n{post}\n\nSummary:"
     prompt = f"Instruction: Summarize the post in 1-2 sentences.\n\nPost:\n{post}\n\nSummary:"
@@ -50,7 +75,6 @@ def generate_summary(post: str) -> str:
             do_sample=False,
             # top_k=50,
             # top_p=0.95,
-            temperature=1.0,
             pad_token_id=tokenizer.pad_token_id,
             use_cache=True
         )
@@ -71,4 +95,4 @@ if __name__ == "__main__":
     print("\n🤖 Generating summary...\n")
 
     summary = generate_summary(args.post)
-    print("✅ Summary:\n", summary)
+    print("✅ Summary:\n", summary)