app.py & generate.py simplified

app/models/llm.py

@@ -13,118 +13,26 @@ _model: Any = None
 _init_lock = threading.Lock()
 
 
-def _validate_gguf_file(model_path: str) -> None:
-    if not os.path.isfile(model_path):
-        raise FileNotFoundError(f"Downloaded model file does not exist: {model_path}")
-
-    size_bytes = os.path.getsize(model_path)
-    with open(model_path, "rb") as fh:
-        magic = fh.read(4)
-        fh.seek(0)
-        first_128 = fh.read(128)
-
-    print(
-        f"[load_model] GGUF file check: size={size_bytes / (1024 ** 2):.1f} MB, "
-        f"magic={magic!r}",
-        flush=True,
-    )
-
-    if magic != b"GGUF":
-        preview = first_128.decode("utf-8", errors="replace")
-        raise RuntimeError(
-            "Downloaded file is not a valid GGUF file. "
-            f"Expected magic b'GGUF', got {magic!r}. First bytes: {preview!r}"
-        )
-
-    if size_bytes < 100 * 1024 * 1024:
-        raise RuntimeError(
-            "Downloaded GGUF file is unexpectedly small. "
-            f"Size was {size_bytes} bytes; this often means a bad upload or LFS pointer."
-        )
-
-
-def _preload_cuda_libs() -> None:
-    """Expose pip-installed CUDA runtime to llama.cpp on ZeroGPU (no system libcudart)."""
-    try:
-        import ctypes
-
-        import nvidia.cublas
-        import nvidia.cuda_runtime
-    except ImportError:
-        return
-
-    lib_dirs: list[str] = []
-    for module, lib_name in (
-        (nvidia.cublas, "libcublas.so.12"),
-        (nvidia.cuda_runtime, "libcudart.so.12"),
-    ):
-        lib_dir = os.path.join(module.__path__[0], "lib")
-        lib_path = os.path.join(lib_dir, lib_name)
-        if os.path.isfile(lib_path):
-            ctypes.CDLL(lib_path, mode=ctypes.RTLD_GLOBAL)
-            lib_dirs.append(lib_dir)
-
-    if lib_dirs:
-        existing = os.environ.get("LD_LIBRARY_PATH", "")
-        merged = lib_dirs + ([existing] if existing else [])
-        os.environ["LD_LIBRARY_PATH"] = ":".join(merged)
-
-
 def load_model() -> Any:
     global _model
-    print("[load_model] called", flush=True)
 
     if _model is not None:
-        print("[load_model] returning cached model", flush=True)
         return _model
 
     with _init_lock:
         if _model is not None:
             return _model
 
-        force_download = os.getenv("LLAMA_FORCE_DOWNLOAD", "0") == "1"
-        print(
-            f"[load_model] downloading/resolving model {HF_REPO}/{HF_FILENAME} "
-            f"(force_download={force_download})...",
-            flush=True,
-        )
-        model_path = hf_hub_download(
-            repo_id=HF_REPO,
-            filename=HF_FILENAME,
-            force_download=force_download,
-        )
-        print(f"[load_model] model resolved at {model_path}", flush=True)
-        _validate_gguf_file(model_path)
+        model_path = hf_hub_download(repo_id=HF_REPO, filename=HF_FILENAME)
 
-        _preload_cuda_libs()
         from llama_cpp import Llama
 
-        gpu_layers = int(os.getenv("LLAMA_GPU_LAYERS", "-1"))
-        n_ctx = int(os.getenv("LLAMA_N_CTX", "2048"))
-        n_threads = int(os.getenv("LLAMA_N_THREADS", "4"))
-        verbose = os.getenv("LLAMA_VERBOSE", "0") == "1"
-        print(
-            f"[load_model] initializing Llama "
-            f"(n_gpu_layers={gpu_layers}, n_ctx={n_ctx}, n_threads={n_threads}, verbose={verbose})",
-            flush=True,
+        _model = Llama(
+            model_path=model_path,
+            n_ctx=int(os.getenv("LLAMA_N_CTX", "2048")),
+            n_gpu_layers=int(os.getenv("LLAMA_GPU_LAYERS", "0")),
+            n_threads=int(os.getenv("LLAMA_N_THREADS", "4")),
+            verbose=os.getenv("LLAMA_VERBOSE", "0") == "1",
         )
 
-        try:
-            _model = Llama(
-                model_path=model_path,
-                n_ctx=n_ctx,
-                n_gpu_layers=gpu_layers,
-                n_threads=n_threads,
-                verbose=verbose,
-            )
-        except Exception as exc:
-            raise RuntimeError(
-                f"llama.cpp failed to load a valid GGUF file from {model_path}. "
-                "If the file check above says magic=b'GGUF' and the size is large, "
-                "this is usually a llama-cpp-python/GGUF compatibility issue or a bad quantized export. "
-                "Try setting LLAMA_VERBOSE=1 for the next run."
-            ) from exc
-
-        print("[load_model] model initialized", flush=True)
-
     return _model

app/recs/generate.py

@@ -11,18 +11,8 @@ from app.models.llm import load_model
 
 TARGET_CPL = 20.0
 
-_IM_END = "<|im_end|>"
-_STOP_SEQUENCES = [_IM_END, "<|im_start|>", "</s>"]
-
 _infer_lock = threading.Lock()
 
-_SYSTEM = (
-    "You are a Google Ads analyst. "
-    "Reply with 3 to 5 markdown bullet points only. "
-    "Each bullet must be one short, actionable insight about the campaign data. "
-    "No introduction, no numbered lists, no step-by-step reasoning."
-)
-
 
 def fallback_explanation(rec: Dict | None = None) -> str:
     return "This recommendation was generated from campaign performance metrics."
@@ -30,12 +20,7 @@ def fallback_explanation(rec: Dict | None = None) -> str:
 
 def _strip_thinking(text: str) -> str:
     text = re.sub(r"<\s*think\s*>.*?<\s*/\s*think\s*>", "", text, flags=re.DOTALL | re.IGNORECASE)
-    text = re.sub(
-        r"<think>.*?</think>",
-        "",
-        text,
-        flags=re.DOTALL | re.IGNORECASE,
-    )
+    text = re.sub(r"<think>.*?</think>", "", text, flags=re.DOTALL | re.IGNORECASE)
     return text.strip()
 
 
@@ -47,8 +32,6 @@ def _looks_like_garbage(text: str) -> bool:
         return True
     if "google ads analyst" in lower and text.count("-") < 2:
         return True
-    if "ads performance analyst" in lower and text.count("-") < 2:
-        return True
     if re.search(r"(?:\d[\s\n]+){6,}", text):
         return True
     digit_ratio = sum(ch.isdigit() for ch in text) / max(len(text), 1)
@@ -57,13 +40,7 @@ def _looks_like_garbage(text: str) -> bool:
 
 
 def is_fallback_output(text: str) -> bool:
-    lower = (text or "").lower()
-    return (
-        not text
-        or lower.startswith("warning:")
-        or "analysis failed:" in lower
-        or text.startswith("This recommendation was generated")
-    )
+    return not text or text.startswith("WARNING:") or text.startswith("This recommendation was generated")
 
 
 def is_bad_llm_output(text: str) -> bool:
@@ -90,54 +67,6 @@ def sanitize_explanation(text: str, rec: Dict | None = None) -> str:
     return flat
 
 
-def _messages_to_prompt(messages: list[dict[str, str]]) -> str:
-    system = next((msg["content"] for msg in messages if msg["role"] == "system"), "")
-    user = next((msg["content"] for msg in messages if msg["role"] == "user"), "")
-    return (
-        f"{system}\n\n"
-        f"Request:\n{user}\n\n"
-        "Answer with only the final bullet points:\n"
-    )
-
-def _message_text(message: dict) -> str:
-    content = (message.get("content") or "").strip()
-    reasoning = (message.get("reasoning_content") or "").strip()
-    if content and reasoning and _looks_like_garbage(content):
-        return reasoning
-    return content or reasoning
-
-
-def _infer(llm, messages: list[dict[str, str]]) -> str:
-    max_tokens = int(os.getenv("LLAMA_MAX_TOKENS", "384"))
-    temperature = float(os.getenv("LLAMA_TEMPERATURE", "0.35"))
-    use_chat_completion = os.getenv("LLAMA_USE_CHAT_COMPLETION", "0") == "1"
-
-    if use_chat_completion:
-        try:
-            out = llm.create_chat_completion(
-                messages=messages,
-                max_tokens=max_tokens,
-                temperature=temperature,
-            )
-            raw = _message_text(out["choices"][0]["message"])
-            if raw and not _looks_like_garbage(raw):
-                print("OK [generate_explanation] via create_chat_completion", flush=True)
-                return raw
-            print("WARNING [generate_explanation] chat_completion empty/garbage - raw fallback", flush=True)
-        except Exception as exc:
-            print(f"WARNING [generate_explanation] chat_completion failed - raw fallback: {repr(exc)}", flush=True)
-
-    print("LLM [generate_explanation] using raw llama prompt", flush=True)
-    out = llm(
-        _messages_to_prompt(messages),
-        max_tokens=max_tokens,
-        temperature=temperature,
-        stop=_STOP_SEQUENCES,
-        echo=False,
-    )
-    return (out["choices"][0].get("text") or "").strip()
-
-
 def _coerce_prompt(prompt: str | Dict, rec: Dict | None) -> tuple[str, Dict | None]:
     if isinstance(prompt, dict):
         rec = rec or prompt
@@ -149,47 +78,31 @@ def _coerce_prompt(prompt: str | Dict, rec: Dict | None) -> tuple[str, Dict | No
 
 
 def generate_explanation(prompt: str | Dict, rec: Dict | None = None, stream: bool = False):
-    print("\nLLM [generate_explanation] CALLED", flush=True)
-
     try:
         user_content, rec = _coerce_prompt(prompt, rec)
-        print(
-            f"PROMPT [generate_explanation] prompt type={type(prompt).__name__} "
-            f"len={len(user_content)}",
-            flush=True,
-        )
         if "/no_think" not in user_content:
             user_content = f"{user_content}\n/no_think"
 
-        messages = [
-            {"role": "system", "content": _SYSTEM},
-            {"role": "user", "content": user_content},
-        ]
-
         with _infer_lock:
             llm = load_model()
-            print("MODEL [generate_explanation] model loaded", flush=True)
-            print("LLM [generate_explanation] calling LLM...", flush=True)
-            raw = _infer(llm, messages)
-
-        print("LLM [generate_explanation] response received", flush=True)
-        print("RAW [generate_explanation] raw output length:", len(raw), flush=True)
-        if raw:
-            print("RAW [generate_explanation] raw preview:", raw[:400], flush=True)
+            out = llm(
+                user_content,
+                max_tokens=int(os.getenv("LLAMA_MAX_TOKENS", "384")),
+                temperature=float(os.getenv("LLAMA_TEMPERATURE", "0.35")),
+                stop=["</s>"],
+                echo=False,
+            )
 
+        raw = (out["choices"][0].get("text") or "").strip()
         clean = sanitize_explanation(raw, rec)
-        if is_bad_llm_output(clean):
-            clean = fallback_explanation(rec)
-        print("OK [generate_explanation] cleaned output ready", flush=True)
+
         if stream:
             return iter([clean])
         return clean
 
     except Exception as e:
-        print("ERROR [generate_explanation] ERROR:", repr(e), flush=True)
         traceback.print_exc()
         err = f"WARNING: Analysis failed: {e}"
         if stream:
             return iter([err])
         return err
-