Update app.py

app.py

@@ -1,111 +1,153 @@
 import os
+import shutil
+import time
+import stat
 import requests
 from huggingface_hub import hf_hub_download, hf_hub_url
 from llama_cpp import Llama
 import gradio as gr
-import tempfile
 
-# -------------------------
-# Config: change if you want
-# -------------------------
+# --------- config ----------
 REPO_ID = "mradermacher/EuroLLM-1.7B-Instruct-GGUF"
 FILENAME = "EuroLLM-1.7B-Instruct.Q8_0.gguf"
 SYSTEM_PROMPT = "You are a helpful assistant. Answer concisely and helpfully."
-
-# local path we'll store the model
 MODEL_DIR = os.path.join(os.path.abspath(os.path.dirname(__file__)), "models")
 os.makedirs(MODEL_DIR, exist_ok=True)
-MODEL_PATH = os.path.join(MODEL_DIR, FILENAME)
+DEST_PATH = os.path.join(MODEL_DIR, FILENAME)
+# ---------------------------
 
-# -------------------------
-# Helper: robust download
-# -------------------------
-def download_from_hf(repo_id: str, filename: str, dest: str) -> str:
-    """Download using huggingface_hub if possible; fallback to direct url via requests."""
+def robust_download(repo_id: str, filename: str, dest: str, max_attempts: int = 2) -> str:
+    """
+    Try hf_hub_download first. If it returns a cached path, copy it into `dest`.
+    If hf_hub_download fails, fallback to huggingface direct URL via requests.
+    After placing the file at dest, fsync and chmod to ensure llama-cpp-python can read it.
+    """
+    # quick exit if already present with some reasonable size
     if os.path.exists(dest) and os.path.getsize(dest) > 0:
-        print(f"Model already exists at {dest}")
+        print(f"[robust_download] Already present: {dest} ({os.path.getsize(dest)} bytes)")
         return dest
 
-    try:
-        print("Trying hf_hub_download...")
-        path = hf_hub_download(repo_id=repo_id, filename=filename, cache_dir=MODEL_DIR)
-        # hf_hub_download may return a cache path; copy/move to dest if needed
-        if os.path.abspath(path) != os.path.abspath(dest):
-            # move the cached file into our models folder with the expected name
-            os.replace(path, dest)
-            path = dest
-        print("Downloaded via hf_hub_download:", path)
-        return path
-    except Exception as e:
-        print("hf_hub_download failed:", e)
-        # fallback: construct the direct URL and download via requests
+    last_err = None
+    for attempt in range(1, max_attempts + 1):
         try:
-            print("Falling back to direct URL via requests...")
-            url = hf_hub_url(repo_id=repo_id, filename=filename)
-            # url is the Hub URL (signed? but usually works for public repos)
-            # If user provided direct URL with ?download=true, you can paste that directly.
-            print("Downloading from:", url)
-            with requests.get(url, stream=True, timeout=60) as r:
-                r.raise_for_status()
-                with open(dest, "wb") as f:
-                    for chunk in r.iter_content(chunk_size=8192):
-                        if chunk:
-                            f.write(chunk)
-            print("Downloaded fallback to:", dest)
+            print(f"[robust_download] Attempt {attempt}: hf_hub_download...")
+            cached_path = hf_hub_download(repo_id=repo_id, filename=filename, cache_dir=MODEL_DIR)
+            print("[robust_download] hf_hub_download returned:", cached_path)
+
+            # If hf_hub_download already saved directly at dest, great
+            if os.path.abspath(cached_path) == os.path.abspath(dest):
+                print("[robust_download] File is already at dest.")
+            else:
+                # copy to dest (safer than os.replace in some mount setups)
+                print(f"[robust_download] Copying cached file -> {dest}")
+                shutil.copy2(cached_path, dest)
+
+            # ensure it's synced to disk and readable
+            with open(dest, "rb") as f:
+                f.flush()
+                try:
+                    os.fsync(f.fileno())
+                except OSError:
+                    # some environments / filesystems may not support fsync; ignore if so
+                    pass
+
+            # set sane permissions
+            os.chmod(dest, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH)  # 0o644
+
+            # verify
+            size = os.path.getsize(dest)
+            if size == 0:
+                raise RuntimeError("Downloaded file has size 0 after copy")
+            print(f"[robust_download] Success: {dest} ({size} bytes)")
             return dest
-        except Exception as e2:
-            raise RuntimeError(f"Both hf_hub_download and direct download failed: {e2}")
-
-# -------------------------
-# Ensure model is present
-# -------------------------
-model_path = download_from_hf(REPO_ID, FILENAME, MODEL_PATH)
-
-# -------------------------
-# Load the model (llama-cpp-python)
-# -------------------------
-llm = Llama(
-    model_path=model_path,
-    n_ctx=2048,      # lower if you need less memory
-    n_threads=4,
-    n_gpu_layers=0,  # CPU-only. If you have GPU layers available, adjust.
-    # stream is set per-call in create_chat_completion below.
-)
-
-# -------------------------
-# Chat formatting helpers
-# -------------------------
+
+        except Exception as e:
+            print(f"[robust_download] hf_hub_download attempt failed: {e}")
+            last_err = e
+            # fallback to direct requests download next loop or after attempts exhausted
+            time.sleep(1)
+
+    # fallback: direct URL using hf_hub_url and requests
+    try:
+        print("[robust_download] Falling back to direct download via requests...")
+        url = hf_hub_url(repo_id=repo_id, filename=filename)
+        print("[robust_download] Downloading from URL:", url)
+        with requests.get(url, stream=True, timeout=120) as r:
+            r.raise_for_status()
+            tmp_path = dest + ".part"
+            with open(tmp_path, "wb") as f:
+                for chunk in r.iter_content(chunk_size=8192):
+                    if chunk:
+                        f.write(chunk)
+                        f.flush()
+            # move into place
+            shutil.move(tmp_path, dest)
+        # fsync + chmod
+        with open(dest, "rb") as f:
+            try:
+                os.fsync(f.fileno())
+            except OSError:
+                pass
+        os.chmod(dest, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH)
+        print("[robust_download] Direct download success:", dest)
+        return dest
+    except Exception as e2:
+        print("[robust_download] Direct download failed:", e2)
+        raise RuntimeError(f"All download attempts failed. last_err={last_err}, last_fallback_err={e2}")
+
+# ---------- ensure model present ----------
+print("Trying to ensure model is present at DEST_PATH:", DEST_PATH)
+model_path = robust_download(REPO_ID, FILENAME, DEST_PATH)
+
+# debug check: list the models folder and stat
+print("DEBUG: listing model dir:", MODEL_DIR)
+for fn in sorted(os.listdir(MODEL_DIR)):
+    p = os.path.join(MODEL_DIR, fn)
+    try:
+        st = os.stat(p)
+        print(f" - {fn}: exists, size={st.st_size}, mode={oct(st.st_mode)}")
+    except FileNotFoundError:
+        print(f" - {fn}: NOT FOUND after copy (weird)")
+
+# small safety wait for FS to settle (rarely needed, but prevents race in some hosted FS)
+time.sleep(0.2)
+
+# ---------- initialize llama -----------
+try:
+    print("Initializing Llama with model_path:", model_path)
+    llm = Llama(
+        model_path=model_path,
+        n_ctx=2048,
+        n_threads=4,
+        n_gpu_layers=0,
+    )
+except ValueError as ve:
+    # dump extra debug info for logs and re-raise with context
+    print("Llama init ValueError:", ve)
+    print("Model dir listing at failure:")
+    for fn in sorted(os.listdir(MODEL_DIR)):
+        p = os.path.join(MODEL_DIR, fn)
+        try:
+            st = os.stat(p)
+            print(f" * {p}: size={st.st_size}, mode={oct(st.st_mode)}")
+        except Exception as ex:
+            print(" * stat failed for", p, ex)
+    raise
+
+# ---------- chat utilities ----------
 def build_messages(history, user_message, system_prompt=SYSTEM_PROMPT):
-    """
-    Convert history (list of [user, assistant]) into chat messages format expected by create_chat_completion.
-    Then append the current user_message at the end.
-    """
     messages = []
     if system_prompt:
         messages.append({"role": "system", "content": system_prompt})
-
-    # history is list of [user, assistant] pairs
     for user_msg, assistant_msg in history:
         messages.append({"role": "user", "content": user_msg})
-        if assistant_msg is not None and assistant_msg != "":
+        if assistant_msg:
             messages.append({"role": "assistant", "content": assistant_msg})
-
-    # now add current user message
     messages.append({"role": "user", "content": user_message})
     return messages
 
-# -------------------------
-# Streaming generator for Gradio
-# -------------------------
 def chat_fn(user_message, history):
-    """
-    Gradio ChatInterface expects either a single return (reply string) or a generator that yields partial strings.
-    We'll stream partial assistant text as it arrives from llama-cpp-python create_chat_completion(..., stream=True).
-    """
-    # history is list of [user, assistant] pairs from Gradio
     messages = build_messages(history or [], user_message)
-
-    # create_chat_completion returns an iterator when stream=True
     stream = llm.create_chat_completion(
         messages=messages,
         max_tokens=512,
@@ -113,11 +155,8 @@ def chat_fn(user_message, history):
         top_p=0.95,
         stream=True
     )
-
-    # accumulate incremental content and yield progressive replies
     partial = ""
     for chunk in stream:
-        # chunk structure: {"id":..., "object":"chat.completion.chunk", "choices":[{"delta":{"content": "..."}}, ...]}
         try:
             if "choices" in chunk and len(chunk["choices"]) > 0:
                 delta = chunk["choices"][0].get("delta", {})
@@ -125,17 +164,8 @@ def chat_fn(user_message, history):
                     partial += delta["content"]
                     yield partial
         except Exception:
-            # ignore malformed chunk and continue
             continue
 
-# -------------------------
-# Launch Gradio
-# -------------------------
-demo = gr.ChatInterface(
-    fn=chat_fn,
-    title="EuroLLM 1.7B (GGUF) — streaming chat",
-    description="Model: mradermacher/EuroLLM-1.7B-Instruct (Q8_0). System prompt enabled. Streaming ON.",
-)
-
+demo = gr.ChatInterface(fn=chat_fn, title="EuroLLM 1.7B (robust loader + streaming)")
 if __name__ == "__main__":
     demo.launch()