Update app.py

app.py

@@ -1,10 +1,10 @@
-import os, io, json, zipfile, time, hashlib
-from typing import List, Dict, Any, Optional
+import os, io, json, zipfile, hashlib
+from typing import List, Dict, Any, Optional, Union
 import gradio as gr
-from pydantic import BaseModel, Field
+from pydantic import BaseModel
 from tenacity import retry, stop_after_attempt, wait_exponential
 
-# --- Optional .env support (Space secrets still preferred) ---
+# --- Optional .env support ---
 try:
     from dotenv import load_dotenv
     load_dotenv()
@@ -24,13 +24,43 @@ try:
 except Exception:
     anthropic = None
 
-# Firecrawl official SDK (v2.x)
-# Docs: https://docs.firecrawl.dev/features/search , /scrape , /crawl
-# Example usage shows Firecrawl(api_key).search(..., scrape_options={formats: [...]})
-from firecrawl import Firecrawl  # type: ignore
+# Firecrawl SDK
+from firecrawl import Firecrawl  # v2.x
 
 # --------------------------
-# Session key handling
+# Utilities
+# --------------------------
+def _to_dict(obj: Any) -> Any:
+    """
+    Recursively convert Firecrawl/Pydantic models (SearchData, ScrapeData, CrawlData, etc.)
+    or other objects into plain Python dict/list primitives.
+    """
+    # Pydantic v2 models
+    if isinstance(obj, BaseModel):
+        return obj.model_dump()
+    # Mapping-like
+    if isinstance(obj, dict):
+        return {k: _to_dict(v) for k, v in obj.items()}
+    # List/Tuple
+    if isinstance(obj, (list, tuple)):
+        return [_to_dict(v) for v in obj]
+    # Objects with __dict__ (fallback)
+    if hasattr(obj, "__dict__") and not isinstance(obj, (str, bytes)):
+        try:
+            return {k: _to_dict(v) for k, v in vars(obj).items()}
+        except Exception:
+            pass
+    return obj
+
+def _pretty_json(data: Any, limit: int = 300_000) -> str:
+    try:
+        s = json.dumps(_to_dict(data), indent=2)
+        return s[:limit]
+    except Exception as e:
+        return f"<!> Could not serialize to JSON: {e}"
+
+# --------------------------
+# Session keys
 # --------------------------
 class Keys(BaseModel):
     openai: Optional[str] = None
@@ -38,11 +68,10 @@ class Keys(BaseModel):
     firecrawl: Optional[str] = None
 
 def resolve_keys(session: Keys) -> Keys:
-    """Priority: UI session input > environment variables > None."""
     return Keys(
-        openai = session.openai or os.getenv("OPENAI_API_KEY"),
-        anthropic = session.anthropic or os.getenv("ANTHROPIC_API_KEY"),
-        firecrawl = session.firecrawl or os.getenv("FIRECRAWL_API_KEY"),
+        openai=session.openai or os.getenv("OPENAI_API_KEY"),
+        anthropic=session.anthropic or os.getenv("ANTHROPIC_API_KEY"),
+        firecrawl=session.firecrawl or os.getenv("FIRECRAWL_API_KEY"),
     )
 
 # --------------------------
@@ -51,7 +80,7 @@ def resolve_keys(session: Keys) -> Keys:
 def fc_client(session: Keys) -> Firecrawl:
     keys = resolve_keys(session)
     if not keys.firecrawl:
-        raise gr.Error("Missing FIRECRAWL_API_KEY. Enter it in Keys ➜ Save.")
+        raise gr.Error("Missing FIRECRAWL_API_KEY. Enter it in Keys → Save.")
     return Firecrawl(api_key=keys.firecrawl)
 
 @retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=1, max=8))
@@ -61,9 +90,9 @@ def fc_search(session: Keys, query: str, limit: int = 5, scrape_formats: Optiona
     if location:
         kwargs["location"] = location
     if scrape_formats:
-        # per docs: search(..., scrape_options={"formats": [...]})
         kwargs["scrape_options"] = {"formats": scrape_formats}
-    return fc.search(**kwargs)
+    result = fc.search(**kwargs)   # returns a Pydantic model
+    return _to_dict(result)
 
 @retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=1, max=8))
 def fc_scrape(session: Keys, url: str, formats: Optional[List[str]] = None, timeout_ms: Optional[int] = None, mobile: bool = False) -> Dict[str, Any]:
@@ -75,7 +104,8 @@ def fc_scrape(session: Keys, url: str, formats: Optional[List[str]] = None, time
         kwargs["timeout"] = timeout_ms
     if mobile:
         kwargs["mobile"] = True
-    return fc.scrape(**kwargs)
+    result = fc.scrape(**kwargs)   # Pydantic model
+    return _to_dict(result)
 
 @retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=1, max=8))
 def fc_crawl(session: Keys, url: str, max_pages: int = 25, formats: Optional[List[str]] = None) -> Dict[str, Any]:
@@ -83,11 +113,18 @@ def fc_crawl(session: Keys, url: str, max_pages: int = 25, formats: Optional[Lis
     kwargs: Dict[str, Any] = {"url": url, "limit": max_pages}
     if formats:
         kwargs["scrape_options"] = {"formats": formats}
-    return fc.crawl(**kwargs)
+    result = fc.crawl(**kwargs)    # Pydantic model
+    return _to_dict(result)
 
 # --------------------------
-# LLM helpers (GPT-5 / Claude Sonnet)
+# LLM helpers
 # --------------------------
+SYSTEM_STEER = (
+    "You are ZEN's VibeCoder: extract web insights, generate clean scaffolds, "
+    "and produce production-ready artifacts. Prefer structured outlines, code blocks, and checklists. "
+    "When asked to clone or refactor, output file trees and exact text."
+)
+
 def use_openai(session: Keys):
     keys = resolve_keys(session)
     if not keys.openai:
@@ -104,36 +141,34 @@ def use_anthropic(session: Keys):
         raise gr.Error("Anthropic SDK not installed.")
     return anthropic.Anthropic(api_key=keys.anthropic)
 
-SYSTEM_STEER = (
-    "You are ZEN's VibeCoder: extract web insights, generate clean scaffolds, "
-    "and produce production-ready artifacts. Prefer structured outlines, code blocks, and checklists. "
-    "When asked to clone or refactor, output file trees and exact text."
-)
-
 def llm_summarize(session: Keys, provider: str, model_name: str, prompt: str, context_md: str, temp: float=0.4) -> str:
     if provider == "openai":
         client = use_openai(session)
         resp = client.chat.completions.create(
-            model=model_name,  # e.g., "gpt-5"
+            model=model_name or "gpt-5",
             temperature=temp,
             messages=[
                 {"role": "system", "content": SYSTEM_STEER},
-                {"role": "user", "content": f"{prompt}\n\n=== SOURCE (markdown) ===\n{context_md[:150000]}"},
+                {"role": "user", "content": f"{prompt}\n\n=== SOURCE (markdown) ===\n{(context_md or '')[:150000]}"},
             ],
         )
-        return resp.choices[0].message.content or ""
+        return (resp.choices[0].message.content or "").strip()
     else:
         client = use_anthropic(session)
         resp = client.messages.create(
-            model=model_name,  # e.g., "claude-3-5-sonnet-latest" or "claude-sonnet-4.5"
+            model=model_name or "claude-3-5-sonnet-latest",
             max_tokens=4000,
             temperature=temp,
             system=SYSTEM_STEER,
-            messages=[
-                {"role": "user", "content": f"{prompt}\n\n=== SOURCE (markdown) ===\n{context_md[:150000]}"},
-            ],
+            messages=[{"role": "user", "content": f"{prompt}\n\n=== SOURCE (markdown) ===\n{(context_md or '')[:150000]}"}],
         )
-        return "".join([blk.text for blk in resp.content if hasattr(blk, "text")])
+        chunks = []
+        for blk in resp.content:
+            # anthropic returns TextBlock objects
+            text = getattr(blk, "text", None)
+            if text:
+                chunks.append(text)
+        return "".join(chunks).strip()
 
 # --------------------------
 # ZIP export
@@ -144,18 +179,15 @@ def pack_zip(pages: List[Dict[str, Any]]) -> bytes:
         manifest = []
         for i, p in enumerate(pages, start=1):
             url = p.get("url") or p.get("metadata", {}).get("sourceURL") or f"page_{i}"
-            slug = hashlib.sha1(url.encode("utf-8")).hexdigest()[:10]
+            slug = hashlib.sha1(str(url).encode("utf-8")).hexdigest()[:10]
             md = p.get("markdown") or p.get("data", {}).get("markdown") or p.get("content") or ""
             html = p.get("html") or p.get("data", {}).get("html") or ""
             links = p.get("links") or p.get("data", {}).get("links") or []
-            # write markdown/html if present
             if md:
                 zf.writestr(f"{i:03d}_{slug}.md", md)
             if html:
                 zf.writestr(f"{i:03d}_{slug}.html", html)
-            # lightweight json record
-            record = {"url": url, "title": p.get("title") or p.get("metadata", {}).get("title"), "links": links}
-            manifest.append(record)
+            manifest.append({"url": url, "title": p.get("title") or p.get("metadata", {}).get("title"), "links": links})
         zf.writestr("manifest.json", json.dumps(manifest, indent=2))
     mem.seek(0)
     return mem.read()
@@ -164,36 +196,41 @@ def pack_zip(pages: List[Dict[str, Any]]) -> bytes:
 # Gradio actions
 # --------------------------
 def save_keys(openai_key, anthropic_key, firecrawl_key):
-    # session state object
-    return Keys(openai=openai_key.strip() or None,
-                anthropic=anthropic_key.strip() or None,
-                firecrawl=firecrawl_key.strip() or None), gr.Info("Keys saved to this session. (Env vars still apply if set.)")
+    return Keys(
+        openai=(openai_key or "").strip() or None,
+        anthropic=(anthropic_key or "").strip() or None,
+        firecrawl=(firecrawl_key or "").strip() or None,
+    ), gr.Info("Keys saved to this session. (Env vars still apply if set.)")
 
 def action_search(session: Keys, query: str, limit: int, scrape_content: bool, location: str):
     if not query.strip():
         raise gr.Error("Enter a search query.")
     formats = ["markdown", "links"] if scrape_content else None
-    result = fc_search(session, query=query.strip(), limit=limit, scrape_formats=formats, location=location or None)
-    # Normalize to a flat list of items for display
-    data = result.get("data")
-    items = []
+    result = fc_search(session, query=query.strip(), limit=limit, scrape_formats=formats, location=(location or None))
+    # Firecrawl v2 search returns {'data': {'web': [...], 'news': [...], ...}} or {'data': [...]}
+    data = result.get("data", result)  # tolerate both shapes
+    items: List[Any] = []
     if isinstance(data, dict):
-        # data may contain keys like "web", "news", "images"
-        for bucket in ["web", "news", "images"]:
-            for it in data.get(bucket, []):
-                items.append(it)
+        for bucket in ("web", "news", "images", "videos", "discussion"):
+            items.extend(_to_dict(data.get(bucket, [])))
     elif isinstance(data, list):
-        items = data
+        items = _to_dict(data)
+    else:
+        items = [_to_dict(data)]
     return json.dumps(items, indent=2)
 
 def action_scrape(session: Keys, url: str, mobile: bool, formats_sel: List[str], timeout_ms: int):
     if not url.strip():
         raise gr.Error("Enter a URL.")
     formats = formats_sel or ["markdown", "html", "links"]
-    out = fc_scrape(session, url.strip(), formats=formats, timeout_ms=timeout_ms or None, mobile=mobile)
-    # Firecrawl returns a dict; try to surface key content
-    pretty = json.dumps(out, indent=2)[:300000]
-    md = out.get("markdown") or out.get("data", {}).get("markdown") or ""
+    out = fc_scrape(session, url.strip(), formats=formats, timeout_ms=(timeout_ms or None), mobile=mobile)
+    pretty = _pretty_json(out)
+    md = (
+        out.get("markdown")
+        or out.get("data", {}).get("markdown")
+        or out.get("content")
+        or ""
+    )
     return pretty, md
 
 def action_crawl(session: Keys, base_url: str, max_pages: int, formats_sel: List[str]):
@@ -201,9 +238,8 @@ def action_crawl(session: Keys, base_url: str, max_pages: int, formats_sel: List
         raise gr.Error("Enter a base URL to crawl.")
     formats = formats_sel or ["markdown", "links"]
     out = fc_crawl(session, base_url.strip(), max_pages=max_pages, formats=formats)
-    # Expect out["data"] as list of pages
-    pages = out.get("data") if isinstance(out, dict) else None
-    if not isinstance(pages, list) or len(pages) == 0:
+    pages = out.get("data")
+    if not isinstance(pages, list) or not pages:
         raise gr.Error("Crawl returned no pages.")
     zip_bytes = pack_zip(pages)
     return gr.File.update(value=io.BytesIO(zip_bytes), visible=True, filename="site_clone.zip"), f"Crawled {len(pages)} pages. ZIP is ready."
@@ -211,7 +247,7 @@ def action_crawl(session: Keys, base_url: str, max_pages: int, formats_sel: List
 def action_generate(session: Keys, provider: str, model_name: str, sys_prompt: str, user_prompt: str, context_md: str, temp: float):
     if not user_prompt.strip():
         raise gr.Error("Enter a prompt or click a starter tile.")
-    model = model_name.strip() or ("gpt-5" if provider == "openai" else "claude-3-5-sonnet-latest")
+    model = (model_name or "").strip() or ("gpt-5" if provider == "openai" else "claude-3-5-sonnet-latest")
     steer = (sys_prompt or "").strip()
     prompt = (("SYSTEM:\n" + steer + "\n\n") if steer else "") + user_prompt.strip()
     out = llm_summarize(session, provider, model, prompt, context_md or "", temp=temp)
@@ -220,11 +256,9 @@ def action_generate(session: Keys, provider: str, model_name: str, sys_prompt: s
 # --------------------------
 # UI (Blocks)
 # --------------------------
-with gr.Blocks(css="""
-#keys .wrap.svelte-1ipelgc { filter: none !important; }
-""") as demo:
+with gr.Blocks(css="#keys .wrap.svelte-1ipelgc { filter: none !important; }") as demo:
     gr.Markdown("## ZEN VibeCoder — Web Clone & Research Foundry")
-    session_state = gr.State(Keys())  # holds keys
+    session_state = gr.State(Keys())
 
     with gr.Accordion("🔐 Keys (session)", open=True):
         with gr.Row():
@@ -236,7 +270,6 @@ with gr.Blocks(css="""
         save_btn.click(save_keys, [openai_key, anthropic_key, firecrawl_key], [session_state, save_msg])
 
     with gr.Tabs():
-        # --- TAB: Search ---
         with gr.Tab("🔎 Search"):
             query = gr.Textbox(label="Query", placeholder='ex: "best open-source vector databases in 2025 site:docs"')
             with gr.Row():
@@ -247,7 +280,6 @@ with gr.Blocks(css="""
             search_json = gr.Code(label="Results JSON", language="json")
             go_search.click(action_search, [session_state, query, limit, scrape_content, location], [search_json])
 
-        # --- TAB: Scrape / Crawl / Clone ---
         with gr.Tab("🕸️ Scrape • Crawl • Clone"):
             with gr.Row():
                 target_url = gr.Textbox(label="URL to Scrape", placeholder="https://example.com")
@@ -271,7 +303,6 @@ with gr.Blocks(css="""
             crawl_status = gr.Markdown()
             run_crawl.click(action_crawl, [session_state, base_url, max_pages, formats_crawl], [zip_file, crawl_status])
 
-        # --- TAB: Vibe Code (LLM Synthesis) ---
         with gr.Tab("✨ Vibe Code (Synthesis)"):
             with gr.Row():
                 provider = gr.Radio(choices=["openai","anthropic"], value="openai", label="Provider")
@@ -287,7 +318,6 @@ with gr.Blocks(css="""
                 gen_btn = gr.Button("Generate", variant="primary")
                 out_md = gr.Markdown()
 
-            # Starter Tiles
             gr.Markdown("**Starter Tiles**")
             with gr.Row():
                 t1 = gr.Button("🔧 Clone Docs ➜ Clean Markdown ➜ README")
@@ -297,14 +327,14 @@ with gr.Blocks(css="""
                 t5 = gr.Button("📊 Dataset Outline ➜ Schema + Fields + ETL")
 
             def fill_tile(tile: str):
-                map_ = {
-                    "t1": ("Create a clean knowledge pack from the context, then output a README.md with:\n- Overview\n- Key features\n- Quickstart\n- API endpoints (if any)\n- Notes & gotchas\n- License\nAlso produce a /docs/ tree outline with suggested pages and headings.",),
-                    "t2": ("From the context, produce a feature matrix, pricing table, ICP notes, moats/risks, and a market POV. Conclude with a ZEN playbook: 5 lever moves for advantage.",),
-                    "t3": ("Using the context, design a Python client that wraps the target API with retry/backoff and typed responses. Output:\n- package layout\n- requirements\n- client.py\n- examples/\n- README with usage.\nInclude robust error handling.",),
-                    "t4": ("Rewrite the landing page in ZEN brand voice: crisp headline, 3 value props, social proof, CTA, and a concise FAQ. Provide HTML sections and copy blocks.",),
-                    "t5": ("Propose a dataset schema based on the context. Output a table of fields, types, constraints, and an ETL plan (sources, transforms, validation, freshness, monitoring).",),
+                tiles = {
+                    "t1": "Create a clean knowledge pack from the context, then output a README.md with:\n- Overview\n- Key features\n- Quickstart\n- API endpoints (if any)\n- Notes & gotchas\n- License\nAlso produce a /docs/ tree outline with suggested pages and headings.",
+                    "t2": "From the context, produce a feature matrix, pricing table, ICP notes, moats/risks, and a market POV. Conclude with a ZEN playbook: 5 lever moves for advantage.",
+                    "t3": "Using the context, design a Python client that wraps the target API with retry/backoff and typed responses. Output:\n- package layout\n- requirements\n- client.py\n- examples/\n- README with usage.\nInclude robust error handling.",
+                    "t4": "Rewrite the landing page in ZEN brand voice: crisp headline, 3 value props, social proof, CTA, and a concise FAQ. Provide HTML sections and copy blocks.",
+                    "t5": "Propose a dataset schema based on the context. Output a table of fields, types, constraints, and an ETL plan (sources, transforms, validation, freshness, monitoring).",
                 }
-                return map_[tile][0]
+                return tiles[tile]
 
             t1.click(lambda: fill_tile("t1"), outputs=[user_prompt])
             t2.click(lambda: fill_tile("t2"), outputs=[user_prompt])
@@ -314,11 +344,8 @@ with gr.Blocks(css="""
 
             gen_btn.click(action_generate, [session_state, provider, model_name, sys_prompt, user_prompt, ctx_md, temp], [out_md])
 
-    gr.Markdown(
-        "Built for **ZEN Arena** pipelines. Export ZIPs → ingest → credentialize achievements via ZEN Cards.\n"
-        "Docs used for Firecrawl behavior: search/scrape/crawl endpoints."
-    )
+    gr.Markdown("Built for **ZEN Arena** pipelines. Export ZIPs → ingest → credentialize via ZEN Cards.")
 
 if __name__ == "__main__":
-    # Spaces expects 'app' or launch() with default port
+    # If SSR causes issues in your Space, you can disable it by: demo.launch(ssr_mode=False)
     demo.launch()