changed app to integrate verilogagent

app.py

@@ -1,7 +1,311 @@
+import os
+import shutil
+from pathlib import Path
 import gradio as gr
 
-def greet(name):
-    return "Hello " + name + "!!"
+from huggingface_hub import snapshot_download
+from langchain_community.vectorstores import FAISS
+from langchain_huggingface import HuggingFaceEmbeddings
 
-demo = gr.Interface(fn=greet, inputs="text", outputs="text")
-demo.launch()
+# ---- Your class from the prompt (unchanged except 1 tweak: import path safety) ----
+import re
+from pathlib import Path as _Path
+try:
+    from openai import OpenAI
+except ImportError:
+    OpenAI = None
+
+class VerilogAgent:
+    """
+    A self-contained agent for generating Verilog code using API-based LLMs (e.g., GPT)
+    and a RAG pipeline.
+    """
+    def __init__(self, model_id, embedding_id, faiss_index_path, api_key):
+        self.model_id = model_id
+        self.embedding_id = embedding_id
+        self.faiss_index_path = faiss_index_path
+        self.api_key = api_key
+        
+        print(f"[INFO] Initializing VerilogAgent for model: {self.model_id}")
+        self._load_dependencies()
+
+    def _load_dependencies(self):
+        print(f"[INFO] Loading embedding model '{self.embedding_id}'...")
+        embedding = HuggingFaceEmbeddings(model_name=self.embedding_id)
+
+        print(f"[INFO] Loading FAISS vector store from '{self.faiss_index_path}'...")
+        if not _Path(self.faiss_index_path).exists():
+            raise FileNotFoundError(f"FAISS index directory not found at {self.faiss_index_path}.")
+        self.vectorstore = FAISS.load_local(self.faiss_index_path, embedding, allow_dangerous_deserialization=True)
+        
+        if not OpenAI:
+            raise ImportError("OpenAI library is not installed. Please add 'openai' to requirements.txt.")
+        if not self.api_key:
+            raise ValueError("OpenAI API key is required.")
+        self.client = OpenAI(api_key=self.api_key)
+        print("[INFO] OpenAI client initialized.")
+        print("[INFO] VerilogAgent initialized successfully.")
+
+    def _build_prompt_messages_rag(self, query: str, docs: list = None) -> list:
+        context_section = ""
+        if docs:
+            context = "\n\n".join([doc.page_content for doc in docs])
+            context_section = f"""
+CONTEXT EXAMPLES:
+```verilog
+{context}
+```"""
+
+        system_prompt = """You are an expert Verilog code generation assistant.
+TASK: Generate fully implemented, syntactically correct Verilog code in response to the user request.
+
+INSTRUCTIONS:
+1. Analyze the user request and the provided context examples to determine the required modules and logic.
+2. The context provides examples of valid, complete Verilog modules.
+3. Implement all required modules. Every `module ... endmodule` block must be complete.
+4. Do not leave logic empty or use placeholders like `// your code here`.
+5. Your entire response MUST be only the Verilog code, wrapped in a single `verilog` markdown block. Do not include any natural language explanations.
+6. Use only Verilog-2005 syntax. Do not use SystemVerilog constructs (e.g., `logic`, `always_ff`).
+7. Ensure all identifiers are declared before use and vector ranges are ordered [MSB:LSB].
+"""
+        user_prompt = f'''TASK: Generate a fully implemented, syntactically correct Verilog module named 'TopModule'. This name is a strict requirement.
+{context_section}
+
+USER REQUEST:
+"""{query}"""
+
+OUTPUT:
+Generate the complete Verilog code.
+'''
+        return [
+            {"role": "system", "content": system_prompt},
+            {"role": "user", "content": user_prompt}
+        ]
+
+    def _build_prompt_messages_baseline(self, query: str, docs: list = None) -> list:
+        context_section = ""
+        if docs:
+            context = "\n\n".join([doc.page_content for doc in docs])
+            context_section = f"""
+CONTEXT EXAMPLES:
+```verilog
+{context}
+```"""
+        system_prompt = "You are an expert Verilog code generation assistant."
+        user_prompt = f'''TASK: Generate a fully implemented, syntactically correct Verilog module named 'TopModule'. This name is a strict requirement.
+{context_section}
+
+USER REQUEST:
+"""{query}"""
+
+OUTPUT:
+Generate the complete Verilog code.
+'''
+        return [
+            {"role": "system", "content": system_prompt},
+            {"role": "user", "content": user_prompt}
+        ]
+
+    def _extract_verilog_code(self, text: str) -> str:
+        verilog_pattern = re.compile(r"```(?:verilog\s*)?(.*?)\s*```", re.DOTALL)
+        match = verilog_pattern.search(text)
+        if match:
+            return match.group(1).strip()
+        module_pattern = re.compile(r"(module.*?endmodule)", re.DOTALL)
+        match = module_pattern.search(text)
+        if match:
+            return match.group(1).strip()
+        return text.strip()
+
+    def _call_api(self, messages: list, generation_params: dict) -> str:
+        try:
+            api_params = {
+                "model": self.model_id,
+                "messages": messages,
+                "max_tokens": generation_params.get("max_new_tokens"),
+                "temperature": generation_params.get("temperature"),
+                "top_p": generation_params.get("top_p")
+            }
+            if "gpt-5" in self.model_id and "verbosity" in generation_params:
+                api_params["verbosity"] = generation_params["verbosity"]
+
+            completion = self.client.chat.completions.create(**api_params)
+            return self._extract_verilog_code(completion.choices[0].message.content)
+        except Exception as e:
+            print(f"[ERROR] Code generation failed for model {self.model_id}: {e}")
+            return f"// ERROR: Generation failed. Details: {e}"
+
+    def generate_with_context(self, spec: str, docs_with_scores: list, generation_params: dict) -> str:
+        relevant_docs = [doc for doc, score in docs_with_scores]
+        messages = self._build_prompt_messages_rag(spec, relevant_docs)
+        return self._call_api(messages, generation_params)
+
+    def generate_baseline(self, spec: str, generation_params: dict) -> str:
+        messages = self._build_prompt_messages_baseline(spec, docs=[])
+        return self._call_api(messages, generation_params)
+
+
+# --------------------------- Space wiring below ---------------------------
+
+# Where we’ll place the FAISS index on disk after downloading from your private dataset:
+CACHE_DIR = Path("/data/faiss_index")   # Spaces ephemeral storage
+CACHE_DIR.mkdir(parents=True, exist_ok=True)
+
+# Env vars you’ll set in the Space “Settings → Repository secrets”
+HF_TOKEN = os.getenv("HF_TOKEN")  # personal access token with read permission
+PRIVATE_DATASET_ID = os.getenv("PRIVATE_DATASET_ID")  # e.g. "yourname/verilog-faiss-index"
+INDEX_SUBDIR = os.getenv("INDEX_SUBDIR", "faiss_index")  # optional subdir within the dataset snapshot
+EMBEDDING_MODEL = os.getenv("EMBEDDING_MODEL", "sentence-transformers/all-MiniLM-L6-v2")
+
+def ensure_index_downloaded() -> Path:
+    """
+    Downloads your private dataset (FAISS index + artifacts) once per container.
+    Avoids committing large binaries to the public Space repo.
+    """
+    target = CACHE_DIR / INDEX_SUBDIR
+    if target.exists() and any(target.iterdir()):
+        print(f"[INFO] Using cached FAISS index at {target}")
+        return target
+
+    if not HF_TOKEN:
+        raise RuntimeError("Missing HF_TOKEN secret. Add it in the Space settings.")
+    if not PRIVATE_DATASET_ID:
+        raise RuntimeError("Missing PRIVATE_DATASET_ID secret (e.g., 'user/private-faiss').")
+
+    print(f"[INFO] Downloading private dataset: {PRIVATE_DATASET_ID}")
+    snapshot_path = snapshot_download(
+        repo_id=PRIVATE_DATASET_ID,
+        repo_type="dataset",
+        token=HF_TOKEN,
+        local_dir=str(CACHE_DIR),
+        local_dir_use_symlinks=False,   # safer for FAISS
+    )
+    # If your index files live under a folder inside the dataset, move/point to it
+    candidate = Path(snapshot_path) / INDEX_SUBDIR
+    if candidate.exists():
+        print(f"[INFO] Found index subdir at {candidate}")
+        return candidate
+
+    # Otherwise assume snapshot root contains the index
+    print(f"[WARN] INDEX_SUBDIR='{INDEX_SUBDIR}' not found; using snapshot root.")
+    return Path(snapshot_path)
+
+# Keep a lightweight global cache so we don’t reload embeddings on every click
+_VECTORSTORE_PATH = None
+
+def get_vectorstore_path() -> Path:
+    global _VECTORSTORE_PATH
+    if _VECTORSTORE_PATH is None:
+        _VECTORSTORE_PATH = ensure_index_downloaded()
+    return _VECTORSTORE_PATH
+
+def run_generation(spec, use_rag, top_k, model_choice, api_key, temperature, top_p, max_new_tokens):
+    if not spec or not api_key:
+        return "// Please provide a design specification and your API key.", "", []
+
+    # Prepare agent
+    faiss_path = get_vectorstore_path()
+    try:
+        agent = VerilogAgent(
+            model_id=model_choice,
+            embedding_id=EMBEDDING_MODEL,
+            faiss_index_path=str(faiss_path),
+            api_key=api_key.strip()
+        )
+    except Exception as e:
+        return f"// Initialization error: {e}", "", []
+
+    # Retrieval (if enabled)
+    docs_with_scores = []
+    retrieved_preview = []
+    if use_rag:
+        try:
+            # similarity_search_with_score returns list[(Document, score)]
+            docs_with_scores = agent.vectorstore.similarity_search_with_score(spec, k=top_k)
+            for doc, score in docs_with_scores:
+                src = doc.metadata.get("source_file", doc.metadata.get("module", "unknown"))
+                retrieved_preview.append(f"{src}  |  score={score:.4f}")
+        except Exception as e:
+            return f"// Retrieval error: {e}", "", []
+
+    # Call model
+    gen_params = {
+        "temperature": float(temperature),
+        "top_p": float(top_p),
+        "max_new_tokens": int(max_new_tokens),
+    }
+
+    if use_rag:
+        code = agent.generate_with_context(spec, docs_with_scores, gen_params)
+    else:
+        code = agent.generate_baseline(spec, gen_params)
+
+    # Clean presentation
+    verilog_block = code.strip()
+    # Show the first few chars of the retrieved examples (for transparency)
+    return verilog_block, ("\n".join(retrieved_preview) if retrieved_preview else ""), [d[0].page_content for d in docs_with_scores]
+
+
+with gr.Blocks(title="DeepRAG for RTL (Model-Agnostic)") as demo:
+    gr.Markdown("## DeepRAG for RTL Code Generation — Model-Agnostic (Bring Your Own API Key)")
+
+    with gr.Row():
+        with gr.Column(scale=2):
+            spec = gr.Textbox(
+                label="Design Specification (natural language or I/O contract)",
+                placeholder="e.g., 8-bit UART transmitter with baud rate generator ...",
+                lines=10
+            )
+            with gr.Row():
+                use_rag = gr.Checkbox(value=True, label="Use Retrieval (RAG)")
+                top_k = gr.Slider(1, 10, value=3, step=1, label="Top-K retrieved examples")
+
+            with gr.Row():
+                model_choice = gr.Dropdown(
+                    choices=[
+                        "gpt-4o",
+                        "gpt-4o-mini",
+                        "gpt-4.1",
+                        "gpt-5",        # hypothetical/future-ready
+                        "gpt-5-mini"    # hypothetical/future-ready
+                    ],
+                    value="gpt-4o",
+                    label="Model"
+                )
+                api_key = gr.Textbox(label="OpenAI API Key", type="password", placeholder="sk-...")
+
+            with gr.Accordion("Generation Settings", open=False):
+                temperature = gr.Slider(0.0, 1.5, value=0.2, step=0.05, label="Temperature")
+                top_p = gr.Slider(0.1, 1.0, value=0.9, step=0.05, label="Top-p")
+                max_new_tokens = gr.Slider(128, 4096, value=768, step=64, label="Max tokens")
+
+            run_btn = gr.Button("Generate Verilog", variant="primary")
+
+        # Right side: code output + retrieval transparency
+        with gr.Column(scale=3):
+            gr.Markdown("**Output**")
+            out_code = gr.Code(
+                label="Generated Verilog (copy-ready)",
+                language="verilog",
+                interactive=False,
+                show_copy_button=True,
+                lines=28
+            )
+            with gr.Tab("Retrieved Items (names + scores)"):
+                retrieved_list = gr.Textbox(
+                    label="Retriever summary",
+                    lines=8,
+                    interactive=False
+                )
+            with gr.Tab("Preview of Retrieved Context (raw)"):
+                # shows the raw text of retrieved docs for transparency (not downloadable)
+                retrieved_raw = gr.HighlightedText(label="(first K documents)", combine_adjacent=True)
+
+    run_btn.click(
+        fn=run_generation,
+        inputs=[spec, use_rag, top_k, model_choice, api_key, temperature, top_p, max_new_tokens],
+        outputs=[out_code, retrieved_list, retrieved_raw]
+    )
+
+if __name__ == "__main__":
+    demo.launch()