[Edit]

llm.py

@@ -31,10 +31,16 @@ def extract_https_links(chunks):
 def fetch_all_links(links, timeout=10, max_workers=10):
     """
     Fetch all HTTPS links in parallel, with per-link timing.
+    Skips banned links.
     Returns a dict {link: content or error}.
     """
     fetched_data = {}
 
+    # Internal banned list
+    banned_links = [
+     
+    ]
+
     def fetch(link):
         start = time.perf_counter()
         try:
@@ -48,38 +54,42 @@ def fetch_all_links(links, timeout=10, max_workers=10):
             print(f"❌ {link} — {elapsed:.2f}s — ERROR: {e}")
             return link, f"ERROR: {e}"
 
-    # Fetch all links in parallel (no banned filtering, no special prioritization)
+    # Filter out banned links before starting fetch
+    links_to_fetch = [l for l in links if l not in banned_links]
+    for banned in set(links) - set(links_to_fetch):
+        print(f"⛔ Skipped banned link: {banned}")
+        fetched_data[banned] = "BANNED"
+
     t0 = time.perf_counter()
     with ThreadPoolExecutor(max_workers=max_workers) as executor:
-        future_to_link = {executor.submit(fetch, link): link for link in links}
+        future_to_link = {executor.submit(fetch, link): link for link in links_to_fetch}
         for future in as_completed(future_to_link):
             link, content = future.result()
             fetched_data[link] = content
-
     print(f"[TIMER] Total link fetching: {time.perf_counter() - t0:.2f}s")
+    print(fetched_data)
     return fetched_data
 
-
 def query_gemini(questions, contexts, max_retries=3):
     import itertools
 
     total_start = time.perf_counter()
 
-    # Join context & questions fresh every call, no caching
+    # Context join
     t0 = time.perf_counter()
     context = "\n\n".join(contexts)
     questions_text = "\n".join([f"{i+1}. {q}" for i, q in enumerate(questions)])
     print(f"[TIMER] Context join: {time.perf_counter() - t0:.2f}s")
 
-    # Extract links and fetch all links, with special URL prioritized
+    # Link extraction & fetching
     links = extract_https_links(contexts)
     if links:
         fetched_results = fetch_all_links(links)
         for link, content in fetched_results.items():
-            if not content.startswith("ERROR") and content != "BANNED":
+            if not content.startswith("ERROR"):
                 context += f"\n\nRetrieved from {link}:\n{content}"
 
-    # Build prompt fresh each time
+    # Prompt building
     t0 = time.perf_counter()
     prompt = fr"""
 - You are an expert insurance assistant generating formal yet user-facing answers to policy questions and Other Human Questions. Your goal is to write professional, structured answers that reflect the language of policy documents — but are still human-readable and easy to understand.
@@ -150,12 +160,14 @@ Respond with only the following JSON — no explanations, no comments, no markdo
 
 
 """
+
     print(f"[TIMER] Prompt build: {time.perf_counter() - t0:.2f}s")
 
     last_exception = None
     total_attempts = len(api_keys) * max_retries
     key_cycle = itertools.cycle(api_keys)
 
+    # Gemini API calls
     for attempt in range(total_attempts):
         key = next(key_cycle)
         try:
@@ -166,14 +178,15 @@ Respond with only the following JSON — no explanations, no comments, no markdo
             api_time = time.perf_counter() - t0
             print(f"[TIMER] Gemini API call (attempt {attempt+1}): {api_time:.2f}s")
 
+            # Response parsing
             t0 = time.perf_counter()
             response_text = getattr(response, "text", "").strip()
             if not response_text:
                 raise ValueError("Empty response received from Gemini API.")
 
-            if response_text.startswith("json"):
-                response_text = response_text.replace("json", "").replace("", "").strip()
-            elif response_text.startswith(""):
+            if response_text.startswith("```json"):
+                response_text = response_text.replace("```json", "").replace("```", "").strip()
+            elif response_text.startswith("```"):
                 response_text = response_text.replace("```", "").strip()
 
             parsed = json.loads(response_text)