Update app/services/reviewer_service.py

app/services/reviewer_service.py

@@ -4,25 +4,22 @@ import re
 import hashlib
 import sqlite3
 import threading
-from typing import List, Dict
-from concurrent.futures import ThreadPoolExecutor, as_completed
+from typing import List
 from app.core.model_loader import llm_engine
 
 logger = logging.getLogger(__name__)
-
 DB_PATH = "review_cache.db"
 
 class AIReviewerService:
     def __init__(self):
-        self.max_concurrent_batches = 3
-        self.batch_size = 10
+        self.batch_size = 3  # Reduced from 10 to speed up individual prompts
         self.db_lock = threading.Lock()
         self._init_db()
 
     def _init_db(self):
-        """Initializes SQLite cache table."""
+        """Initializes SQLite cache table with proper timeout."""
         with self.db_lock:
-            with sqlite3.connect(DB_PATH) as conn:
+            with sqlite3.connect(DB_PATH, timeout=30.0) as conn:
                 cursor = conn.cursor()
                 cursor.execute("""
                     CREATE TABLE IF NOT EXISTS review_cache (
@@ -33,15 +30,13 @@ class AIReviewerService:
                 conn.commit()
 
     def _generate_file_hash(self, file_obj) -> str:
-        """Generates a unique SHA256 digest based on path and content."""
         content = file_obj.content or ""
         raw_payload = f"{file_obj.fileName}:{content}"
         return hashlib.sha256(raw_payload.encode("utf-8")).hexdigest()
 
     def _get_cached_result(self, file_hash: str) -> dict:
-        """Thread-safe SQLite fetch."""
         with self.db_lock:
-            with sqlite3.connect(DB_PATH) as conn:
+            with sqlite3.connect(DB_PATH, timeout=30.0) as conn:
                 cursor = conn.cursor()
                 cursor.execute("SELECT result_json FROM review_cache WHERE file_hash = ?", (file_hash,))
                 row = cursor.fetchone()
@@ -53,11 +48,10 @@ class AIReviewerService:
         return None
 
     def _save_cached_results(self, hash_result_pairs: List[tuple]):
-        """Thread-safe batch insert into SQLite."""
         if not hash_result_pairs:
             return
         with self.db_lock:
-            with sqlite3.connect(DB_PATH) as conn:
+            with sqlite3.connect(DB_PATH, timeout=30.0) as conn:
                 cursor = conn.cursor()
                 cursor.executemany(
                     "INSERT OR REPLACE INTO review_cache (file_hash, result_json) VALUES (?, ?)",
@@ -87,41 +81,32 @@ class AIReviewerService:
             logger.info(f"⚡ Served all {len(files)} files instantly from SQLite cache.")
             return results
 
-        logger.info(f"SQLite cache hit: {len(files) - len(uncached_files)}/{len(files)} files. Reviewing {len(uncached_files)} uncached files.")
-
-        # 2. Batch process uncached files via Gemini
+        # 2. Sequential batch processing (prevents LLM thread lock contention)
         batches = [
             uncached_files[i : i + self.batch_size]
             for i in range(0, len(uncached_files), self.batch_size)
         ]
-
+        
         new_results = []
-        with ThreadPoolExecutor(max_workers=self.max_concurrent_batches) as pool:
-            future_to_batch = {
-                pool.submit(self._process_batch, batch): batch for batch in batches
-            }
-            for future in as_completed(future_to_batch):
-                batch = future_to_batch[future]
-                try:
-                    batch_results = future.result()
-                    new_results.extend(batch_results)
-                except Exception as e:
-                    logger.error(f"Batch failure: {e}")
-                    for f_item in batch:
-                        new_results.append(
-                            {
-                                "fileName": f_item.fileName,
-                                "vulnerabilities": [],
-                                "improvement_suggestions": [],
-                                "metrics": {"complexity": 0, "maintainability": 0},
-                                "error": "analysis_failed",
-                            }
-                        )
+        for batch in batches:
+            try:
+                batch_results = self._process_batch(batch)
+                new_results.extend(batch_results)
+            except Exception as e:
+                logger.error(f"Batch processing error: {e}")
+                for f_item in batch:
+                    new_results.append({
+                        "fileName": f_item.fileName,
+                        "vulnerabilities": [],
+                        "improvement_suggestions": [],
+                        "metrics": {"complexity": 0, "maintainability": 0},
+                        "error": "analysis_failed",
+                    })
 
         new_results_map = {item.get("fileName"): item for item in new_results}
         to_cache = []
 
-        # 3. Assemble final response list and write to SQLite
+        # 3. Assemble final response and update cache
         for idx, f in zip(uncached_indices, uncached_files):
             res = new_results_map.get(
                 f.fileName,
@@ -133,11 +118,9 @@ class AIReviewerService:
                     "error": "analysis_failed",
                 },
             )
-
             if "error" not in res:
                 file_hash = self._generate_file_hash(f)
                 to_cache.append((file_hash, res))
-
             results[idx] = res
 
         self._save_cached_results(to_cache)
@@ -148,43 +131,27 @@ class AIReviewerService:
         file_names = []
         for f in batch:
             raw_content = f.content or ""
-            minified_content = self._minify_code(raw_content[:5000], f.fileName)
+            minified_content = self._minify_code(raw_content[:2000], f.fileName)  # Reduced preview limit to 2000 chars
             combined_code += f"\n--- FILE: {f.fileName} ---\n{minified_content}\n"
             file_names.append(f.fileName)
 
-        prompt = f"""Analyze these {len(batch)} source files for vulnerabilities and quality.
+        prompt = f"""Analyze {len(batch)} files. Be ultra-concise.
 {combined_code}
-Instructions:
-1. Identify severe security/logic issues as 'vulnerabilities' with a 'suggestion' fix.
-2. If no vulnerabilities exist, give 1-2 concise 'improvement_suggestions'.
-3. Assign maintainability and complexity metrics (1-10).
-4. Keep descriptions short to maintain fast execution.
-
-Return JSON array (1 object per file):
+Return ONLY valid JSON array:
 [
   {{
-    "fileName": "exact/path/from/header",
-    "vulnerabilities": [
-      {{
-        "type": "SQLi/Logic/etc",
-        "line": 10,
-        "description": "Short explanation",
-        "suggestion": "Specific fix"
-      }}
-    ],
-    "improvement_suggestions": ["Tip 1"],
+    "fileName": "name",
+    "vulnerabilities": [{{"type": "type", "line": 1, "description": "short", "suggestion": "fix"}}],
+    "improvement_suggestions": ["tip"],
     "metrics": {{"complexity": 3, "maintainability": 8}}
   }}
 ]"""
-
         batch_results = []
         try:
-            response_text = llm_engine.generate(prompt, max_tokens=4096)
+            # Capped token length to 1024 max per batch turn
+            response_text = llm_engine.generate(prompt, max_tokens=1024)
             parsed = self._parse_json(response_text)
-
-            processed_map = {
-                item.get("fileName"): item for item in parsed if isinstance(item, dict)
-            }
+            processed_map = {item.get("fileName"): item for item in parsed if isinstance(item, dict)}
 
             for fn in file_names:
                 if fn in processed_map:
@@ -192,44 +159,31 @@ Return JSON array (1 object per file):
                     res.setdefault("vulnerabilities", [])
                     res.setdefault("improvement_suggestions", [])
                     res.setdefault("metrics", {"complexity": 5, "maintainability": 5})
-
-                    for vuln in res["vulnerabilities"]:
-                        if "suggestion" not in vuln:
-                            vuln["suggestion"] = "Review implementation logic for safety."
                     batch_results.append(res)
                 else:
-                    batch_results.append(
-                        {
-                            "fileName": fn,
-                            "vulnerabilities": [],
-                            "improvement_suggestions": ["No immediate improvements identified."],
-                            "metrics": {"complexity": 1, "maintainability": 10},
-                        }
-                    )
-        except Exception as e:
-            logger.error(f"Batch processing error: {e}")
-            error_str = str(e)
-            is_config_error = any(
-                x in error_str
-                for x in ["QUOTA_EXCEEDED", "RATE_LIMIT_EXCEEDED", "MISSING_API_KEY", "GENERATION_FAILED"]
-            )
-            for fn in file_names:
-                batch_results.append(
-                    {
+                    batch_results.append({
                         "fileName": fn,
                         "vulnerabilities": [],
                         "improvement_suggestions": [],
-                        "metrics": {"complexity": 0, "maintainability": 0},
-                        "error": error_str if is_config_error else "analysis_failed",
-                    }
-                )
+                        "metrics": {"complexity": 1, "maintainability": 10},
+                    })
+        except Exception as e:
+            logger.error(f"Batch processing error: {e}")
+            for fn in file_names:
+                batch_results.append({
+                    "fileName": fn,
+                    "vulnerabilities": [],
+                    "improvement_suggestions": [],
+                    "metrics": {"complexity": 0, "maintainability": 0},
+                    "error": str(e),
+                })
         return batch_results
 
     def _minify_code(self, code: str, filename: str) -> str:
         lines = code.split("\n")
         cleaned_lines = []
         is_python = filename.endswith(".py")
-        is_js_style = filename.endswith((".js", ".ts", ".jsx", ".tsx", ".java", ".kt", ".c", ".cpp"))
+        is_js_style = filename.endswith((".js", ".ts", ".jsx", ".tsx"))
 
         for line in lines:
             stripped = line.strip()
@@ -239,28 +193,20 @@ Return JSON array (1 object per file):
                 continue
             if is_js_style and stripped.startswith(("//", "/*", "*")):
                 continue
-            if any(log in stripped for log in ["console.log", "print(", "logger.", "Log.d", "Log.e"]):
-                continue
-            if is_python and " #" in line:
-                line = line.split(" #", 1)[0]
-            if is_js_style and " //" in line:
-                line = line.split(" //", 1)[0]
-            if line.strip():
-                cleaned_lines.append(line.rstrip())
+            cleaned_lines.append(stripped)
 
-        return "\n".join(cleaned_lines)
+        return "\n".join(cleaned_lines[:50])  # Cap at 50 lines max per file
 
     def _parse_json(self, text: str):
         text = text.strip()
         if not text:
             return []
-        text = re.sub(r"^[^[]*\[", "[", text)
-        text = re.sub(r"\][^]]*$", "]", text)
+        text = re.sub(r"^[^\[]*\[", "[", text)
+        text = re.sub(r"\][^\]]*$", "]", text)
         text = text.replace("```json", "").replace("```", "").strip()
         try:
             return json.loads(text)
-        except json.JSONDecodeError as e:
-            logger.warning(f"JSON Decode Error: {e}")
+        except json.JSONDecodeError:
             return []
 
 service = AIReviewerService()