"""LLM-based code quality checks.""" from pathlib import Path from typing import Any import anthropic import openai from shared.config import settings from shared.logger import setup_logger logger = setup_logger(__name__) class LLMChecker: """Perform LLM-based quality checks.""" def __init__(self) -> None: """Initialize LLM checker.""" self.provider = settings.llm_provider self.model = settings.llm_model if self.provider == "anthropic": self.client = anthropic.Anthropic(api_key=settings.anthropic_api_key) elif self.provider == "openai": self.client = openai.OpenAI(api_key=settings.openai_api_key) elif self.provider == "aipipe": # Use OpenAI client with AIPipe endpoints self.client = openai.OpenAI( api_key=settings.aipipe_token, base_url=settings.aipipe_base_url ) logger.info(f"Initialized LLMChecker with {self.provider}/{self.model}") def check_readme_quality(self, readme_path: Path) -> dict[str, Any]: """Evaluate README.md quality using LLM. Args: readme_path: Path to README.md Returns: Check result """ try: if not readme_path.exists(): return { "passed": False, "score": 0.0, "reason": "README.md not found", } content = readme_path.read_text() prompt = f"""Evaluate this README.md for quality. Rate it on a scale of 0.0 to 1.0. Criteria: - Has a clear title and description - Explains setup/installation - Explains usage - Has code examples or explanations - Is well-formatted and professional - Mentions license README content: {content} Respond with ONLY a JSON object in this format: {{ "score": 0.85, "reason": "Clear title and good structure, but missing detailed setup instructions" }}""" response = self._call_llm(prompt) # Parse JSON from response import json import re json_match = re.search(r"\{.*\}", response, re.DOTALL) if json_match: result = json.loads(json_match.group(0)) score = float(result.get("score", 0.0)) reason = result.get("reason", "No reason provided") return { "passed": score >= 0.7, "score": score, "reason": reason, } return { "passed": False, "score": 0.0, "reason": "Could not parse LLM response", } except Exception as e: logger.error(f"Error in README quality check: {e}") return {"passed": False, "score": 0.0, "reason": f"Error: {e}"} def check_code_quality(self, code_dir: Path) -> dict[str, Any]: """Evaluate code quality using LLM. Args: code_dir: Directory containing code Returns: Check result """ try: # Gather code files code_files = {} for ext in [".html", ".js", ".css"]: for file in code_dir.rglob(f"*{ext}"): if ".git" not in str(file): rel_path = file.relative_to(code_dir) code_files[str(rel_path)] = file.read_text()[:2000] # Limit size if not code_files: return { "passed": False, "score": 0.0, "reason": "No code files found", } # Format code for LLM code_text = "\n\n".join( f"=== {name} ===\n{content}" for name, content in code_files.items() ) prompt = f"""Evaluate this web application code for quality. Rate it on a scale of 0.0 to 1.0. Criteria: - Code is clean and well-organized - Proper use of HTML semantics - Good JavaScript practices - Reasonable styling - Comments where helpful - No obvious bugs or security issues Code: {code_text} Respond with ONLY a JSON object in this format: {{ "score": 0.8, "reason": "Clean code with good structure, minor improvements possible" }}""" response = self._call_llm(prompt) # Parse JSON import json import re json_match = re.search(r"\{.*\}", response, re.DOTALL) if json_match: result = json.loads(json_match.group(0)) score = float(result.get("score", 0.0)) reason = result.get("reason", "No reason provided") return { "passed": score >= 0.6, "score": score, "reason": reason, } return { "passed": False, "score": 0.0, "reason": "Could not parse LLM response", } except Exception as e: logger.error(f"Error in code quality check: {e}") return {"passed": False, "score": 0.0, "reason": f"Error: {e}"} def _call_llm(self, prompt: str) -> str: """Call LLM API. Args: prompt: Prompt text Returns: LLM response """ try: if self.provider == "anthropic": response = self.client.messages.create( model=self.model, max_tokens=1024, temperature=0.3, messages=[{"role": "user", "content": prompt}], ) return response.content[0].text elif self.provider in ["openai", "aipipe"]: # Both OpenAI and AIPipe use the same API format response = self.client.chat.completions.create( model=self.model, messages=[{"role": "user", "content": prompt}], temperature=0.3, max_tokens=1024, ) return response.choices[0].message.content except Exception as e: logger.error(f"LLM API call failed: {e}") raise