instructor/checks/llm_checks.py

"""LLM-based code quality checks."""
from pathlib import Path
from typing import Any

import anthropic
import openai

from shared.config import settings
from shared.logger import setup_logger

logger = setup_logger(__name__)


class LLMChecker:
    """Perform LLM-based quality checks."""

    def __init__(self) -> None:
        """Initialize LLM checker."""
        self.provider = settings.llm_provider
        self.model = settings.llm_model

        if self.provider == "anthropic":
            self.client = anthropic.Anthropic(api_key=settings.anthropic_api_key)
        elif self.provider == "openai":
            self.client = openai.OpenAI(api_key=settings.openai_api_key)
        elif self.provider == "aipipe":
            # Use OpenAI client with AIPipe endpoints
            self.client = openai.OpenAI(
                api_key=settings.aipipe_token,
                base_url=settings.aipipe_base_url
            )

        logger.info(f"Initialized LLMChecker with {self.provider}/{self.model}")

    def check_readme_quality(self, readme_path: Path) -> dict[str, Any]:
        """Evaluate README.md quality using LLM.

        Args:
            readme_path: Path to README.md

        Returns:
            Check result
        """
        try:
            if not readme_path.exists():
                return {
                    "passed": False,
                    "score": 0.0,
                    "reason": "README.md not found",
                }

            content = readme_path.read_text()

            prompt = f"""Evaluate this README.md for quality. Rate it on a scale of 0.0 to 1.0.

Criteria:
- Has a clear title and description
- Explains setup/installation
- Explains usage
- Has code examples or explanations
- Is well-formatted and professional
- Mentions license

README content:
{content}

Respond with ONLY a JSON object in this format:
{{
  "score": 0.85,
  "reason": "Clear title and good structure, but missing detailed setup instructions"
}}"""

            response = self._call_llm(prompt)

            # Parse JSON from response
            import json
            import re

            json_match = re.search(r"\{.*\}", response, re.DOTALL)
            if json_match:
                result = json.loads(json_match.group(0))
                score = float(result.get("score", 0.0))
                reason = result.get("reason", "No reason provided")

                return {
                    "passed": score >= 0.7,
                    "score": score,
                    "reason": reason,
                }

            return {
                "passed": False,
                "score": 0.0,
                "reason": "Could not parse LLM response",
            }

        except Exception as e:
            logger.error(f"Error in README quality check: {e}")
            return {"passed": False, "score": 0.0, "reason": f"Error: {e}"}

    def check_code_quality(self, code_dir: Path) -> dict[str, Any]:
        """Evaluate code quality using LLM.

        Args:
            code_dir: Directory containing code

        Returns:
            Check result
        """
        try:
            # Gather code files
            code_files = {}
            for ext in [".html", ".js", ".css"]:
                for file in code_dir.rglob(f"*{ext}"):
                    if ".git" not in str(file):
                        rel_path = file.relative_to(code_dir)
                        code_files[str(rel_path)] = file.read_text()[:2000]  # Limit size

            if not code_files:
                return {
                    "passed": False,
                    "score": 0.0,
                    "reason": "No code files found",
                }

            # Format code for LLM
            code_text = "\n\n".join(
                f"=== {name} ===\n{content}" for name, content in code_files.items()
            )

            prompt = f"""Evaluate this web application code for quality. Rate it on a scale of 0.0 to 1.0.

Criteria:
- Code is clean and well-organized
- Proper use of HTML semantics
- Good JavaScript practices
- Reasonable styling
- Comments where helpful
- No obvious bugs or security issues

Code:
{code_text}

Respond with ONLY a JSON object in this format:
{{
  "score": 0.8,
  "reason": "Clean code with good structure, minor improvements possible"
}}"""

            response = self._call_llm(prompt)

            # Parse JSON
            import json
            import re

            json_match = re.search(r"\{.*\}", response, re.DOTALL)
            if json_match:
                result = json.loads(json_match.group(0))
                score = float(result.get("score", 0.0))
                reason = result.get("reason", "No reason provided")

                return {
                    "passed": score >= 0.6,
                    "score": score,
                    "reason": reason,
                }

            return {
                "passed": False,
                "score": 0.0,
                "reason": "Could not parse LLM response",
            }

        except Exception as e:
            logger.error(f"Error in code quality check: {e}")
            return {"passed": False, "score": 0.0, "reason": f"Error: {e}"}

    def _call_llm(self, prompt: str) -> str:
        """Call LLM API.

        Args:
            prompt: Prompt text

        Returns:
            LLM response
        """
        try:
            if self.provider == "anthropic":
                response = self.client.messages.create(
                    model=self.model,
                    max_tokens=1024,
                    temperature=0.3,
                    messages=[{"role": "user", "content": prompt}],
                )
                return response.content[0].text

            elif self.provider in ["openai", "aipipe"]:
                # Both OpenAI and AIPipe use the same API format
                response = self.client.chat.completions.create(
                    model=self.model,
                    messages=[{"role": "user", "content": prompt}],
                    temperature=0.3,
                    max_tokens=1024,
                )
                return response.choices[0].message.content

        except Exception as e:
            logger.error(f"LLM API call failed: {e}")
            raise