backend/modules/documentation/doc_generator.py

"""Documentation Generation Engine powered by IBM Bob with a repository-scan fallback."""

from __future__ import annotations

import re
from collections import Counter

from core.watsonx import watsonx_generate
from utils.cache import analysis_cache, repo_cache
from utils.context_builder import build_docs_context


async def generate_documentation(repo_id: str, doc_type: str = "onboarding") -> dict:
    repo_data = repo_cache.get(repo_id)
    if not repo_data:
        raise ValueError("Repository not loaded")

    normalized = doc_type if doc_type in {"onboarding", "api", "architecture"} else "onboarding"
    content = await _generate_with_bob(repo_id, repo_data, normalized)
    source = "ibm_bob"
    if not _looks_like_markdown_doc(content):
        generators = {
            "onboarding": _generate_onboarding,
            "api": _generate_api_reference,
            "architecture": _generate_architecture_adr,
        }
        content = generators[normalized](repo_id, repo_data)
        source = "repository_scan_fallback"

    return {
        "doc_type": normalized,
        "content": content,
        "repo_id": repo_id,
        "format": "markdown",
        "source": source,
    }


async def _generate_with_bob(repo_id: str, repo_data: dict, doc_type: str) -> str:
    context = build_docs_context(repo_data)
    analysis = analysis_cache.get(repo_id, {}).get("analysis", {})
    architecture_hint = ""
    if analysis:
        architecture_hint = f"""
BOB ARCHITECTURE INTELLIGENCE:
Architecture type: {analysis.get('architecture_type')}
Summary: {analysis.get('summary')}
Core services: {[service.get('name') for service in analysis.get('core_services', [])[:8]]}
Workflows: {[workflow.get('name') for workflow in analysis.get('business_workflows', [])[:6]]}
"""

    instructions = {
        "onboarding": "Create a comprehensive developer onboarding guide with setup, architecture, key files, workflows, APIs, risk areas, and troubleshooting.",
        "api": "Create complete API documentation with detected endpoints, request/response guidance, authentication/security notes, and testing guidance.",
        "architecture": "Create an architecture ADR with context, decisions, service boundaries, data flow, security model, risks, and consequences.",
    }[doc_type]

    prompt = f"""You are IBM Bob documenting a real software repository.
Use ONLY the repository context below. Do not describe CodeAtlas unless the repository itself is CodeAtlas.
Do not save files, edit files, or say that a file was saved. Return the markdown document directly in this response.
Use repo-specific details, file paths, services, endpoints, and workflows. Avoid generic filler.
Use plain ASCII markdown.

TASK:
{instructions}

REPOSITORY CONTEXT:
{context}
{architecture_hint}

Return only the markdown document.
###END###"""

    return await watsonx_generate(
        prompt,
        max_tokens=3500,
        use_code_model=doc_type != "onboarding",
        working_dir=repo_data.get("repo_path"),
    )


def _looks_like_markdown_doc(content: str) -> bool:
    if not content or len(content.strip()) < 500:
        return False
    lowered = content.lower()
    if "saved to `" in lowered or "has been saved" in lowered:
        return False
    return "#" in content[:400] or "##" in content[:800]


def _generate_onboarding(repo_id: str, repo_data: dict) -> str:
    tech = repo_data["technologies"]
    structure = repo_data["structure"]
    analysis = analysis_cache.get(repo_id, {}).get("analysis", {})
    important_files = _important_files(repo_data)
    endpoints = _detect_api_endpoints(repo_data)
    setup = _setup_steps(repo_data)

    return f"""# Developer Onboarding Guide

## System Overview
Repository: `{repo_data["github_url"]}`

This codebase contains {structure["total_files"]} files and {structure["total_lines"]} scanned lines. CodeAtlas detected { _join_or_none(tech.get("frameworks", [])) } as the primary framework layer and { _join_or_none([item["name"] for item in tech.get("languages", [])[:4]]) } as the main implementation languages.

{analysis.get("summary", "The repository is organized around entry points, feature modules, service utilities, and configuration files. Start with the application entry points, then trace the API or UI flows into the shared services.")}

## Architecture Overview
- Architecture type: `{analysis.get("architecture_type", "Modular application")}`
- Frontend/frameworks: {_join_or_none(tech.get("frameworks", []))}
- Databases: {_join_or_none(tech.get("databases", []))}
- DevOps: {_join_or_none(tech.get("devops", []))}
- API style: {_join_or_none(tech.get("apis", []))}

## Local Setup Instructions
{setup}

## Key Files to Understand
{_bullet_list(important_files[:12])}

## Core Business Workflows
{_workflow_section(analysis)}

## API Reference
{_endpoint_section(endpoints)}

## Database Schema Overview
{_database_section(repo_data)}

## Coding Conventions and Patterns
- Keep entry-point files thin and move business behavior into service modules.
- Review importers before changing shared utilities, schemas, authentication, or routing files.
- Prefer focused tests around the workflow touched by a change.
- Keep environment-specific values in example env/config files instead of hardcoding secrets.

## Common Development Tasks
- Start the local backend or frontend using the commands listed in package manifests.
- Add or update API routes in the detected routing modules.
- Add tests near changed behavior and run the most focused test command first.
- Use CodeAtlas Impact Analysis before modifying shared or security-sensitive files.

## Deployment Process
Review CI/CD files, deployment manifests, Docker files, platform config, and environment examples before release. For this repository, detected deployment or operations files include:

{_bullet_list(_ops_files(repo_data)[:10])}

## Troubleshooting Guide
- If the app does not start, verify dependencies from package manifests and environment variables from `.env.example` files.
- If APIs fail, inspect route files and service modules listed above.
- If security or scan workflows fail, inspect auth/security/networking modules and run an integration workflow.
- If a change has unclear blast radius, run CodeAtlas Impact Analysis on the changed file.

## Ownership and Knowledge Areas
- Entry points and routing: application startup and API ownership.
- Services/utilities: domain behavior and integrations.
- UI components: user-facing workflow behavior.
- Config and operations: local setup, deploy, logging, and runtime configuration.
"""


def _generate_api_reference(repo_id: str, repo_data: dict) -> str:
    endpoints = _detect_api_endpoints(repo_data)
    route_files = [item for item in _important_files(repo_data) if _has_token(item, ("route", "router", "api", "app.py", "main.py"))]

    return f"""# API Reference

## Overview
Repository: `{repo_data["github_url"]}`

CodeAtlas detected {_join_or_none(repo_data["technologies"].get("apis", []))} API patterns. The endpoint list below is extracted from scanned route and application files.

## Routing Files
{_bullet_list(route_files[:15])}

## Detected Endpoints
{_endpoint_section(endpoints)}

## Request and Response Guidance
- Confirm request bodies and response schemas in the route handler and adjacent service/model files.
- Check authentication or security helpers before changing protected endpoints.
- Add integration tests for route changes that affect user-visible behavior.

## Authentication
{_auth_section(repo_data)}

## Example Test Checklist
- Start the backend locally.
- Call the endpoint with valid input.
- Call the endpoint with invalid input.
- Validate status code, response shape, and error message.
- Run related service-layer tests before deployment.
"""


def _generate_architecture_adr(repo_id: str, repo_data: dict) -> str:
    analysis = analysis_cache.get(repo_id, {}).get("analysis", {})
    tech = repo_data["technologies"]
    services = analysis.get("core_services", [])
    dependencies = analysis.get("critical_dependencies", [])

    return f"""# Architecture Decision Record

## Context
Repository: `{repo_data["github_url"]}`

The system uses {_join_or_none(tech.get("frameworks", []))} with primary languages {_join_or_none([item["name"] for item in tech.get("languages", [])[:5]])}. CodeAtlas classified the architecture as `{analysis.get("architecture_type", "Modular application")}`.

## Decision
Keep the application organized around explicit entry points, routing modules, service modules, and configuration boundaries. Changes to shared services, security modules, and data contracts should be reviewed with dependency and workflow context.

## Architecture Summary
{analysis.get("summary", "The repository is structured as a modular application with clear runtime entry points and supporting service modules.")}

## Core Services
{_services_section(services)}

## Critical Dependencies
{_dependencies_section(dependencies)}

## Data Flow
1. Runtime starts through the primary entry point.
2. Requests or UI events enter routing/controller modules.
3. Service modules perform domain work and integration calls.
4. Persistence, external tools, or platform services are invoked as needed.
5. Results are returned to the UI or API caller.

## Consequences
- Entry-point and routing changes have broad workflow impact.
- Shared utility and service changes should be paired with focused tests.
- Architecture diagrams should be regenerated after large module moves.

## Risks and Mitigations
{_risk_section(analysis, repo_data)}
"""


def _important_files(repo_data: dict) -> list[str]:
    files = repo_data["structure"]["files"]
    weights = ("main", "app", "server", "route", "router", "api", "service", "model", "schema", "auth", "security", "config", "package", "requirements", "readme")

    def score(file_info: dict) -> tuple[int, int]:
        path = file_info["path"].lower()
        return (sum(4 for token in weights if token in path) + max(0, 5 - path.count("/")), file_info.get("lines", 0))

    return [item["path"] for item in sorted(files, key=score, reverse=True)]


def _detect_api_endpoints(repo_data: dict) -> list[dict]:
    endpoints = []
    patterns = [
        re.compile(r"@(?:app|router|api_bp|bp)\.(get|post|put|patch|delete)\([\"']([^\"']+)[\"']", re.IGNORECASE),
        re.compile(r"@(get|post|put|patch|delete)\([\"']([^\"']+)[\"']", re.IGNORECASE),
        re.compile(r"(?:app|router)\.(get|post|put|patch|delete)\([\"']([^\"']+)[\"']", re.IGNORECASE),
    ]
    for path, content in repo_data.get("file_contents", {}).items():
        if not _has_token(path, ("route", "router", "api", "app.py", "main.py", "server", "controller")):
            continue
        for pattern in patterns:
            for method, route in pattern.findall(content):
                endpoints.append({"method": method.upper(), "path": route, "file": path})
    unique = {}
    for endpoint in endpoints:
        unique[(endpoint["method"], endpoint["path"], endpoint["file"])] = endpoint
    return list(unique.values())[:30]


def _setup_steps(repo_data: dict) -> str:
    deps = repo_data.get("dependencies", {})
    files = {item["path"] for item in repo_data["structure"]["files"]}
    steps = []
    if "package.json" in files:
        steps.extend(["1. Install Node dependencies: `npm install`.", "2. Start the JavaScript app using the script in `package.json`."])
    if "frontend/package.json" in files:
        steps.append("3. Install frontend dependencies: `cd frontend && npm install`.")
    if "requirements.txt" in files or "backend/requirements.txt" in files:
        steps.append("4. Install Python dependencies from the detected requirements file.")
    if any(path.endswith(".env.example") for path in files):
        steps.append("5. Copy `.env.example` values into a local `.env` file and fill local settings.")
    if deps.get("npm") or deps.get("pip"):
        steps.append("6. Run the app, then execute focused tests or smoke checks for the changed workflow.")
    return "\n".join(steps) if steps else "- Review README/setup files, install dependencies, configure environment variables, and start the app locally."


def _workflow_section(analysis: dict) -> str:
    workflows = analysis.get("business_workflows") or []
    if not workflows:
        return "- Start application\n- Route request or UI event\n- Invoke service logic\n- Return response or update UI"
    lines = []
    for workflow in workflows[:6]:
        lines.append(f"### {workflow.get('name', 'Workflow')}")
        lines.append(_bullet_list(workflow.get("steps", [])))
    return "\n".join(lines)


def _endpoint_section(endpoints: list[dict]) -> str:
    if not endpoints:
        return "- No explicit route decorators were detected in scanned files. Inspect routing, controller, or server entry-point files manually."
    return "\n".join(f"- `{endpoint['method']} {endpoint['path']}` in `{endpoint['file']}`" for endpoint in endpoints)


def _database_section(repo_data: dict) -> str:
    databases = repo_data["technologies"].get("databases", [])
    schema_files = [path for path in _important_files(repo_data) if _has_token(path, ("schema", "model", "migration", "database", "db"))]
    if not databases and not schema_files:
        return "No database signature was detected. State may be file-based, external-tool based, or managed outside this repository."
    return f"Detected databases: {_join_or_none(databases)}\n\nRelevant files:\n{_bullet_list(schema_files[:12])}"


def _auth_section(repo_data: dict) -> str:
    auth_files = [path for path in _important_files(repo_data) if _has_token(path, ("auth", "security", "oauth", "token", "permission"))]
    if not auth_files:
        return "No dedicated authentication module was detected in scanned files. Verify whether protection is handled by middleware, gateway, or deployment configuration."
    return "Authentication/security files:\n" + _bullet_list(auth_files[:10])


def _services_section(services: list[dict]) -> str:
    if not services:
        return "- Core services are inferred from entry-point, route, service, and utility files."
    lines = []
    for service in services:
        files = ", ".join(f"`{file}`" for file in service.get("files", [])[:4]) or "No linked files"
        lines.append(f"- **{service.get('name')}** (`{service.get('type')}`): {service.get('description')} Files: {files}")
    return "\n".join(lines)


def _dependencies_section(dependencies: list[dict]) -> str:
    if not dependencies:
        return "- No explicit critical dependencies were available. Use the architecture graph and import search for detailed blast radius."
    return "\n".join(
        f"- `{dep.get('from')}` -> `{dep.get('to')}` ({dep.get('type')}): {dep.get('description')}"
        for dep in dependencies
    )


def _risk_section(analysis: dict, repo_data: dict) -> str:
    risks = analysis.get("risk_areas") or []
    if risks:
        return "\n".join(f"- **{risk.get('name')}** ({risk.get('severity')}): {risk.get('description')}" for risk in risks[:8])
    risky_tokens = Counter()
    for file_info in repo_data["structure"]["files"]:
        path = file_info["path"].lower()
        for token in ("auth", "security", "route", "service", "config", "schema"):
            if token in path:
                risky_tokens[token] += 1
    if not risky_tokens:
        return "- Run focused tests around entry points and shared modules before deployment."
    return "\n".join(f"- `{token}` area: {count} related files detected." for token, count in risky_tokens.most_common(6))


def _ops_files(repo_data: dict) -> list[str]:
    tokens = ("docker", "compose", "github/workflows", "vercel", "railway", "terraform", "k8s", "deploy", "env.example", "config")
    return [item["path"] for item in repo_data["structure"]["files"] if _has_token(item["path"], tokens)]


def _bullet_list(items: list[str]) -> str:
    return "\n".join(f"- `{item}`" for item in items) if items else "- None detected."


def _join_or_none(items: list[str]) -> str:
    return ", ".join(items) if items else "none detected"


def _has_token(value: str, tokens: tuple[str, ...]) -> bool:
    lower = value.lower()
    return any(token in lower for token in tokens)