tests/scripts/detect_breaking_changes.py

#!/usr/bin/env python3
"""Detect breaking changes in OpenAPI specification using openapi-diff."""
import argparse
import json
import os
import subprocess
import sys
from pathlib import Path

# Set PYTHONPATH to include backend directory
backend_dir = Path(__file__).parent.parent.parent
if str(backend_dir) not in sys.path:
    sys.path.insert(0, str(backend_dir))


def check_openapi_diff_installed():
    """Check if openapi-diff is available."""
    try:
        result = subprocess.run(
            ["npx", "openapi-diff", "--version"],
            capture_output=True,
            text=True,
            timeout=10
        )
        return result.returncode == 0
    except (FileNotFoundError, subprocess.TimeoutExpired):
        return False


def detect_breaking_changes(base_spec, current_spec, output_format="json"):
    """Compare OpenAPI specs and detect breaking changes.

    Args:
        base_spec: Path to baseline OpenAPI spec (e.g., openapi.json)
        current_spec: Path to current OpenAPI spec (newly generated)
        output_format: Output format (json or text)

    Returns:
        Dict with breaking_changes detected and details
    """
    if not check_openapi_diff_installed():
        print("ERROR: openapi-diff not found. Install with: npm install -g openapi-diff")
        sys.exit(1)

    print(f"Comparing OpenAPI specs:")
    print(f"  Base: {base_spec}")
    print(f"  Current: {current_spec}")

    result = subprocess.run([
        "npx", "openapi-diff",
        base_spec,
        current_spec
    ], capture_output=True, text=True, timeout=30)

    diff_data = {
        "breaking_changes": [],
        "non_breaking_changes": [],
        "exit_code": result.returncode,
        "has_breaking_changes": result.returncode != 0,
        "raw_output": result.stdout,
        "raw_error": result.stderr
    }

    # Check if this is a validation error (spec issue, not diff issue)
    is_validation_error = "Validation errors" in result.stderr

    # Distinguish false positives (Pydantic 2.0+) from real validation errors
    is_pydantic_false_positive = (
        is_validation_error and
        ("anyOf" in result.stderr or "null" in result.stderr)
    )

    # Parse output for breaking changes
    if result.returncode != 0 and not is_validation_error:
        # Genuine breaking changes detected
        diff_data["breaking_changes"] = ["Breaking changes detected (see output)"]
        diff_data["has_breaking_changes"] = True
    elif is_validation_error and not is_pydantic_false_positive:
        # Real validation error - spec is malformed
        diff_data["validation_errors"] = True
        diff_data["has_breaking_changes"] = True  # FAIL on real validation errors
        diff_data["breaking_changes"] = ["OpenAPI spec validation error - see stderr"]
    elif is_pydantic_false_positive:
        # Pydantic 2.0+ false positive - anyOf + null pattern
        diff_data["validation_errors"] = True
        diff_data["pydantic_false_positive"] = True
        diff_data["has_breaking_changes"] = False
        diff_data["breaking_changes"] = []

    return diff_data


def main():
    """CLI entry point for breaking change detection."""
    parser = argparse.ArgumentParser(
        description="Detect breaking API changes using OpenAPI diff"
    )
    parser.add_argument(
        "--base",
        default=str(backend_dir / "openapi.json"),
        help="Baseline OpenAPI spec (default: backend/openapi.json)"
    )
    parser.add_argument(
        "--current",
        help="Current OpenAPI spec (default: auto-generate)"
    )
    parser.add_argument(
        "--allow-breaking",
        action="store_true",
        help="Exit 0 even if breaking changes found (for documentation)"
    )
    parser.add_argument(
        "--update-baseline",
        action="store_true",
        help="Update baseline with current spec (use with care)"
    )

    args = parser.parse_args()

    # Generate current spec if not provided
    if args.current:
        current_spec = args.current
    else:
        # Generate temporary current spec
        current_spec = "/tmp/openapi_current.json"
        print("Generating current OpenAPI spec...")
        env = os.environ.copy()
        env['PYTHONPATH'] = str(backend_dir)
        subprocess.run([
            "python3", "tests/scripts/generate_openapi_spec.py",
            "-o", current_spec
        ], check=True, env=env)

    base_spec = args.base

    # Verify files exist
    if not Path(base_spec).exists():
        print(f"ERROR: Baseline spec not found: {base_spec}")
        print("Run: python3 tests/scripts/generate_openapi_spec.py")
        sys.exit(1)

    if not Path(current_spec).exists():
        print(f"ERROR: Current spec not found: {current_spec}")
        sys.exit(1)

    # Detect breaking changes
    result = detect_breaking_changes(base_spec, current_spec)

    # Report results
    if result.get("raw_output"):
        print(f"\n{result['raw_output']}")

    if result.get("pydantic_false_positive"):
        print("\n⚠️  OpenAPI spec validation warning (Pydantic 2.0+ false positive)")
        print("   The diff tool detected 'anyOf + null' patterns from Pydantic 2.0+")
        print("   These are known false positives and don't affect functionality")
        print("\n✅ No breaking changes detected between specs")
    elif result.get("validation_errors"):
        print("\n❌ OpenAPI spec validation error")
        print("   The OpenAPI spec is malformed and cannot be validated")
        print("   Check the stderr output above for specific validation issues")
        print("\n❌ Build failed due to validation errors")
    elif result.get("has_breaking_changes"):
        if result.get("breaking_changes"):
            print(f"\n❌ Found {len(result['breaking_changes'])} breaking changes")
        else:
            print("\n❌ Breaking changes detected")
    else:
        print("\n✅ No breaking changes detected")

    # Update baseline if requested
    if args.update_baseline:
        import shutil
        shutil.copy(current_spec, base_spec)
        print(f"\nUpdated baseline: {base_spec}")

    # Exit with error on breaking changes (unless --allow-breaking)
    if result.get("has_breaking_changes") and not args.allow_breaking:
        sys.exit(1)
    else:
        sys.exit(0)


if __name__ == "__main__":
    main()