analyze_code_quality.py

#!/usr/bin/env python3
"""
Code quality analyzer: find duplicates, dead code, unused imports, inconsistencies.
"""
import os
import re
import json
from pathlib import Path
from collections import defaultdict

BACKEND_PATH = Path("d:/FYP_SmartBIZ/smartbiz-ai/backend")
ROUTES_PATH = BACKEND_PATH / "routes"
PIPELINE_PATH = BACKEND_PATH / "pipeline"

def find_function_definitions():
    """Find all function definitions in Python files."""
    functions = defaultdict(list)
    
    for py_file in BACKEND_PATH.rglob("*.py"):
        rel_path = py_file.relative_to(BACKEND_PATH)
        
        with open(py_file, 'r', encoding='utf-8', errors='ignore') as f:
            lines = f.readlines()
        
        for i, line in enumerate(lines, 1):
            # Find function definitions
            if re.match(r'^\s*(?:async\s+)?def\s+\w+', line):
                func_name = re.search(r'def\s+(\w+)', line).group(1)
                functions[func_name].append({
                    'file': str(rel_path),
                    'line': i,
                    'is_private': func_name.startswith('_')
                })
    
    return functions

def find_imports():
    """Find all imports and identify unused ones."""
    all_imports = defaultdict(list)
    
    for py_file in BACKEND_PATH.rglob("*.py"):
        rel_path = py_file.relative_to(BACKEND_PATH)
        
        with open(py_file, 'r', encoding='utf-8', errors='ignore') as f:
            content = f.read()
        
        # Find imports
        imports = re.findall(r'(?:from|import)\s+[\w\.]+', content)
        for imp in imports:
            all_imports[imp].append(str(rel_path))
    
    return all_imports

def find_duplicate_functions():
    """Find duplicate function definitions."""
    functions = find_function_definitions()
    
    duplicates = {}
    for func_name, locations in functions.items():
        if len(locations) > 1 and not func_name.startswith('__'):
            duplicates[func_name] = locations
    
    return duplicates

def analyze_code_quality():
    """Run comprehensive code analysis."""
    print("=" * 80)
    print("SMARTBIZ CODE QUALITY ANALYSIS")
    print("=" * 80)
    
    # 1. Find duplicate functions
    print("\n[1] DUPLICATE FUNCTIONS")
    print("-" * 80)
    duplicates = find_duplicate_functions()
    if duplicates:
        for func_name, locations in sorted(duplicates.items()):
            print(f"\n  ⚠️  {func_name}")
            for loc in locations:
                print(f"     - {loc['file']} (line {loc['line']})")
    else:
        print("  ✓ No duplicate functions found")
    
    # 2. Check for unused private functions
    print("\n[2] PRIVATE FUNCTIONS (potential cleanup targets)")
    print("-" * 80)
    functions = find_function_definitions()
    private_funcs = {k: v for k, v in functions.items() if k.startswith('_') and not k.startswith('__')}
    
    if private_funcs:
        print(f"  Found {len(private_funcs)} private functions:")
        # Show only the problematic ones (duplicates)
        for func_name, locations in sorted(private_funcs.items()):
            if len(locations) > 1:
                print(f"  ⚠️  DUPLICATE: {func_name}")
                for loc in locations:
                    print(f"     - {loc['file']} (line {loc['line']})")
    else:
        print("  ✓ No private functions found")
    
    # 3. File count and structure
    print("\n[3] CODEBASE STRUCTURE")
    print("-" * 80)
    py_files = list(BACKEND_PATH.rglob("*.py"))
    routes_files = list(ROUTES_PATH.rglob("*.py"))
    pipeline_files = list(PIPELINE_PATH.rglob("*.py"))
    
    print(f"  Total Python files: {len(py_files)}")
    print(f"  - Routes: {len(routes_files)}")
    print(f"  - Pipeline: {len(pipeline_files)}")
    print(f"  - Other: {len(py_files) - len(routes_files) - len(pipeline_files)}")
    
    # 4. Check for test files that might be redundant
    print("\n[4] TEST FILES (root level)")
    print("-" * 80)
    root_path = Path("d:/FYP_SmartBIZ/smartbiz-ai")
    test_files = sorted(root_path.glob("test_*.py"))
    
    if test_files:
        print(f"  Found {len(test_files)} test files:")
        for tf in test_files:
            size_kb = tf.stat().st_size / 1024
            with open(tf, 'r', encoding='utf-8', errors='ignore') as f:
                lines = len(f.readlines())
            print(f"  - {tf.name} ({lines} lines, {size_kb:.1f} KB)")
    
    # 5. Check for endpoints consistency
    print("\n[5] API ENDPOINTS")
    print("-" * 80)
    endpoints = {}
    
    for py_file in ROUTES_PATH.glob("*.py"):
        with open(py_file, 'r', encoding='utf-8', errors='ignore') as f:
            content = f.read()
        
        routes = re.findall(r'@router\.(\w+)\("([^"]+)"', content)
        if routes:
            endpoints[py_file.stem] = routes
    
    for route_file, routes_list in sorted(endpoints.items()):
        print(f"  {route_file}.py:")
        for method, path in routes_list:
            print(f"    - {method.upper()} {path}")
    
    # 6. Summary statistics
    print("\n[6] STATISTICS")
    print("-" * 80)
    all_functions = find_function_definitions()
    print(f"  Total function definitions: {len(all_functions)}")
    print(f"  Total locations across codebase: {sum(len(v) for v in all_functions.values())}")
    
    duplicate_count = sum(1 for v in all_functions.values() if len(v) > 1)
    print(f"  Function names with multiple definitions: {duplicate_count}")
    
    print("\n" + "=" * 80)
    print("RECOMMENDATIONS FOR MVP OPTIMIZATION")
    print("=" * 80)
    print("""
1. CONSOLIDATION
   ✓ Created backend/pipeline/query/utils.py with shared utilities
   ✓ Updated dashboard.py and report.py to use shared guess_x_axis/guess_y_axis

2. UNUSED TEST FILES (consider archiving or consolidating)
   - test_pipeline_duckdb.py
   - test_pipeline_comprehensive.py  
   - test_multi_dataset.py
   - test_multi_api.py
   - check_db.py
   → Keep test_system_checkup.py and test_query_cars.py as validation suite

3. ROUTE PREFIX FIX
   ✓ Added /api prefix to process and upload routers for consistency

4. NEXT STEPS
   - Run full integration test with updated endpoints
   - Measure report generation performance
   - Archive or consolidate test files into single validation suite
""")

if __name__ == "__main__":
    analyze_code_quality()