app/api/product_quality_checker_lib.py

#!/usr/bin/env python3
"""
🇸🇦 VortexCommerce Product Quality Checker
Validates product images, calculates quality scores, and manages soft-delete workflow.
"""
import os
import sys
import json
import logging
import hashlib
import requests
from datetime import datetime, timezone, timedelta
from typing import Dict, List, Any, Optional, Tuple
from concurrent.futures import ThreadPoolExecutor, as_completed
import urllib3

urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

if __name__ == "__main__":
    import os
    import sys
    sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

try:
    from app.db.base import engine, SessionLocal, Base
    from app.models.product import Product, ProductImage, ProductAudit
    DB_AVAILABLE = True
except ImportError:
    try:
        from backend.app.db.base import engine, SessionLocal, Base
        from backend.app.models.product import Product, ProductImage, ProductAudit
        DB_AVAILABLE = True
    except ImportError:
        # Standalone mode - we'll define mock models if needed or just skip DB parts
        engine = None
        SessionLocal = None
        Base = None
        Product = None
        DB_AVAILABLE = False


try:
    logging.basicConfig(
        level=logging.INFO,
        format='%(asctime)s - %(levelname)s - %(message)s',
        handlers=[
            logging.FileHandler(f'product_quality_{datetime.now().strftime("%Y%m%d_%H%M%S")}.log'),
            logging.StreamHandler()
        ]
    )
except Exception:
    logging.basicConfig(
        level=logging.INFO,
        format='%(asctime)s - %(levelname)s - %(message)s',
        handlers=[logging.StreamHandler()]
    )
logger = logging.getLogger(__name__)


class QualityReport:
    """Quality check report container."""
    
    def __init__(self):
        self.timestamp = datetime.now(timezone.utc)
        self.total_checked = 0
        self.total_valid = 0
        self.total_invalid = 0
        self.deleted_products: List[Dict[str, Any]] = []
        self.updated_scores: List[Dict[str, Any]] = []
        self.errors: List[str] = []
        
    def to_dict(self) -> Dict[str, Any]:
        return {
            "timestamp": self.timestamp.isoformat(),
            "total_checked": self.total_checked,
            "total_valid": self.total_valid,
            "total_invalid": self.total_invalid,
            "deleted_products": self.deleted_products,
            "updated_scores": self.updated_scores,
            "errors": self.errors
        }


class ProductQualityChecker:
    """Comprehensive product quality validation system."""
    
    def __init__(self, db_session=None):
        self.db = db_session
        self.report = QualityReport()
        self.session = requests.Session()
        self.session.headers.update({
            'User-Agent': 'VortexCommerce-QualityChecker/1.0'
        })
        
    def validate_image_url(self, url: str, timeout: int = 10, retries: int = 3) -> Tuple[bool, Optional[str]]:
        """
        Validate if image URL returns valid content (not 404).
        Returns (is_valid, error_message).
        """
        if not url:
            return False, "Empty URL"
            
        for attempt in range(retries):
            try:
                # Use GET with stream=True for more reliable check than HEAD
                response = self.session.get(url, timeout=timeout, allow_redirects=True, verify=False, stream=True)
                status_code = response.status_code
                content_type = response.headers.get('Content-Type', '')
                response.close()
                
                if status_code == 200 and content_type.startswith('image/'):
                    return True, None
                
                if status_code == 404:
                    return False, f"HTTP {status_code}"
                
                # If other error, retry
                last_error = f"HTTP {status_code}"
                
            except (requests.exceptions.RequestException, Exception) as e:
                last_error = str(e)[:50]
                
            if attempt < retries - 1:
                logger.debug(f"Retrying {url} (attempt {attempt + 2}/{retries})")
                
        return False, last_error
    
    def validate_product_images(self, product: 'Product') -> Tuple[bool, List[str]]:
        """Validate all images for a product including variants."""
        errors = []
        
        # Check primary images
        if not product.images:
            errors.append("No primary images attached to product")
        else:
            for img in product.images:
                is_valid, error = self.validate_image_url(img.image_url)
                if not is_valid:
                    errors.append(f"Primary Image {img.id}: {error}")
        
        # Check variant images if present in specs
        if product.specs and isinstance(product.specs, dict) and 'variants' in product.specs:
            variants = product.specs['variants']
            for i, var in enumerate(variants):
                img_url = var.get('image_url') or var.get('image')
                if img_url:
                    is_valid, error = self.validate_image_url(img_url)
                    if not is_valid:
                        errors.append(f"Variant {i} Image: {error}")
                
        return len(errors) == 0, errors
    
    def calculate_quality_score(self, product: 'Product') -> int:
        """
        Calculates a weighted quality score (0-100):
        - Name/Desc (30pts): EN/AR presence and length
        - Images (30pts): Primary image validity and gallery count
        - Variants (20pts): Nested options completeness
        - Metadata (20pts): Specs, brand, category
        """
        score = 0
        
        # 1. Names & Descriptions (30 pts)
        if product.name_en and len(product.name_en) > 10: score += 7
        if product.name_ar and len(product.name_ar) > 10: score += 8
        if product.description_en and len(product.description_en) > 50: score += 7
        if product.description_ar and len(product.description_ar) > 50: score += 8
        
        # 2. Images (30 pts)
        if product.images and len(product.images) > 0:
            score += 15 # Has at least one image
            if len(product.images) >= 3: score += 15 # Good gallery
            elif len(product.images) == 2: score += 10
        
        # 3. Variants/Options (20 pts)
        has_options = False
        if product.specs and isinstance(product.specs, dict):
            options = product.specs.get('options')
            if options and len(options) > 0:
                has_options = True
                score += 20
        
        # 4. Metadata (20 pts)
        if product.category_id: score += 5
        if product.price > 0: score += 5
        if product.stock > 0: score += 5
        if product.specs and len(product.specs) > 2: score += 5

        return min(100, score)
    
    def perform_soft_delete(self, product: 'Product', reason: str, deleted_by: Optional[int] = None):
        """
        Soft-deletes a product and archives a snapshot for 30-day recovery.
        """
        try:
            # Prepare recovery snapshot
            snapshot = {
                "name_en": product.name_en,
                "name_ar": product.name_ar,
                "price": product.price,
                "stock": product.stock,
                "specs": product.specs,
                "images": [img.image_url for img in product.images],
                "soft_deleted_at": datetime.now(timezone.utc).isoformat()
            }
            
            product.deleted_at = datetime.now(timezone.utc)
            product.is_active = False
            product.deletion_reason = reason
            product.deleted_by = deleted_by
            
            # Log audit trail
            audit = ProductAudit(
                product_id=product.id,
                action="IMAGE_FAILURE_AUTO_DELETE",
                reason=reason,
                snapshot=snapshot,
                performed_by=deleted_by
            )
            self.db.add(audit)
            
            self.report.deleted_products.append({
                "id": product.id,
                "reason": reason
            })
            self.report.total_invalid += 1
            logger.warning(f"🚨 AUTO-DELETED product {product.id} due to {reason}")
            return True
        except Exception as e:
            logger.error(f"Failed to soft-delete {product.id}: {e}")
            return False
    
    def process_product(self, product: 'Product') -> bool:
        """Process quality lifecycle for a single product."""
        self.report.total_checked += 1
        
        # 1. Critical Image Integrity Check (Phase 3 Requirement)
        if not product.images:
            return self.perform_soft_delete(product, "Missing all images")
            
        for img in product.images:
            is_valid, error = self.validate_image_url(img.image_url)
            if not is_valid:
                return self.perform_soft_delete(product, f"Image 404/Error: {error}")
        
        # 2. Score Calculation
        old_score = product.quality_score
        new_score = self.calculate_quality_score(product)
        
        if old_score != new_score:
            product.quality_score = new_score
            self.report.updated_scores.append({
                "id": product.id,
                "old": old_score,
                "new": new_score
            })
            
        self.report.total_valid += 1
        return True
    
    def run_quality_check(self, batch_size: int = 50) -> QualityReport:
        """Run full quality check on all active products."""
        if not self.db:
            logger.error("No database session available")
            return self.report
            
        logger.info("=" * 60)
        logger.info("Starting Product Quality Check")
        logger.info("=" * 60)
        
        try:
            products = self.db.query(Product).filter(
                Product.is_active == True,
                Product.deleted_at == None
            ).all()
            
            logger.info(f"Found {len(products)} active products to check")
            
            for idx, product in enumerate(products, 1):
                self.process_product(product)
                
                if idx % batch_size == 0:
                    self.db.commit()
                    logger.info(f"Processed {idx}/{len(products)} products...")
                    
            self.db.commit()
            
        except Exception as e:
            logger.error(f"Quality check failed: {str(e)}")
            self.db.rollback()
            self.report.errors.append(f"Quality check failed: {str(e)}")
            
        logger.info("=" * 60)
        logger.info(f"Quality Check Complete")
        logger.info(f"  Total Checked: {self.report.total_checked}")
        logger.info(f"  Valid Products: {self.report.total_valid}")
        logger.info(f"  Invalid (Deleted): {self.report.total_invalid}")
        logger.info(f"  Score Updates: {len(self.report.updated_scores)}")
        logger.info("=" * 60)
        
        return self.report


class StandaloneQualityChecker:
    """Run quality check on JSON file without database."""
    
    def __init__(self, json_path: str):
        self.json_path = json_path
        self.report = QualityReport()
        self.session = requests.Session()
        
    def validate_image_url(self, url: str, timeout: int = 10) -> Tuple[bool, Optional[str]]:
        """Validate single image URL."""
        if not url:
            return False, "Empty URL"
            
        try:
            response = self.session.head(url, timeout=timeout, allow_redirects=True, verify=False)
            
            if response.status_code >= 400:
                response = self.session.get(url, stream=True, timeout=timeout, verify=False)
                response.close()
                
            if response.status_code >= 400:
                return False, f"HTTP {response.status_code}"
                
            content_type = response.headers.get('Content-Type', '')
            if not content_type.startswith('image/'):
                return False, f"Invalid content type: {content_type}"
                
            return True, None
            
        except Exception as e:
            return False, str(e)[:50]
    
    def check_json_products(self) -> Dict[str, Any]:
        """Check products from JSON file."""
        logger.info(f"Loading products from {self.json_path}")
        
        with open(self.json_path, 'r', encoding='utf-8') as f:
            data = json.load(f)
            
        products = data.get('products', [])
        logger.info(f"Found {len(products)} products to validate")
        
        results = {
            "valid_products": [],
            "invalid_products": [],
            "summary": {
                "total": len(products),
                "valid": 0,
                "invalid": 0
            }
        }
        
        for idx, product in enumerate(products, 1):
            product_id = product.get('id', f'unknown_{idx}')
            name_en = product.get('name_en', 'Unknown')
            images = product.get('images', [])
            
            invalid_images = []
            valid_images = []
            
            for img_url in images:
                is_valid, error = self.validate_image_url(img_url)
                if is_valid:
                    valid_images.append(img_url)
                else:
                    invalid_images.append({"url": img_url, "error": error})
            
            product_result = {
                "id": product_id,
                "name_en": name_en,
                "images": {
                    "total": len(images),
                    "valid": len(valid_images),
                    "invalid": len(invalid_images),
                    "invalid_details": invalid_images[:2]
                }
            }
            
            if invalid_images:
                results["invalid_products"].append(product_result)
                results["summary"]["invalid"] += 1
                logger.warning(f"Product {product_id}: {len(invalid_images)} invalid images")
            else:
                results["valid_products"].append(product_result)
                results["summary"]["valid"] += 1
                
            if idx % 50 == 0:
                logger.info(f"Processed {idx}/{len(products)} products...")
        
        logger.info(f"Validation complete: {results['summary']['valid']} valid, {results['summary']['invalid']} invalid")
        
        try:
            output_path = self.json_path.replace('.json', '_validation.json')
            with open(output_path, 'w', encoding='utf-8') as f:
                json.dump(results, f, ensure_ascii=False, indent=2)
            logger.info(f"Results saved to {output_path}")
        except Exception as e:
            logger.warning(f"Could not save validation results to disk: {e}")
        
        return results


def create_backup(db_session) -> Optional[str]:
    """Create a backup of current products before quality operations."""
    try:
        products = db_session.query(Product).all()
        
        backup_data = {
            "timestamp": datetime.now(timezone.utc).isoformat(),
            "total_products": len(products),
            "products": []
        }
        
        for p in products:
            product_dict = {
                "id": p.id,
                "name_en": p.name_en,
                "name_ar": p.name_ar,
                "price": p.price,
                "stock": p.stock,
                "is_active": p.is_active,
                "deleted_at": p.deleted_at.isoformat() if p.deleted_at else None,
                "quality_score": p.quality_score,
                "images": [{"id": img.id, "image_url": img.image_url} for img in p.images]
            }
            backup_data["products"].append(product_dict)
        
        backup_dir = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "backups")
        os.makedirs(backup_dir, exist_ok=True)
        
        backup_filename = f"products_backup_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json"
        backup_path = os.path.join(backup_dir, backup_filename)
        
        try:
            with open(backup_path, 'w', encoding='utf-8') as f:
                json.dump(backup_data, f, ensure_ascii=False, indent=2)
            logger.info(f"Backup created: {backup_path}")
        except Exception as e:
            logger.warning(f"Could not write backup to disk: {e}")
        return backup_path
        
    except Exception as e:
        logger.error(f"Backup creation failed: {str(e)}")
        return None


def main():
    """Main entry point."""
    import argparse
    
    parser = argparse.ArgumentParser(description='VortexCommerce Product Quality Checker')
    parser.add_argument('--mode', choices=['db', 'json'], default='db',
                        help='Run against database or JSON file')
    parser.add_argument('--json-path', 
                        default=os.path.join(os.path.dirname(os.path.dirname(__file__)), 
                                            'backend', 'real_products_200.json'),
                        help='Path to JSON file for standalone mode')
    parser.add_argument('--backup', action='store_true',
                        help='Create backup before quality operations')
    parser.add_argument('--batch-size', type=int, default=50,
                        help='Batch size for database commits')
    
    args = parser.parse_args()
    
    logger.info("=" * 60)
    logger.info("VortexCommerce Product Quality Checker v2.0")
    logger.info("=" * 60)
    
    if args.mode == 'json':
        logger.info(f"Running in standalone JSON mode: {args.json_path}")
        checker = StandaloneQualityChecker(args.json_path)
        results = checker.check_json_products()
        logger.info(f"Summary: {results['summary']}")
        return 0
    
    if not DB_AVAILABLE:
        logger.error("Database modules not available. Use --mode json for standalone mode.")
        return 1
    
    db = SessionLocal()
    
    try:
        if args.backup:
            backup_path = create_backup(db)
            if not backup_path:
                logger.warning("Continuing without backup...")
        
        checker = ProductQualityChecker(db)
        report = checker.run_quality_check(args.batch_size)
        
        try:
            report_path = os.path.join(os.path.dirname(__file__), f"quality_report_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json")
            with open(report_path, 'w', encoding='utf-8') as f:
                json.dump(report.to_dict(), f, ensure_ascii=False, indent=2)
            logger.info(f"Quality report saved: {report_path}")
        except Exception as e:
            logger.warning(f"Could not save quality report to disk: {e}")
        
    finally:
        db.close()
    
    return 0


if __name__ == "__main__":
    sys.exit(main())