Update main.py

main.py

@@ -10,8 +10,6 @@ from flask_sqlalchemy import SQLAlchemy
 from sqlalchemy.exc import IntegrityError
 from thefuzz import process, fuzz
 from werkzeug.utils import secure_filename
-import tempfile
-import sqlite3
 
 # ───────────────────────────────────────────────────────────────────────────────
 # CONFIGURATION
@@ -23,34 +21,24 @@ log = logging.getLogger("product-pipeline-api")
 app = Flask(__name__)
 CORS(app)
 
-# --- HUGGING FACE SPACES OPTIMIZED DATABASE CONFIGURATION ---
-# Use in-memory SQLite database for Hugging Face Spaces compatibility
-# This avoids all file permission issues while maintaining full functionality
-app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///:memory:'
+# --- App Configuration ---
+# --- FIX 1: Switched to a persistent file-based SQLite database ---
+# This ensures data survives between requests on Hugging Face Spaces.
+DB_FOLDER = 'data'
+DB_PATH = os.path.join(DB_FOLDER, 'products.db')
+os.makedirs(DB_FOLDER, exist_ok=True)
+app.config['SQLALCHEMY_DATABASE_URI'] = f'sqlite:///{DB_PATH}'
 app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
-
-# --- Upload Folder Configuration ---
-def get_upload_folder():
-    """Get a writable upload folder with fallbacks."""
-    try:
-        # Use system temp directory which is always writable
-        temp_dir = tempfile.mkdtemp(prefix='product_uploads_')
-        log.info(f"Using upload folder: {temp_dir}")
-        return temp_dir
-    except Exception as e:
-        log.error(f"Failed to create upload folder: {e}")
-        return tempfile.gettempdir()
-
-app.config['UPLOAD_FOLDER'] = get_upload_folder()
+app.config['UPLOAD_FOLDER'] = 'uploads'
+os.makedirs(app.config['UPLOAD_FOLDER'], exist_ok=True)
 
 # --- File Upload Configuration ---
 ALLOWED_EXTENSIONS = {'csv', 'xls', 'xlsx'}
 
-# --- Database Initialization ---
 db = SQLAlchemy(app)
 
 # ───────────────────────────────────────────────────────────────────────────────
-# DATABASE MODEL
+# DATABASE MODEL (Based on products-20.sql)
 # ───────────────────────────────────────────────────────────────────────────────
 
 class Product(db.Model):
@@ -84,105 +72,41 @@ HS_CODES_DATA = []
 EXISTING_PRODUCT_NAMES = []
 HS_CODE_DESCRIPTIONS = {}
 
-def initialize_database():
-    """Initialize database and load reference data."""
-    try:
-        with app.app_context():
-            # Create all tables
-            db.create_all()
-            log.info("In-memory database tables created successfully")
-            
-            # Pre-populate with reference data from the CSV
-            reference_products = [
-                "RUSSELL HOBBS GLOSSY BLACK TOASTER",
-                "RUSSELL HOBBS OVAL SLOW COOKER 6.5LTR",
-                "RUSSELL HOBBS PRESSURE COOKER 6LTR",
-                "SAACHI SANDWICH MAKER",
-                "SANFORD BLENDER",
-                "TOTALLY HOME DINNER PLATE",
-                # Add more reference products as needed
-            ]
-            
-            for product_name in reference_products:
-                try:
-                    existing_product = Product.query.filter_by(name=product_name).first()
-                    if not existing_product:
-                        new_product = Product(
-                            name=product_name,
-                            primary_category="Kitchen Appliances",
-                            hs_code="85167200"  # Sample HS code for kitchen appliances
-                        )
-                        db.session.add(new_product)
-                except Exception as e:
-                    log.warning(f"Failed to add reference product {product_name}: {e}")
-                    continue
-            
-            try:
-                db.session.commit()
-                log.info(f"Database initialized with {len(reference_products)} reference products")
-            except Exception as e:
-                db.session.rollback()
-                log.error(f"Failed to commit reference products: {e}")
-                
-            return True
-            
-    except Exception as e:
-        log.error(f"Database initialization failed: {e}")
-        return False
-
 def parse_hs_codes_pdf(filepath='HS Codes for use under FDMS.pdf'):
-    """Parse HS Codes from PDF with error handling."""
-    log.info(f"Attempting to parse HS Codes from '{filepath}'...")
+    log.info(f"Parsing HS Codes from '{filepath}'...")
     if not os.path.exists(filepath):
-        log.warning(f"HS Code PDF not found at '{filepath}'. Using sample HS codes.")
-        # Provide sample HS codes for kitchen/household items
-        sample_codes = {
-            "Kitchen appliances": "85167200",
-            "Cookware": "73239100",
-            "Glassware": "70134900",
-            "Cutlery": "82159900",
-            "Textiles": "63079000"
-        }
-        global HS_CODE_DESCRIPTIONS
-        HS_CODE_DESCRIPTIONS.update(sample_codes)
-        return list(sample_codes.items())
-    
+        log.error(f"HS Code PDF not found at '{filepath}'. Categorization will fail.")
+        return []
     codes = []
     try:
         with pdfplumber.open(filepath) as pdf:
-            for page_num, page in enumerate(pdf.pages):
-                try:
-                    text = page.extract_text()
-                    if not text:
-                        continue
-                        
-                    matches = re.findall(r'\"(\d{8})\"\s*,\s*\"(.*?)\"', text, re.DOTALL)
-                    for code, desc in matches:
-                        clean_desc = desc.replace('\n', ' ').strip()
-                        if code and clean_desc:
-                            codes.append({'code': code, 'description': clean_desc})
-                            HS_CODE_DESCRIPTIONS[clean_desc] = code
-                            
-                except Exception as e:
-                    log.warning(f"Error processing page {page_num + 1}: {e}")
-                    continue
-                    
+            for page in pdf.pages:
+                text = page.extract_text()
+                # Improved regex to handle variations in PDF formatting
+                matches = re.findall(r'\"(\d{8})\"\s*,\s*\"(.*?)\"', text, re.DOTALL)
+                for code, desc in matches:
+                    clean_desc = desc.replace('\n', ' ').strip()
+                    if code and clean_desc:
+                        codes.append({'code': code, 'description': clean_desc})
+                        HS_CODE_DESCRIPTIONS[clean_desc] = code
     except Exception as e:
-        log.error(f"Failed to parse PDF '{filepath}': {e}")
-    
-    log.info(f"Successfully parsed {len(codes)} HS codes from PDF.")
+        log.error(f"Failed to parse PDF: {e}")
+    log.info(f"Successfully parsed {len(codes)} HS codes.")
     return codes
 
-def load_existing_products():
-    """Load existing products from database and CSV reference."""
+def load_existing_products(filepath='Product List.csv'):
+    log.info(f"Loading master product list from '{filepath}'...")
+    if not os.path.exists(filepath):
+        log.error(f"Master product list not found at '{filepath}'. Validation may be inaccurate.")
+        return []
     try:
-        # Load from database
-        with app.app_context():
-            db_products = [p.name for p in Product.query.all()]
-            log.info(f"Loaded {len(db_products)} products from database")
-            return db_products
+        # Based on the CSV structure, the 'name' is in the second column.
+        df = pd.read_csv(filepath, usecols=[1], names=['name'], header=0)
+        product_names = df['name'].dropna().unique().tolist()
+        log.info(f"Loaded {len(product_names)} unique existing products.")
+        return product_names
     except Exception as e:
-        log.error(f"Failed to load products from database: {e}")
+        log.error(f"Failed to load master product list: {e}")
         return []
 
 # ───────────────────────────────────────────────────────────────────────────────
@@ -190,178 +114,82 @@ def load_existing_products():
 # ───────────────────────────────────────────────────────────────────────────────
 
 def process_uploaded_file(filepath, filename):
-    """Process uploaded file with comprehensive error handling and validation."""
+    """The main pipeline to validate, clean, categorize, and store product data."""
     log.info(f"Starting processing for file: {filepath}")
     results = {
         "processed": 0, "added": 0, "updated": 0, "skipped_duplicates": 0,
         "errors": [], "processed_data": []
     }
-    
-    # Verify database is accessible
-    try:
-        with app.app_context():
-            db.engine.execute('SELECT 1')
-    except Exception as e:
-        results['errors'].append(f"Database connection failed: {str(e)}")
-        return results
-    
+    df = None
+
     try:
-        # Read file with robust error handling
-        df = None
-        file_ext = filename.rsplit('.', 1)[1].lower() if '.' in filename else ''
-        
-        # Try different encodings for CSV files
+        file_ext = filename.rsplit('.', 1)[1].lower()
+        # --- FIX 2: Robustly parse the second column (index 1) for names ---
+        # The user's uploaded `list.csv` clearly has the product name in the second column.
         if file_ext == 'csv':
-            for encoding in ['utf-8', 'latin-1', 'cp1252']:
-                try:
-                    df = pd.read_csv(filepath, encoding=encoding, header=None)
-                    log.info(f"Successfully read CSV with {encoding} encoding")
-                    break
-                except (UnicodeDecodeError, ValueError):
-                    continue
-                except Exception as e:
-                    log.warning(f"Error reading CSV with {encoding}: {e}")
-                    continue
+            df = pd.read_csv(filepath, header=None, usecols=[1], names=['product_name'])
         elif file_ext in ['xls', 'xlsx']:
-            try:
-                df = pd.read_excel(filepath, header=None, engine='openpyxl')
-                log.info("Successfully read Excel file")
-            except Exception as e:
-                log.error(f"Error reading Excel file: {e}")
-                
-        if df is None or df.empty:
-            results['errors'].append("Could not read the uploaded file or file is empty")
-            return results
-        
-        # Determine the correct column for product names
-        # Based on your CSV structure, product names are in column 1 (second column)
-        product_column_idx = 1
-        
-        if len(df.columns) < 2:
-            results['errors'].append("File must have at least 2 columns. Product names should be in the second column.")
-            return results
-        
-        log.info(f"Processing {len(df)} rows from uploaded file")
-        log.info(f"Using column index {product_column_idx} for product names")
-        
-        # Load current existing products for fuzzy matching
-        current_existing = load_existing_products()
-        
-        for index, row in df.iterrows():
-            try:
-                raw_name = row.iloc[product_column_idx] if len(row) > product_column_idx else None
-                results['processed'] += 1
-                
-                if pd.isna(raw_name) or not str(raw_name).strip():
-                    log.debug(f"Skipping empty product name at row {index + 1}")
-                    continue
-                
-                cleaned_name = str(raw_name).strip()
-                
-                # Skip if name is too short or invalid
-                if len(cleaned_name) < 3:
-                    log.debug(f"Skipping too short product name: '{cleaned_name}'")
-                    continue
-                
-                # Fuzzy matching with existing products
-                best_match, score = (cleaned_name, 100)
-                if current_existing:
-                    try:
-                        match_result = process.extractOne(
-                            cleaned_name, current_existing, scorer=fuzz.token_sort_ratio
-                        )
-                        if match_result:
-                            best_match, score = match_result
-                    except Exception as e:
-                        log.warning(f"Fuzzy matching failed for '{cleaned_name}': {e}")
-                
-                validated_name = best_match if score >= FUZZY_MATCH_THRESHOLD else cleaned_name
-                
-                # HS Code matching
-                best_hs_desc = None
-                hs_code = None
-                if HS_CODE_DESCRIPTIONS:
-                    try:
-                        hs_match = process.extractOne(
-                            validated_name, list(HS_CODE_DESCRIPTIONS.keys())
-                        )
-                        if hs_match:
-                            best_hs_desc, _ = hs_match
-                            hs_code = HS_CODE_DESCRIPTIONS.get(best_hs_desc)
-                    except Exception as e:
-                        log.warning(f"HS code matching failed for '{validated_name}': {e}")
-                
-                processed_entry = {
-                    "raw_name": str(raw_name),
-                    "cleaned_name": validated_name,
-                    "hs_code": hs_code,
-                    "primary_category": best_hs_desc or "N/A",
-                    "status": ""
-                }
-                
-                # Database operations with comprehensive error handling
-                try:
-                    with app.app_context():
-                        existing_product = Product.query.filter_by(name=validated_name).first()
-                        
-                        if existing_product:
-                            # Update existing product if we have new information
-                            updated = False
-                            if hs_code and existing_product.hs_code != hs_code:
-                                existing_product.hs_code = hs_code
-                                updated = True
-                            if best_hs_desc and existing_product.primary_category != best_hs_desc:
-                                existing_product.primary_category = best_hs_desc
-                                updated = True
-                                
-                            if updated:
-                                db.session.commit()
-                                results['updated'] += 1
-                                processed_entry['status'] = 'Updated'
-                                log.debug(f"Updated product: {validated_name}")
-                            else:
-                                results['skipped_duplicates'] += 1
-                                processed_entry['status'] = 'Skipped (Duplicate)'
-                                log.debug(f"Skipped duplicate: {validated_name}")
-                        else:
-                            # Add new product
-                            new_product = Product(
-                                name=validated_name,
-                                hs_code=hs_code,
-                                primary_category=best_hs_desc or 'N/A'
-                            )
-                            db.session.add(new_product)
-                            db.session.commit()
-                            results['added'] += 1
-                            processed_entry['status'] = 'Added'
-                            current_existing.append(validated_name)  # Update local cache
-                            log.debug(f"Added new product: {validated_name}")
-                        
-                        results['processed_data'].append(processed_entry)
-                        
-                except Exception as e:
-                    try:
-                        db.session.rollback()
-                    except:
-                        pass
-                    error_msg = f"Database error for '{validated_name}': {str(e)}"
-                    log.error(error_msg)
-                    results['errors'].append(error_msg)
-                    processed_entry['status'] = 'Error'
-                    results['processed_data'].append(processed_entry)
-                    
-            except Exception as e:
-                error_msg = f"Error processing row {index + 1}: {str(e)}"
-                log.error(error_msg)
-                results['errors'].append(error_msg)
-                continue
-    
+            df = pd.read_excel(filepath, header=None, usecols=[1], names=['product_name'], engine='openpyxl')
+    except ValueError:
+         results['errors'].append("Could not find the product name column. Ensure the product name is in the second column.")
+         return results
     except Exception as e:
-        error_msg = f"Critical error processing file: {str(e)}"
-        log.error(error_msg)
-        results['errors'].append(error_msg)
-    
-    log.info(f"Processing complete. Added: {results['added']}, Updated: {results['updated']}, Skipped: {results['skipped_duplicates']}, Errors: {len(results['errors'])}")
+        log.error(f"Could not read the uploaded file: {e}")
+        results['errors'].append(f"Invalid file format or corrupt file: {e}")
+        return results
+
+    if df.empty:
+        results['errors'].append("The uploaded file is empty.")
+        return results
+
+    for index, row in df.iterrows():
+        raw_name = row['product_name']
+        results['processed'] += 1
+
+        if not isinstance(raw_name, str) or not raw_name.strip():
+            continue
+
+        cleaned_name = raw_name.strip()
+
+        best_match, score = process.extractOne(
+            cleaned_name, EXISTING_PRODUCT_NAMES, scorer=fuzz.token_sort_ratio
+        ) if EXISTING_PRODUCT_NAMES else (cleaned_name, 100)
+        validated_name = best_match if score >= FUZZY_MATCH_THRESHOLD else cleaned_name
+
+        best_hs_desc, _ = process.extractOne(
+            validated_name, HS_CODE_DESCRIPTIONS.keys()
+        ) if HS_CODE_DESCRIPTIONS else (None, 0)
+        hs_code = HS_CODE_DESCRIPTIONS.get(best_hs_desc)
+
+        processed_entry = {
+            "raw_name": raw_name, "cleaned_name": validated_name, "hs_code": hs_code,
+            "primary_category": best_hs_desc or "N/A", "status": ""
+        }
+        try:
+            # Each operation needs its own app context to interact with the database
+            with app.app_context():
+                existing_product = Product.query.filter_by(name=validated_name).first()
+                if existing_product:
+                    if hs_code and existing_product.hs_code != hs_code:
+                        existing_product.hs_code = hs_code
+                        existing_product.primary_category = best_hs_desc
+                        db.session.commit()
+                        results['updated'] += 1
+                        processed_entry['status'] = 'Updated'
+                    else:
+                        results['skipped_duplicates'] += 1
+                        processed_entry['status'] = 'Skipped (Duplicate)'
+                else:
+                    new_product = Product(name=validated_name, hs_code=hs_code, primary_category=best_hs_desc or 'N/A')
+                    db.session.add(new_product)
+                    db.session.commit()
+                    results['added'] += 1
+                    processed_entry['status'] = 'Added'
+                results['processed_data'].append(processed_entry)
+        except Exception as e:
+            db.session.rollback()
+            log.error(f"Database error for '{validated_name}': {e}")
+            results['errors'].append(f"DB Error on '{validated_name}': {e}")
     return results
 
 # ───────────────────────────────────────────────────────────────────────────────
@@ -373,191 +201,49 @@ def allowed_file(filename):
 
 @app.get("/")
 def root():
-    try:
-        # Test database connection
-        with app.app_context():
-            product_count = Product.query.count()
-            
-        return jsonify({
-            "ok": True, 
-            "message": "The Product Validation server is running.",
-            "database": "in-memory SQLite",
-            "products_count": product_count,
-            "upload_folder": app.config['UPLOAD_FOLDER']
-        })
-    except Exception as e:
-        return jsonify({
-            "ok": False,
-            "message": "Server running but database error",
-            "error": str(e)
-        }), 500
-
-@app.get("/api/health")
-def health_check():
-    """Comprehensive health check endpoint."""
-    try:
-        with app.app_context():
-            # Test database connection and get counts
-            product_count = Product.query.count()
-            
-            # Test a simple database operation
-            test_product = Product.query.first()
-            
-        return jsonify({
-            "ok": True,
-            "database": "connected",
-            "database_type": "in-memory SQLite",
-            "products_count": product_count,
-            "has_sample_data": test_product is not None,
-            "upload_folder_exists": os.path.exists(app.config['UPLOAD_FOLDER'])
-        })
-    except Exception as e:
-        log.error(f"Health check failed: {e}")
-        return jsonify({
-            "ok": False,
-            "database": "disconnected",
-            "error": str(e)
-        }), 500
+    return jsonify({"ok": True, "message": "The Product Validation server is running."})
 
 @app.post("/api/upload")
 def upload_products():
-    """Upload and process product file with comprehensive validation."""
     if 'file' not in request.files:
         return jsonify({"ok": False, "error": "No file part in the request"}), 400
-    
     file = request.files['file']
     if file.filename == '':
         return jsonify({"ok": False, "error": "No file selected"}), 400
 
-    if not allowed_file(file.filename):
-        return jsonify({
-            "ok": False, 
-            "error": f"Invalid file type. Allowed types are: {', '.join(ALLOWED_EXTENSIONS)}"
-        }), 400
-
-    # Verify database is accessible before processing
-    try:
-        with app.app_context():
-            db.engine.execute('SELECT 1')
-    except Exception as e:
-        return jsonify({
-            "ok": False,
-            "error": f"Database not available: {str(e)}"
-        }), 500
-
-    try:
+    if file and allowed_file(file.filename):
         filename = secure_filename(file.filename)
         filepath = os.path.join(app.config['UPLOAD_FOLDER'], filename)
         file.save(filepath)
-        log.info(f"File saved to: {filepath}")
-        
-        # Process the uploaded file
         results = process_uploaded_file(filepath, filename)
-        
-        # Clean up uploaded file
-        try:
-            os.remove(filepath)
-            log.debug(f"Cleaned up file: {filepath}")
-        except Exception as e:
-            log.warning(f"Failed to clean up file {filepath}: {e}")
-        
-        # Return results with proper error status
-        if results['errors'] and not (results['added'] or results['updated']):
-            # If we have errors and no successful operations, return error status
-            return jsonify({
-                "ok": False, 
-                "message": "File processing failed", 
-                "results": results
-            }), 400
-        else:
-            # Return success even if there were some errors, as long as some operations succeeded
-            return jsonify({
-                "ok": True, 
-                "message": "File processed successfully", 
-                "results": results
-            })
-            
-    except Exception as e:
-        error_msg = f"Upload processing failed: {str(e)}"
-        log.error(error_msg)
-        return jsonify({
-            "ok": False, 
-            "error": error_msg
-        }), 500
+        return jsonify({"ok": True, "message": "File processed successfully", "results": results})
+
+    return jsonify({"ok": False, "error": f"Invalid file type. Allowed types are: {', '.join(ALLOWED_EXTENSIONS)}"}), 400
 
 @app.get("/api/products")
 def get_products():
-    """Get all products from database."""
     log.info("Request received to fetch all products.")
     try:
-        with app.app_context():
-            all_products = Product.query.all()
-            products_list = [product.to_dict() for product in all_products]
-            
+        all_products = Product.query.all()
+        products_list = [product.to_dict() for product in all_products]
         log.info(f"Successfully retrieved {len(products_list)} products.")
-        return jsonify({
-            "ok": True, 
-            "count": len(products_list), 
-            "products": products_list
-        })
-    except Exception as e:
-        error_msg = f"Failed to retrieve products: {str(e)}"
-        log.error(error_msg)
-        return jsonify({
-            "ok": False, 
-            "error": error_msg
-        }), 500
-
-@app.delete("/api/products")
-def clear_products():
-    """Clear all products from database (useful for testing)."""
-    try:
-        with app.app_context():
-            count = Product.query.count()
-            Product.query.delete()
-            db.session.commit()
-            
-        log.info(f"Cleared {count} products from database")
-        return jsonify({
-            "ok": True,
-            "message": f"Cleared {count} products from database"
-        })
+        return jsonify({"ok": True, "count": len(products_list), "products": products_list})
     except Exception as e:
-        try:
-            db.session.rollback()
-        except:
-            pass
-        error_msg = f"Failed to clear products: {str(e)}"
-        log.error(error_msg)
-        return jsonify({
-            "ok": False,
-            "error": error_msg
-        }), 500
+        log.error(f"Could not retrieve products from database: {e}")
+        return jsonify({"ok": False, "error": "Failed to retrieve products from the database."}), 500
 
 # ───────────────────────────────────────────────────────────────────────────────
 # MAIN (Server Initialization)
 # ───────────────────────────────────────────────────────────────────────────────
 
 if __name__ == "__main__":
-    log.info("===== Application Startup =====")
-    
-    # Initialize database
-    if not initialize_database():
-        log.error("Failed to initialize database. Exiting.")
-        exit(1)
-    
-    # Load supporting data (non-critical)
-    try:
-        log.info("Loading supporting data...")
+    with app.app_context():
+        log.info("Initializing server...")
+        db.create_all()
         HS_CODES_DATA = parse_hs_codes_pdf()
         EXISTING_PRODUCT_NAMES = load_existing_products()
-        log.info("Supporting data loaded successfully")
-    except Exception as e:
-        log.warning(f"Failed to load supporting data: {e}")
-        log.info("Server will continue with basic functionality")
-    
-    log.info("Server initialization complete")
-    log.info("Using in-memory SQLite database (data will not persist between restarts)")
-    log.info("This is optimized for Hugging Face Spaces deployment")
-    
-    port = int(os.environ.get("PORT", "7860"))
+        log.info(f"Server is ready. Database is at: {DB_PATH}")
+
+    port = int(os.environ.get("PORT", "7860"))
+    app.run(host="0.0.0.0", port=port, debug=False)
+