Add model versioning and training history tracking system

app.py

@@ -6,6 +6,7 @@ Integrated with feedback learning pipeline for continuous model improvement
 from flask import Flask, request, jsonify
 from inference_core import run_pipeline_for_image, download_image_from_url, upload_to_cloudinary, model, device
 from scripts.feedback_learning_pipeline import initialize_feedback_pipeline, run_feedback_training
+from scripts.model_versioning import initialize_model_tracker
 import os
 
 app = Flask(__name__)
@@ -13,6 +14,9 @@ app = Flask(__name__)
 # Initialize feedback learning pipeline
 feedback_pipeline = initialize_feedback_pipeline(model, device)
 
+# Initialize model versioning tracker
+model_tracker = initialize_model_tracker()
+
 
 @app.route("/", methods=["GET"])
 def home():
@@ -24,7 +28,11 @@ def home():
             "/health": "GET - Health check",
             "/infer": "POST - Run inference on image URL",
             "/feedback/stats": "GET - Get feedback statistics and training status",
-            "/feedback/train": "POST - Manually trigger feedback training cycle"
+            "/feedback/train": "POST - Manually trigger feedback training cycle",
+            "/model/current": "GET - Get current model version and parameters",
+            "/model/versions": "GET - Get model version history",
+            "/model/training-history": "GET - Get training cycle history",
+            "/model/compare": "POST - Compare model versions"
         },
         "example_request": {
             "method": "POST",
@@ -37,6 +45,11 @@ def home():
             "description": "User corrections are automatically fetched from Supabase",
             "training_trigger": "Automatic when 10+ new feedback samples available",
             "manual_training": "POST /feedback/train to trigger immediately"
+        },
+        "versioning_info": {
+            "description": "Model versions and training history tracked automatically",
+            "view_current": "GET /model/current to see active model parameters",
+            "view_history": "GET /model/versions to see all versions"
         }
     })
 
@@ -72,6 +85,74 @@ def trigger_training():
         return jsonify({"error": str(e)}), 500
 
 
+@app.route("/model/current", methods=["GET"])
+def get_current_model():
+    """
+    Get current active model version and parameters
+    """
+    try:
+        current_state = model_tracker.get_current_model_state()
+        return jsonify(current_state), 200
+    except Exception as e:
+        return jsonify({"error": str(e)}), 500
+
+
+@app.route("/model/versions", methods=["GET"])
+def get_model_versions():
+    """
+    Get model version history
+    Query params: limit (default: 20)
+    """
+    try:
+        limit = int(request.args.get('limit', 20))
+        versions = model_tracker.get_version_history(limit=limit)
+        return jsonify({
+            "total": len(versions),
+            "versions": versions
+        }), 200
+    except Exception as e:
+        return jsonify({"error": str(e)}), 500
+
+
+@app.route("/model/training-history", methods=["GET"])
+def get_training_history():
+    """
+    Get training cycle history
+    Query params: limit (default: 20)
+    """
+    try:
+        limit = int(request.args.get('limit', 20))
+        history = model_tracker.get_training_history(limit=limit)
+        return jsonify({
+            "total": len(history),
+            "training_cycles": history
+        }), 200
+    except Exception as e:
+        return jsonify({"error": str(e)}), 500
+
+
+@app.route("/model/compare", methods=["POST"])
+def compare_versions():
+    """
+    Compare multiple model versions
+    Request JSON: {"version_ids": ["id1", "id2", ...]}
+    """
+    try:
+        data = request.get_json()
+        if not data or "version_ids" not in data:
+            return jsonify({"error": "Missing version_ids"}), 400
+        
+        version_ids = data["version_ids"]
+        comparison = model_tracker.generate_comparison_table(version_ids)
+        
+        return jsonify({
+            "comparison": comparison,
+            "version_count": len(version_ids)
+        }), 200
+    except Exception as e:
+        return jsonify({"error": str(e)}), 500
+
+
 @app.route("/infer", methods=["POST"])
 def infer():
     """

scripts/feedback_learning_pipeline.py

@@ -13,6 +13,14 @@ from supabase import create_client, Client
 from PIL import Image
 import tempfile
 
+# Import model versioning system
+try:
+    from scripts.model_versioning import ModelVersionTracker
+    MODEL_VERSIONING_AVAILABLE = True
+except ImportError:
+    MODEL_VERSIONING_AVAILABLE = False
+    print("[Warning] Model versioning not available")
+
 # Supabase configuration
 SUPABASE_URL = os.getenv("SUPABASE_URL", "https://xbcgrpqiibicestnhytt.supabase.co")
 SUPABASE_KEY = os.getenv("SUPABASE_SERVICE_ROLE_KEY", "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6InhiY2dycHFpaWJpY2VzdG5oeXR0Iiwicm9sZSI6InNlcnZpY2Vfcm9sZSIsImlhdCI6MTc1NTkxMzk3MywiZXhwIjoyMDcxNDg5OTczfQ.sANBuVZ6gdYc5kHkxTXZ67jtE9QHPw5HFaUKffP1Jrs")
@@ -40,6 +48,12 @@ class FeedbackLearningPipeline:
         self.device = device
         self.training_state = self._load_training_state()
         
+        # Initialize model versioning tracker
+        if MODEL_VERSIONING_AVAILABLE:
+            self.version_tracker = ModelVersionTracker()
+        else:
+            self.version_tracker = None
+        
     def _load_training_state(self) -> Dict[str, Any]:
         """Load training state from disk"""
         if os.path.exists(TRAINING_STATE_FILE):
@@ -252,6 +266,13 @@ class FeedbackLearningPipeline:
         """
         print(f"\n[Feedback Pipeline] Starting training cycle at {datetime.now()}")
         
+        # Capture model state BEFORE training
+        before_state = None
+        if self.version_tracker:
+            before_state = self.version_tracker.get_current_model_state()
+            self.version_tracker.log_model_version(before_state)
+            print(f"[Model Versioning] Captured state before training: {before_state['version_id'][:8]}...")
+        
         # Fetch new feedback
         feedback_logs = self.fetch_new_feedback(limit=1000)
         
@@ -290,6 +311,26 @@ class FeedbackLearningPipeline:
         
         self._save_training_state()
         
+        # Capture model state AFTER training
+        after_state = None
+        training_cycle_id = None
+        if self.version_tracker:
+            after_state = self.version_tracker.get_current_model_state()
+            self.version_tracker.log_model_version(after_state)
+            print(f"[Model Versioning] Captured state after training: {after_state['version_id'][:8]}...")
+            
+            # Log the training cycle with before/after comparison
+            training_cycle_id = self.version_tracker.log_training_cycle(
+                before_state=before_state,
+                after_state=after_state,
+                feedback_count=len(corrections),
+                patterns=patterns,
+                performance_metrics=None  # TODO: Calculate actual metrics
+            )
+            
+            if training_cycle_id:
+                print(f"[Training History] Logged training cycle: {training_cycle_id[:8]}...")
+        
         print(f"[Feedback Pipeline] Training cycle completed successfully")
         print(f"[Feedback Pipeline] Total feedback processed: {self.training_state['total_feedback_processed']}")
         
@@ -297,7 +338,10 @@ class FeedbackLearningPipeline:
             "status": "success",
             "corrections_processed": len(corrections),
             "patterns": patterns,
-            "total_feedback_processed": self.training_state["total_feedback_processed"]
+            "total_feedback_processed": self.training_state["total_feedback_processed"],
+            "before_version_id": before_state["version_id"] if before_state else None,
+            "after_version_id": after_state["version_id"] if after_state else None,
+            "training_cycle_id": training_cycle_id
         }
     
     def get_feedback_stats(self) -> Dict[str, Any]:

scripts/model_versioning.py

@@ -0,0 +1,436 @@
+"""
+Model Versioning & Training History System
+Tracks model parameters, thresholds, and training evolution in Supabase
+"""
+import os
+import json
+from datetime import datetime
+from typing import Dict, Any, Optional, List
+from supabase import create_client, Client
+import uuid
+
+# Supabase configuration
+SUPABASE_URL = os.getenv("SUPABASE_URL", "https://xbcgrpqiibicestnhytt.supabase.co")
+SUPABASE_KEY = os.getenv("SUPABASE_SERVICE_ROLE_KEY", "")
+
+# Initialize Supabase client
+supabase: Client = create_client(SUPABASE_URL, SUPABASE_KEY)
+
+
+class ModelVersionTracker:
+    """
+    Tracks model versions, parameters, and training history
+    """
+    
+    def __init__(self):
+        """Initialize the model version tracker"""
+        self.supabase = supabase
+        
+    def get_current_model_state(self) -> Dict[str, Any]:
+        """
+        Get current model parameters and thresholds
+        
+        Returns:
+            Dictionary containing current model state
+        """
+        # Read current model adjustments if they exist
+        adjustments = {}
+        if os.path.exists("model_adjustments.json"):
+            with open("model_adjustments.json", "r") as f:
+                adjustments = json.load(f)
+        
+        # Read training state
+        training_state = {}
+        if os.path.exists("feedback_training_state.json"):
+            with open("feedback_training_state.json", "r") as f:
+                training_state = json.load(f)
+        
+        # Define current model parameters
+        model_state = {
+            "version_id": str(uuid.uuid4()),
+            "timestamp": datetime.now().isoformat(),
+            
+            # Model Configuration
+            "model_architecture": "PatchCore",
+            "backbone": "Wide ResNet-50",
+            "layers": ["layer2", "layer3"],
+            "input_size": [256, 256],
+            
+            # Detection Thresholds
+            "anomaly_threshold": 128,  # Binary mask threshold
+            "confidence_range": [0.3, 0.99],
+            "min_detection_size": 100,  # Minimum pixels for detection
+            
+            # Classification Thresholds
+            "red_color_threshold": {
+                "hue_range": [0, 10, 170, 180],
+                "saturation_min": 100,
+                "value_min": 100
+            },
+            "yellow_color_threshold": {
+                "hue_range": [20, 30],
+                "saturation_min": 100,
+                "value_min": 100
+            },
+            "orange_color_threshold": {
+                "hue_range": [10, 20],
+                "saturation_min": 100,
+                "value_min": 100
+            },
+            
+            # Post-processing Parameters
+            "merge_distance_threshold": 20,
+            "iou_threshold": 0.4,
+            "min_contour_area": 100,
+            
+            # Learned Adjustments (from feedback)
+            "false_positive_rate": adjustments.get("fp_rate", 0.0),
+            "false_negative_rate": adjustments.get("fn_rate", 0.0),
+            "threshold_recommendation": adjustments.get("recommendation", "Not yet calculated"),
+            
+            # Training Metadata
+            "total_feedback_processed": training_state.get("total_feedback_processed", 0),
+            "last_training_time": training_state.get("last_training_time"),
+            "training_runs_count": len(training_state.get("training_runs", []))
+        }
+        
+        return model_state
+    
+    def log_model_version(self, model_state: Dict[str, Any]) -> Optional[str]:
+        """
+        Log current model version to Supabase
+        
+        Args:
+            model_state: Dictionary containing model parameters
+            
+        Returns:
+            Version ID if successful, None otherwise
+        """
+        try:
+            # Prepare record for database
+            record = {
+                "version_id": model_state["version_id"],
+                "timestamp": model_state["timestamp"],
+                "model_architecture": model_state["model_architecture"],
+                "backbone": model_state["backbone"],
+                "parameters": {
+                    "layers": model_state["layers"],
+                    "input_size": model_state["input_size"],
+                    "anomaly_threshold": model_state["anomaly_threshold"],
+                    "confidence_range": model_state["confidence_range"],
+                    "min_detection_size": model_state["min_detection_size"]
+                },
+                "thresholds": {
+                    "red_color": model_state["red_color_threshold"],
+                    "yellow_color": model_state["yellow_color_threshold"],
+                    "orange_color": model_state["orange_color_threshold"],
+                    "merge_distance": model_state["merge_distance_threshold"],
+                    "iou": model_state["iou_threshold"],
+                    "min_contour_area": model_state["min_contour_area"]
+                },
+                "learned_adjustments": {
+                    "false_positive_rate": model_state["false_positive_rate"],
+                    "false_negative_rate": model_state["false_negative_rate"],
+                    "recommendation": model_state["threshold_recommendation"]
+                },
+                "training_metadata": {
+                    "total_feedback_processed": model_state["total_feedback_processed"],
+                    "last_training_time": model_state["last_training_time"],
+                    "training_runs_count": model_state["training_runs_count"]
+                },
+                "is_active": True
+            }
+            
+            # Insert into database
+            response = self.supabase.table('model_versions').insert(record).execute()
+            
+            if response.data:
+                print(f"[Model Versioning] Logged version {model_state['version_id']}")
+                return model_state["version_id"]
+            else:
+                print("[Model Versioning] Failed to log version")
+                return None
+                
+        except Exception as e:
+            print(f"[Model Versioning] Error logging version: {e}")
+            return None
+    
+    def log_training_cycle(self, 
+                          before_state: Dict[str, Any],
+                          after_state: Dict[str, Any],
+                          feedback_count: int,
+                          patterns: Dict[str, Any],
+                          performance_metrics: Optional[Dict[str, Any]] = None) -> Optional[str]:
+        """
+        Log a training cycle with before/after comparison
+        
+        Args:
+            before_state: Model state before training
+            after_state: Model state after training
+            feedback_count: Number of feedback samples processed
+            patterns: Pattern analysis from feedback
+            performance_metrics: Optional performance metrics
+            
+        Returns:
+            Training cycle ID if successful
+        """
+        try:
+            cycle_id = str(uuid.uuid4())
+            
+            # Calculate parameter changes
+            parameter_changes = self._calculate_parameter_changes(before_state, after_state)
+            
+            record = {
+                "cycle_id": cycle_id,
+                "timestamp": datetime.now().isoformat(),
+                "before_version_id": before_state["version_id"],
+                "after_version_id": after_state["version_id"],
+                "feedback_samples_processed": feedback_count,
+                
+                # Pattern Analysis
+                "feedback_patterns": {
+                    "label_changes": patterns.get("label_changes", []),
+                    "bbox_adjustments": patterns.get("bbox_adjustments", []),
+                    "false_positives": patterns.get("false_positives", 0),
+                    "false_negatives": patterns.get("false_negatives", 0)
+                },
+                
+                # Parameter Changes
+                "parameter_changes": parameter_changes,
+                
+                # Performance Metrics (if available)
+                "performance_metrics": performance_metrics or {
+                    "accuracy_improvement": "Not yet calculated",
+                    "precision_improvement": "Not yet calculated",
+                    "recall_improvement": "Not yet calculated"
+                },
+                
+                # Recommendations
+                "threshold_recommendation": after_state.get("threshold_recommendation", ""),
+                
+                # Status
+                "status": "completed",
+                "notes": f"Processed {feedback_count} feedback samples"
+            }
+            
+            # Insert into database
+            response = self.supabase.table('training_history').insert(record).execute()
+            
+            if response.data:
+                print(f"[Training History] Logged cycle {cycle_id}")
+                return cycle_id
+            else:
+                print("[Training History] Failed to log cycle")
+                return None
+                
+        except Exception as e:
+            print(f"[Training History] Error logging cycle: {e}")
+            return None
+    
+    def _calculate_parameter_changes(self, before: Dict[str, Any], after: Dict[str, Any]) -> Dict[str, Any]:
+        """Calculate what changed between before and after states"""
+        changes = {}
+        
+        # Compare false positive/negative rates
+        if before["false_positive_rate"] != after["false_positive_rate"]:
+            changes["false_positive_rate"] = {
+                "before": before["false_positive_rate"],
+                "after": after["false_positive_rate"],
+                "delta": after["false_positive_rate"] - before["false_positive_rate"]
+            }
+        
+        if before["false_negative_rate"] != after["false_negative_rate"]:
+            changes["false_negative_rate"] = {
+                "before": before["false_negative_rate"],
+                "after": after["false_negative_rate"],
+                "delta": after["false_negative_rate"] - before["false_negative_rate"]
+            }
+        
+        # Compare training metadata
+        if before["total_feedback_processed"] != after["total_feedback_processed"]:
+            changes["total_feedback_processed"] = {
+                "before": before["total_feedback_processed"],
+                "after": after["total_feedback_processed"],
+                "delta": after["total_feedback_processed"] - before["total_feedback_processed"]
+            }
+        
+        if before["threshold_recommendation"] != after["threshold_recommendation"]:
+            changes["threshold_recommendation"] = {
+                "before": before["threshold_recommendation"],
+                "after": after["threshold_recommendation"]
+            }
+        
+        return changes
+    
+    def get_version_history(self, limit: int = 20) -> List[Dict[str, Any]]:
+        """
+        Get recent model version history
+        
+        Args:
+            limit: Maximum number of versions to retrieve
+            
+        Returns:
+            List of model versions
+        """
+        try:
+            response = self.supabase.table('model_versions')\
+                .select('*')\
+                .order('timestamp', desc=True)\
+                .limit(limit)\
+                .execute()
+            
+            return response.data if response.data else []
+            
+        except Exception as e:
+            print(f"[Model Versioning] Error fetching history: {e}")
+            return []
+    
+    def get_training_history(self, limit: int = 20) -> List[Dict[str, Any]]:
+        """
+        Get recent training cycles
+        
+        Args:
+            limit: Maximum number of cycles to retrieve
+            
+        Returns:
+            List of training cycles
+        """
+        try:
+            response = self.supabase.table('training_history')\
+                .select('*')\
+                .order('timestamp', desc=True)\
+                .limit(limit)\
+                .execute()
+            
+            return response.data if response.data else []
+            
+        except Exception as e:
+            print(f"[Training History] Error fetching history: {e}")
+            return []
+    
+    def get_active_version(self) -> Optional[Dict[str, Any]]:
+        """
+        Get currently active model version
+        
+        Returns:
+            Active model version or None
+        """
+        try:
+            response = self.supabase.table('model_versions')\
+                .select('*')\
+                .eq('is_active', True)\
+                .order('timestamp', desc=True)\
+                .limit(1)\
+                .execute()
+            
+            if response.data:
+                return response.data[0]
+            return None
+            
+        except Exception as e:
+            print(f"[Model Versioning] Error fetching active version: {e}")
+            return None
+    
+    def generate_comparison_table(self, version_ids: List[str]) -> str:
+        """
+        Generate a comparison table between model versions
+        
+        Args:
+            version_ids: List of version IDs to compare
+            
+        Returns:
+            Formatted comparison table string
+        """
+        try:
+            versions = []
+            for vid in version_ids:
+                response = self.supabase.table('model_versions')\
+                    .select('*')\
+                    .eq('version_id', vid)\
+                    .execute()
+                if response.data:
+                    versions.append(response.data[0])
+            
+            if not versions:
+                return "No versions found"
+            
+            # Generate comparison table
+            table = "\n" + "=" * 100 + "\n"
+            table += "MODEL VERSION COMPARISON\n"
+            table += "=" * 100 + "\n\n"
+            
+            for i, v in enumerate(versions):
+                table += f"Version {i+1}: {v['version_id'][:8]}...\n"
+                table += f"Timestamp: {v['timestamp']}\n"
+                table += f"Architecture: {v['model_architecture']} ({v['backbone']})\n"
+                table += f"False Positive Rate: {v['learned_adjustments']['false_positive_rate']:.2%}\n"
+                table += f"False Negative Rate: {v['learned_adjustments']['false_negative_rate']:.2%}\n"
+                table += f"Feedback Processed: {v['training_metadata']['total_feedback_processed']}\n"
+                table += f"Recommendation: {v['learned_adjustments']['recommendation']}\n"
+                table += "-" * 100 + "\n"
+            
+            return table
+            
+        except Exception as e:
+            print(f"[Model Versioning] Error generating comparison: {e}")
+            return f"Error: {e}"
+
+
+def initialize_model_tracker():
+    """Initialize the model version tracker"""
+    return ModelVersionTracker()
+
+
+# SQL for creating the required tables (run in Supabase Dashboard)
+CREATE_TABLES_SQL = """
+-- Table: model_versions
+-- Stores each model version with parameters and thresholds
+CREATE TABLE IF NOT EXISTS model_versions (
+    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    version_id VARCHAR(255) UNIQUE NOT NULL,
+    timestamp TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+    model_architecture VARCHAR(100) NOT NULL,
+    backbone VARCHAR(100),
+    parameters JSONB,
+    thresholds JSONB,
+    learned_adjustments JSONB,
+    training_metadata JSONB,
+    is_active BOOLEAN DEFAULT TRUE,
+    created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+CREATE INDEX IF NOT EXISTS idx_model_versions_timestamp ON model_versions(timestamp DESC);
+CREATE INDEX IF NOT EXISTS idx_model_versions_active ON model_versions(is_active) WHERE is_active = TRUE;
+
+-- Table: training_history
+-- Stores training cycle information with before/after comparisons
+CREATE TABLE IF NOT EXISTS training_history (
+    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    cycle_id VARCHAR(255) UNIQUE NOT NULL,
+    timestamp TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+    before_version_id VARCHAR(255),
+    after_version_id VARCHAR(255),
+    feedback_samples_processed INTEGER,
+    feedback_patterns JSONB,
+    parameter_changes JSONB,
+    performance_metrics JSONB,
+    threshold_recommendation TEXT,
+    status VARCHAR(50),
+    notes TEXT,
+    created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+CREATE INDEX IF NOT EXISTS idx_training_history_timestamp ON training_history(timestamp DESC);
+CREATE INDEX IF NOT EXISTS idx_training_history_status ON training_history(status);
+
+-- Foreign key constraints
+ALTER TABLE training_history 
+    ADD CONSTRAINT fk_before_version 
+    FOREIGN KEY (before_version_id) 
+    REFERENCES model_versions(version_id);
+
+ALTER TABLE training_history 
+    ADD CONSTRAINT fk_after_version 
+    FOREIGN KEY (after_version_id) 
+    REFERENCES model_versions(version_id);
+"""