Senum2001 commited on
Commit
a023a85
·
1 Parent(s): 376e494

Add automated training scheduler - checks every 2 minutes and trains when 10+ feedback samples available

Browse files
Files changed (2) hide show
  1. app.py +61 -2
  2. requirements.txt +3 -0
app.py CHANGED
@@ -7,7 +7,10 @@ from flask import Flask, request, jsonify
7
  from inference_core import run_pipeline_for_image, download_image_from_url, upload_to_cloudinary, model, device
8
  from scripts.feedback_learning_pipeline import initialize_feedback_pipeline, run_feedback_training
9
  from scripts.model_versioning import initialize_model_tracker
 
 
10
  import os
 
11
 
12
  app = Flask(__name__)
13
 
@@ -18,12 +21,67 @@ feedback_pipeline = initialize_feedback_pipeline(model, device)
18
  model_tracker = initialize_model_tracker()
19
 
20
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
21
  @app.route("/", methods=["GET"])
22
  def home():
23
  """Home page with API documentation"""
24
  return jsonify({
25
  "service": "Anomaly Detection API with Feedback Learning",
26
- "version": "2.0",
27
  "endpoints": {
28
  "/health": "GET - Health check",
29
  "/infer": "POST - Run inference on image URL",
@@ -43,7 +101,8 @@ def home():
43
  },
44
  "feedback_info": {
45
  "description": "User corrections are automatically fetched from Supabase",
46
- "training_trigger": "Automatic when 10+ new feedback samples available",
 
47
  "manual_training": "POST /feedback/train to trigger immediately"
48
  },
49
  "versioning_info": {
 
7
  from inference_core import run_pipeline_for_image, download_image_from_url, upload_to_cloudinary, model, device
8
  from scripts.feedback_learning_pipeline import initialize_feedback_pipeline, run_feedback_training
9
  from scripts.model_versioning import initialize_model_tracker
10
+ from apscheduler.schedulers.background import BackgroundScheduler
11
+ from datetime import datetime
12
  import os
13
+ import atexit
14
 
15
  app = Flask(__name__)
16
 
 
21
  model_tracker = initialize_model_tracker()
22
 
23
 
24
+ # ===== Automated Training Scheduler =====
25
+ def automated_training_check():
26
+ """
27
+ Background task that checks for new feedback and triggers training automatically
28
+ Runs periodically to keep the model updated with user corrections
29
+ """
30
+ try:
31
+ print(f"\n[Automated Training] Running scheduled check at {datetime.now()}")
32
+
33
+ # Check if there's enough feedback to warrant training
34
+ stats = feedback_pipeline.get_feedback_stats()
35
+
36
+ if stats.get("ready_for_retraining", False):
37
+ unprocessed = stats.get("total_feedback_in_db", 0) - stats.get("total_processed", 0)
38
+ print(f"[Automated Training] Found {unprocessed} unprocessed feedback samples")
39
+ print(f"[Automated Training] Starting training cycle...")
40
+
41
+ # Trigger training
42
+ results = run_feedback_training(feedback_pipeline)
43
+
44
+ if results.get("status") == "success":
45
+ print(f"[Automated Training] ✓ Training completed successfully")
46
+ print(f"[Automated Training] Processed {results.get('corrections_processed')} corrections")
47
+ else:
48
+ print(f"[Automated Training] Training status: {results.get('status')}")
49
+ else:
50
+ unprocessed = stats.get("total_feedback_in_db", 0) - stats.get("total_processed", 0)
51
+ print(f"[Automated Training] Not enough feedback for training ({unprocessed} new samples)")
52
+
53
+ except Exception as e:
54
+ print(f"[Automated Training] Error during automated check: {e}")
55
+
56
+
57
+ # Initialize background scheduler
58
+ scheduler = BackgroundScheduler(daemon=True)
59
+
60
+ # Schedule training checks every 2 minutes
61
+ # You can adjust the interval: hours, minutes, seconds
62
+ scheduler.add_job(
63
+ func=automated_training_check,
64
+ trigger="interval",
65
+ minutes=2, # Check every 2 minutes
66
+ id='automated_training',
67
+ name='Automated Feedback Training Check',
68
+ replace_existing=True
69
+ )
70
+
71
+ # Start the scheduler
72
+ scheduler.start()
73
+ print("[Automated Training] Scheduler started - checking for new feedback every 2 minutes")
74
+
75
+ # Shutdown scheduler gracefully when app exits
76
+ atexit.register(lambda: scheduler.shutdown())
77
+
78
+
79
  @app.route("/", methods=["GET"])
80
  def home():
81
  """Home page with API documentation"""
82
  return jsonify({
83
  "service": "Anomaly Detection API with Feedback Learning",
84
+ "version": "2.1",
85
  "endpoints": {
86
  "/health": "GET - Health check",
87
  "/infer": "POST - Run inference on image URL",
 
101
  },
102
  "feedback_info": {
103
  "description": "User corrections are automatically fetched from Supabase",
104
+ "automated_training": "Checks for new feedback every 2 minutes and trains automatically",
105
+ "training_threshold": "10+ new feedback samples triggers training",
106
  "manual_training": "POST /feedback/train to trigger immediately"
107
  },
108
  "versioning_info": {
requirements.txt CHANGED
@@ -33,3 +33,6 @@ requests
33
  # Supabase for feedback storage
34
  supabase
35
 
 
 
 
 
33
  # Supabase for feedback storage
34
  supabase
35
 
36
+ # Background task scheduling
37
+ APScheduler
38
+