test

BestModel.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ee571cc7163696ad7b3b4e2ef470d0524e55601975241350659e37ee446c25c5
+size 404203145

app.py

@@ -1,41 +1,322 @@
-from datetime import datetime
+# app.py - Enhanced Flask API with all GUI.py features
+import os
+import sys
+from datetime import datetime, timedelta
+from typing import Optional, Dict, List, Tuple
+
+import numpy as np
+import torch
 from flask import Flask, jsonify, request
-from flask_cors import CORS #type:ignore
+from flask_cors import CORS
 
 app = Flask(__name__)
 CORS(app)
 app.config["JSON_SORT_KEYS"] = False
 
+# Ensure we can import project modules in `code/`
+ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+if ROOT_DIR not in sys.path:
+    sys.path.append(ROOT_DIR)
+
+# Import project modules
+try:
+    from code import config
+    from code.DataPreprocessing import DataPreprocessing
+    from code.LSTMModel import ConvLSTMModel
+    from code.WeatherModel import WeatherModel
+    from code.TimeseriesModel import TimeseriesModel
+except Exception as e:
+    print(f"Warning: Could not import project modules: {e}")
+    ConvLSTMModel = None
+    DataPreprocessing = None
+    WeatherModel = None
+    TimeseriesModel = None
+    config = None
+
+
+# Global variables for models and data
+MODEL = None
+MODEL_DEVICE = "cpu"
+DATA_PREP = None
+WEATHER_MODEL = None
+TIMESERIES_MODEL = None
+FEATURES = None
+LABELS = None
+DATA_PIVOT = None
+CRIME_DATA = None
+NYC_SHAPE = None
+
+
+def initialize_system():
+    """
+    Initialize all models and data preprocessing.
+    This mimics the caching behavior of GUI.py's @st.cache decorators.
+    """
+    global MODEL, MODEL_DEVICE, DATA_PREP, WEATHER_MODEL, TIMESERIES_MODEL
+    global FEATURES, LABELS, DATA_PIVOT, CRIME_DATA, NYC_SHAPE
+    
+    if config is None or ConvLSTMModel is None:
+        print("Warning: Running in mock mode - models not available")
+        return False
+    
+    try:
+        # Load NYC Shape (for filtering grids not on map)
+        print("Loading NYC Shape...")
+        nyc_shape_path = os.path.join(config.PROJECT_DIR, 'Data/PreprocessedDatasets/NYCGridsShape.pkl')
+        if os.path.isfile(nyc_shape_path):
+            import pickle
+            with open(nyc_shape_path, 'rb') as file:
+                NYC_SHAPE = pickle.load(file)
+        else:
+            NYC_SHAPE = []  # Empty for custom datasets like Bengaluru
+        
+        # Load Dataset
+        print("Loading Dataset...")
+        DATA_PREP = DataPreprocessing(config.PROJECT_DIR)
+        FEATURES = DATA_PREP.features
+        LABELS = DATA_PREP.labels
+        DATA_PIVOT = DATA_PREP.dataPivot
+        CRIME_DATA = DATA_PREP.data
+        
+        # Load ConvLSTM Model
+        print("Loading ConvLSTM Model...")
+        model_path = 'BestModel.pt'
+        if os.path.isfile(model_path):
+            checkpoint = torch.load(model_path, map_location=torch.device(config.DEVICE))
+            MODEL_DEVICE = config.DEVICE
+            MODEL = ConvLSTMModel(
+                input_dim=config.CRIME_TYPE_NUM,
+                hidden_dim=config.HIDDEN_DIM,
+                kernel_size=config.KERNEL_SIZE,
+                bias=True
+            )
+            state = checkpoint.get("model") if isinstance(checkpoint, dict) else checkpoint
+            MODEL.load_state_dict(state)
+            MODEL.to(torch.device(MODEL_DEVICE))
+            MODEL.eval()
+        
+        # Load Weather Model
+        print("Loading Weather Model...")
+        WEATHER_MODEL = WeatherModel(config.PROJECT_DIR)
+        
+        # Load Timeseries Model
+        print("Loading Timeseries Model...")
+        TIMESERIES_MODEL = TimeseriesModel(config.PROJECT_DIR, CRIME_DATA)
+        
+        print("System initialization complete!")
+        return True
+        
+    except Exception as e:
+        print(f"Error during initialization: {e}")
+        import traceback
+        traceback.print_exc()
+        return False
+
+
+def get_date_range():
+    """Get valid date range for predictions."""
+    if config is None:
+        return None, None
+    
+    start_date = datetime.strptime(config.START_SELECT_DATE[1:-1], '%Y-%m-%d')
+    end_date = datetime.strptime(config.END_DATE[1:-1], '%Y-%m-%d')
+    return start_date, end_date
+
 
-def load_model():
+def validate_date(date_str: str) -> Tuple[bool, Optional[datetime], Optional[str]]:
     """
-    will load your ML model here later.
+    Validate if a date is within the valid prediction range.
+    Returns: (is_valid, datetime_object, error_message)
     """
-    return None
+    if config is None:
+        return False, None, "Configuration not available"
+    
+    try:
+        dt = datetime.strptime(date_str, '%Y-%m-%d')
+    except ValueError:
+        return False, None, "Invalid date format. Use YYYY-MM-DD"
+    
+    minus_days = config.SEQ_LEN + 1
+    start_date = datetime.strptime(config.START_DATE[1:-1], '%Y-%m-%d')
+    left_limit = start_date + timedelta(days=minus_days)
+    right_limit = datetime.strptime(config.END_DATE[1:-1], '%Y-%m-%d')
+    
+    if dt <= left_limit:
+        return False, None, f"Date must be after {left_limit.strftime('%Y-%m-%d')}"
+    elif dt > right_limit:
+        return False, None, f"Date must be before or on {right_limit.strftime('%Y-%m-%d')}"
+    
+    return True, dt, None
 
 
-MODEL = load_model()
+def get_prediction_data_by_date(
+    date: str,
+    crime_type_index: int,
+    use_temporal_factors: bool = True
+) -> Optional[Dict]:
+    """
+    Get predictions for a specific date and crime type.
+    This replicates the getPredDataByDate function from GUI.py.
+    
+    Args:
+        date: Date string in format 'YYYY-MM-DD'
+        crime_type_index: Index of crime type (0-7)
+        use_temporal_factors: Whether to apply weather and timeseries factors
+        
+    Returns:
+        Dictionary with prediction data or None on error
+    """
+    if MODEL is None or DATA_PIVOT is None or FEATURES is None:
+        return None
+    
+    # Validate date
+    is_valid, dt, error = validate_date(date)
+    if not is_valid:
+        return {"error": error}
+    
+    # Determine start index
+    minus_days = config.SEQ_LEN + 1
+    if DATA_PIVOT.query(f"date < {config.START_DATE}").shape[0] == 0:
+        start_index = 0
+    else:
+        start_index = int(
+            DATA_PIVOT.query(f"date < {config.START_DATE}").shape[0] / config.CRIME_TYPE_NUM - minus_days
+        )
+    
+    # Get feature index for the given date
+    date_query = f"'{date}'"
+    found_index = int(
+        DATA_PIVOT.query(f"date < {date_query}").shape[0] / config.CRIME_TYPE_NUM - minus_days
+    ) - start_index
+    
+    if found_index < 0 or found_index >= len(FEATURES):
+        return {"error": "Date index out of range"}
+    
+    # Get features and labels
+    features_by_date = FEATURES[found_index]
+    labels_by_date = LABELS[found_index] if LABELS is not None else None
+    
+    # Run prediction through ConvLSTM
+    processed_features = torch.from_numpy(features_by_date).to(MODEL_DEVICE).unsqueeze(0).float()
+    with torch.no_grad():
+        pred_data = MODEL(processed_features)[0][0]
+    
+    # Get temporal factors
+    weather_factor = 1.0
+    timeseries_factors = [1.0] * config.CRIME_TYPE_NUM
+    
+    if use_temporal_factors and WEATHER_MODEL is not None and TIMESERIES_MODEL is not None:
+        try:
+            weather_factor = WEATHER_MODEL.getWeatherFactor(date)
+            crime_types = [crime.lower() for crime in config.CRIME_TYPE]
+            timeseries_factors = [
+                TIMESERIES_MODEL.getTimeseriesFactor(crime_name, date)
+                for crime_name in crime_types
+            ]
+        except Exception as e:
+            print(f"Warning: Could not get temporal factors: {e}")
+    
+    return {
+        "date": date,
+        "crime_type_index": crime_type_index,
+        "prediction": pred_data.cpu().numpy(),
+        "labels": labels_by_date if labels_by_date is not None else None,
+        "weather_factor": weather_factor,
+        "timeseries_factors": timeseries_factors,
+        "use_temporal_factors": use_temporal_factors
+    }
 
 
-def predict_hotspots(city: str, threshold: float):
+def get_hexagon_data(
+    pred_data: np.ndarray,
+    weather_factor: float,
+    timeseries_factors: List[float],
+    crime_type_index: int,
+    threshold: float,
+    use_temporal_factors: bool = True
+) -> List[Dict]:
     """
-    placeholder function to simulate hotspot predictions.
+    Convert prediction data to hotspot list with lat/lon coordinates.
+    This replicates the getHexagonData function from GUI.py.
+    
+    Args:
+        pred_data: Prediction array from model
+        weather_factor: Weather adjustment factor
+        timeseries_factors: Timeseries adjustment factors per crime type
+        crime_type_index: Which crime type to extract
+        threshold: Minimum probability threshold
+        use_temporal_factors: Whether to apply temporal adjustments
+        
+    Returns:
+        List of hotspot dictionaries
     """
+    if NYC_SHAPE is None or config is None:
+        return []
+    
+    hotspots = []
+    hotspot_id = 0
+    
+    for x in range(pred_data.shape[1]):
+        for y in range(pred_data.shape[2]):
+            # Skip grids not on the map (NYC shape filtering)
+            if (((x, y) in NYC_SHAPE or (x+1, y) in NYC_SHAPE or 
+                 (x, y+1) in NYC_SHAPE or (x+1, y+1) in NYC_SHAPE) and 
+                (x < config.LAT_GRIDS - 1 and y < config.LON_GRIDS - 1)):
+                continue
+            
+            if x >= config.LAT_GRIDS - 1 or y >= config.LON_GRIDS - 1:
+                continue
+            
+            # Get base weight from prediction
+            weight = float(pred_data[crime_type_index][x][y])
+            
+            # Apply temporal factors if enabled
+            if use_temporal_factors:
+                weight = weight * weather_factor * timeseries_factors[crime_type_index]
+            
+            # Apply threshold multiplier for values below threshold
+            if weight < threshold:
+                weight = weight * config.MULTIPLY_FACTOR
+            
+            # Calculate lat/lon for this grid cell
+            lat = config.LAT_BINS[x] + config.DIFF_LAT
+            lon = config.LON_BINS[y] + config.DIFF_LON
+            
+            # Only include if above absolute minimum
+            if weight > 0.01:
+                risk_level = "high" if weight >= 0.75 else "medium" if weight >= 0.6 else "low"
+                
+                hotspots.append({
+                    "id": f"hotspot-{hotspot_id}",
+                    "latitude": float(lat),
+                    "longitude": float(lon),
+                    "risk": float(weight),
+                    "riskLevel": risk_level,
+                    "crimeCount": int(weight * 50) + 10,
+                })
+                hotspot_id += 1
+    
+    return hotspots
 
+
+def get_mock_hotspots(city: str, threshold: float) -> List[Dict]:
+    """Fallback mock data when models are not available."""
     mock_data = {
         "bangalore": [
             {"lat": 12.9352, "lon": 77.6245, "risk": 0.85},
             {"lat": 12.9716, "lon": 77.5946, "risk": 0.72},
             {"lat": 13.0027, "lon": 77.5914, "risk": 0.61},
+            {"lat": 12.9141, "lon": 77.6411, "risk": 0.78},
+            {"lat": 12.9698, "lon": 77.6489, "risk": 0.65},
         ],
         "delhi": [
             {"lat": 28.7041, "lon": 77.1025, "risk": 0.89},
             {"lat": 28.6328, "lon": 77.2197, "risk": 0.76},
+            {"lat": 28.5355, "lon": 77.3910, "risk": 0.68},
         ],
     }
-
+    
     city_data = mock_data.get(city.lower(), mock_data["bangalore"])
-
     results = []
     for i, point in enumerate(city_data):
         if point["risk"] >= threshold:
@@ -43,6 +324,7 @@ def predict_hotspots(city: str, threshold: float):
                 "id": f"{city}-hotspot-{i}",
                 "latitude": point["lat"],
                 "longitude": point["lon"],
+                "risk": point["risk"],
                 "riskLevel": (
                     "high" if point["risk"] >= 0.75
                     else "medium" if point["risk"] >= 0.6
@@ -50,24 +332,186 @@ def predict_hotspots(city: str, threshold: float):
                 ),
                 "crimeCount": int(point["risk"] * 50) + 10,
             })
-
     return results
 
+
+# ==================== API ENDPOINTS ====================
+
 @app.route("/api/health", methods=["GET"])
 def health():
+    """Health check endpoint."""
+    models_loaded = MODEL is not None and DATA_PREP is not None
     return jsonify({
         "status": "ok",
-        "timestamp": datetime.utcnow().isoformat()
+        "timestamp": datetime.utcnow().isoformat(),
+        "models_loaded": models_loaded,
+        "weather_model": WEATHER_MODEL is not None,
+        "timeseries_model": TIMESERIES_MODEL is not None,
+    })
+
+
+@app.route("/api/info", methods=["GET"])
+def info():
+    """Get system information and available date range."""
+    if config is None:
+        return jsonify({"error": "Configuration not available"}), 500
+    
+    start_date, end_date = get_date_range()
+    crime_types = [crime.lower() for crime in config.CRIME_TYPE]
+    
+    return jsonify({
+        "date_range": {
+            "start": start_date.strftime('%Y-%m-%d') if start_date else None,
+            "end": end_date.strftime('%Y-%m-%d') if end_date else None,
+        },
+        "crime_types": crime_types,
+        "grid_info": {
+            "lat_min": float(config.LAT_MIN),
+            "lat_max": float(config.LAT_MAX),
+            "lon_min": float(config.LON_MIN),
+            "lon_max": float(config.LON_MAX),
+            "lat_grids": config.LAT_GRIDS,
+            "lon_grids": config.LON_GRIDS,
+        },
+        "model_info": {
+            "seq_len": config.SEQ_LEN,
+            "hidden_dim": config.HIDDEN_DIM,
+            "kernel_size": config.KERNEL_SIZE,
+        }
+    })
+
+
+@app.route("/api/crime-types", methods=["GET"])
+def crime_types():
+    """Get list of available crime types."""
+    if config is None:
+        return jsonify({"error": "Configuration not available"}), 500
+    
+    crime_types_list = [crime.lower() for crime in config.CRIME_TYPE]
+    return jsonify({
+        "crime_types": crime_types_list,
+        "count": len(crime_types_list)
     })
 
 
+@app.route("/api/predict", methods=["POST"])
+def predict():
+    """
+    Advanced prediction endpoint with full temporal factors.
+    
+    Request body:
+    {
+        "date": "2024-01-15",
+        "crime_type": "theft",  // or crime_type_index
+        "threshold": 0.5,
+        "use_temporal_factors": true
+    }
+    """
+    if MODEL is None or DATA_PREP is None:
+        # Fallback to mock data
+        data = request.get_json() or {}
+        city = data.get("city", "bangalore")
+        threshold = float(data.get("threshold", 0.5))
+        return jsonify({
+            "city": city,
+            "threshold": threshold,
+            "hotspots": get_mock_hotspots(city, threshold),
+            "mock": True
+        })
+    
+    try:
+        data = request.get_json() or {}
+        
+        # Parse parameters
+        date = data.get("date")
+        if not date:
+            return jsonify({"error": "Date parameter required"}), 400
+        
+        # Get crime type index
+        crime_type = data.get("crime_type")
+        crime_type_index = data.get("crime_type_index")
+        
+        if crime_type is not None:
+            crime_types = [crime.lower() for crime in config.CRIME_TYPE]
+            try:
+                crime_type_index = crime_types.index(crime_type.lower())
+            except ValueError:
+                return jsonify({"error": f"Invalid crime type. Available: {crime_types}"}), 400
+        elif crime_type_index is None:
+            crime_type_index = 0  # Default to first crime type
+        
+        threshold = float(data.get("threshold", 0.5))
+        use_temporal_factors = data.get("use_temporal_factors", True)
+        
+        # Get prediction data
+        pred_result = get_prediction_data_by_date(
+            date=date,
+            crime_type_index=crime_type_index,
+            use_temporal_factors=use_temporal_factors
+        )
+        
+        if pred_result is None or "error" in pred_result:
+            return jsonify(pred_result or {"error": "Prediction failed"}), 400
+        
+        # Convert to hotspots
+        hotspots = get_hexagon_data(
+            pred_data=pred_result["prediction"],
+            weather_factor=pred_result["weather_factor"],
+            timeseries_factors=pred_result["timeseries_factors"],
+            crime_type_index=crime_type_index,
+            threshold=threshold,
+            use_temporal_factors=use_temporal_factors
+        )
+        
+        crime_types = [crime.lower() for crime in config.CRIME_TYPE]
+        
+        return jsonify({
+            "date": date,
+            "crime_type": crime_types[crime_type_index],
+            "crime_type_index": crime_type_index,
+            "threshold": threshold,
+            "use_temporal_factors": use_temporal_factors,
+            "temporal_factors": {
+                "weather": float(pred_result["weather_factor"]),
+                "timeseries": [float(f) for f in pred_result["timeseries_factors"]],
+            },
+            "count": len(hotspots),
+            "hotspots": hotspots,
+        })
+        
+    except Exception as e:
+        import traceback
+        traceback.print_exc()
+        return jsonify({"error": str(e)}), 500
+
+
 @app.route("/api/hotspots", methods=["GET"])
 def get_hotspots():
+    """
+    Simple hotspot endpoint (backward compatible).
+    Query params: city, threshold
+    """
     city = request.args.get("city", "bangalore")
     threshold = float(request.args.get("threshold", 0.5))
-
-    hotspots = predict_hotspots(city, threshold)
-
+    
+    if MODEL is None:
+        hotspots = get_mock_hotspots(city, threshold)
+    else:
+        # Use latest available date
+        date = datetime.strptime(config.END_DATE[1:-1], '%Y-%m-%d').strftime('%Y-%m-%d')
+        pred_result = get_prediction_data_by_date(date=date, crime_type_index=0)
+        
+        if pred_result and "error" not in pred_result:
+            hotspots = get_hexagon_data(
+                pred_data=pred_result["prediction"],
+                weather_factor=pred_result["weather_factor"],
+                timeseries_factors=pred_result["timeseries_factors"],
+                crime_type_index=0,
+                threshold=threshold
+            )
+        else:
+            hotspots = get_mock_hotspots(city, threshold)
+    
     return jsonify({
         "city": city,
         "threshold": threshold,
@@ -76,12 +520,133 @@ def get_hotspots():
     })
 
 
+@app.route("/api/cumulative", methods=["GET"])
+def cumulative_heatmap():
+    """
+    Get cumulative crime data for heatmap visualization.
+    This replicates the "Cumulative Heatmap (All Data)" mode from GUI.py.
+    
+    Query params:
+    - crime_types: comma-separated list (optional, defaults to all)
+    - lat_min, lat_max, lon_min, lon_max: bounding box (optional)
+    """
+    if CRIME_DATA is None or config is None:
+        return jsonify({"error": "Crime data not available"}), 500
+    
+    try:
+        # Parse crime type filter
+        crime_types_param = request.args.get("crime_types")
+        if crime_types_param:
+            selected_types = [t.strip() for t in crime_types_param.split(",")]
+        else:
+            selected_types = CRIME_DATA['TYPE'].unique().tolist()
+        
+        # Parse bounding box
+        lat_min = float(request.args.get("lat_min", config.LAT_MIN))
+        lat_max = float(request.args.get("lat_max", config.LAT_MAX))
+        lon_min = float(request.args.get("lon_min", config.LON_MIN))
+        lon_max = float(request.args.get("lon_max", config.LON_MAX))
+        
+        # Filter data
+        filtered_data = CRIME_DATA[
+            (CRIME_DATA['Longitude'] >= lon_min) &
+            (CRIME_DATA['Longitude'] <= lon_max) &
+            (CRIME_DATA['Latitude'] >= lat_min) &
+            (CRIME_DATA['Latitude'] <= lat_max) &
+            (CRIME_DATA['TYPE'].isin(selected_types))
+        ]
+        
+        # Convert to list of points
+        points = []
+        for _, row in filtered_data.iterrows():
+            points.append({
+                "latitude": float(row['Latitude']),
+                "longitude": float(row['Longitude']),
+                "type": row['TYPE'],
+                "date": row.get('Date', None),
+            })
+        
+        return jsonify({
+            "crime_types": selected_types,
+            "bounds": {
+                "lat_min": lat_min,
+                "lat_max": lat_max,
+                "lon_min": lon_min,
+                "lon_max": lon_max,
+            },
+            "count": len(points),
+            "points": points,
+        })
+        
+    except Exception as e:
+        import traceback
+        traceback.print_exc()
+        return jsonify({"error": str(e)}), 500
+
+
+@app.route("/api/temporal-factors", methods=["GET"])
+def temporal_factors():
+    """
+    Get weather and timeseries factors for a specific date.
+    
+    Query params:
+    - date: Date in YYYY-MM-DD format
+    """
+    if WEATHER_MODEL is None or TIMESERIES_MODEL is None or config is None:
+        return jsonify({"error": "Temporal models not available"}), 500
+    
+    date = request.args.get("date")
+    if not date:
+        return jsonify({"error": "Date parameter required"}), 400
+    
+    # Validate date
+    is_valid, dt, error = validate_date(date)
+    if not is_valid:
+        return jsonify({"error": error}), 400
+    
+    try:
+        weather_factor = WEATHER_MODEL.getWeatherFactor(date)
+        crime_types = [crime.lower() for crime in config.CRIME_TYPE]
+        timeseries_factors = [
+            TIMESERIES_MODEL.getTimeseriesFactor(crime_name, date)
+            for crime_name in crime_types
+        ]
+        
+        return jsonify({
+            "date": date,
+            "weather_factor": float(weather_factor),
+            "timeseries_factors": {
+                crime_types[i]: float(timeseries_factors[i])
+                for i in range(len(crime_types))
+            }
+        })
+        
+    except Exception as e:
+        return jsonify({"error": str(e)}), 500
+
+
 @app.route("/api/predictions", methods=["GET"])
 def predictions():
+    """Legacy endpoint for backward compatibility."""
     city = request.args.get("city", "bangalore")
-
-    data = predict_hotspots(city, threshold=0.0)
-
+    
+    if MODEL is None:
+        data = get_mock_hotspots(city, threshold=0.0)
+    else:
+        date = datetime.strptime(config.END_DATE[1:-1], '%Y-%m-%d').strftime('%Y-%m-%d')
+        pred_result = get_prediction_data_by_date(date=date, crime_type_index=0)
+        
+        if pred_result and "error" not in pred_result:
+            data = get_hexagon_data(
+                pred_data=pred_result["prediction"],
+                weather_factor=pred_result["weather_factor"],
+                timeseries_factors=pred_result["timeseries_factors"],
+                crime_type_index=0,
+                threshold=0.0
+            )
+        else:
+            data = get_mock_hotspots(city, 0.0)
+    
     return jsonify({
         "city": city,
         "timestamp": datetime.utcnow().isoformat(),
@@ -91,7 +656,25 @@ def predictions():
 
 @app.errorhandler(404)
 def not_found(_):
-    return jsonify({"error": "not found"}), 404
+    return jsonify({"error": "Not found"}), 404
+
+
+@app.errorhandler(500)
+def internal_error(error):
+    return jsonify({"error": "Internal server error"}), 500
+
+
+# Initialize system on startup
+print("=" * 60)
+print("Initializing Crime Hotspot Prediction System...")
+print("=" * 60)
+initialization_success = initialize_system()
+if initialization_success:
+    print("✓ System ready!")
+else:
+    print("⚠ Running in mock mode - some features unavailable")
+print("=" * 60)
+
 
 if __name__ == "__main__":
-    app.run(host="0.0.0.0", port=7860)
+    app.run(host="0.0.0.0", port=5000, debug=True)