Spaces:
Running
Running
Upload 9 files
Browse files- .gitattributes +35 -35
- Dockerfile +8 -0
- README.md +10 -10
- alert_config.py +177 -0
- alerts.py +417 -0
- ml_api.py +185 -0
- ml_inference.py +813 -0
- requirements.txt +2 -0
- sensor_buffer.py +97 -0
.gitattributes
CHANGED
|
@@ -1,35 +1,35 @@
|
|
| 1 |
-
*.7z filter=lfs diff=lfs merge=lfs -text
|
| 2 |
-
*.arrow filter=lfs diff=lfs merge=lfs -text
|
| 3 |
-
*.bin filter=lfs diff=lfs merge=lfs -text
|
| 4 |
-
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
| 5 |
-
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
| 6 |
-
*.ftz filter=lfs diff=lfs merge=lfs -text
|
| 7 |
-
*.gz filter=lfs diff=lfs merge=lfs -text
|
| 8 |
-
*.h5 filter=lfs diff=lfs merge=lfs -text
|
| 9 |
-
*.joblib filter=lfs diff=lfs merge=lfs -text
|
| 10 |
-
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
| 11 |
-
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
| 12 |
-
*.model filter=lfs diff=lfs merge=lfs -text
|
| 13 |
-
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
| 14 |
-
*.npy filter=lfs diff=lfs merge=lfs -text
|
| 15 |
-
*.npz filter=lfs diff=lfs merge=lfs -text
|
| 16 |
-
*.onnx filter=lfs diff=lfs merge=lfs -text
|
| 17 |
-
*.ot filter=lfs diff=lfs merge=lfs -text
|
| 18 |
-
*.parquet filter=lfs diff=lfs merge=lfs -text
|
| 19 |
-
*.pb filter=lfs diff=lfs merge=lfs -text
|
| 20 |
-
*.pickle filter=lfs diff=lfs merge=lfs -text
|
| 21 |
-
*.pkl filter=lfs diff=lfs merge=lfs -text
|
| 22 |
-
*.pt filter=lfs diff=lfs merge=lfs -text
|
| 23 |
-
*.pth filter=lfs diff=lfs merge=lfs -text
|
| 24 |
-
*.rar filter=lfs diff=lfs merge=lfs -text
|
| 25 |
-
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
| 26 |
-
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
| 27 |
-
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
| 28 |
-
*.tar filter=lfs diff=lfs merge=lfs -text
|
| 29 |
-
*.tflite filter=lfs diff=lfs merge=lfs -text
|
| 30 |
-
*.tgz filter=lfs diff=lfs merge=lfs -text
|
| 31 |
-
*.wasm filter=lfs diff=lfs merge=lfs -text
|
| 32 |
-
*.xz filter=lfs diff=lfs merge=lfs -text
|
| 33 |
-
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
-
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
-
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
| 1 |
+
*.7z filter=lfs diff=lfs merge=lfs -text
|
| 2 |
+
*.arrow filter=lfs diff=lfs merge=lfs -text
|
| 3 |
+
*.bin filter=lfs diff=lfs merge=lfs -text
|
| 4 |
+
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
| 5 |
+
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
| 6 |
+
*.ftz filter=lfs diff=lfs merge=lfs -text
|
| 7 |
+
*.gz filter=lfs diff=lfs merge=lfs -text
|
| 8 |
+
*.h5 filter=lfs diff=lfs merge=lfs -text
|
| 9 |
+
*.joblib filter=lfs diff=lfs merge=lfs -text
|
| 10 |
+
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
| 11 |
+
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
| 12 |
+
*.model filter=lfs diff=lfs merge=lfs -text
|
| 13 |
+
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
| 14 |
+
*.npy filter=lfs diff=lfs merge=lfs -text
|
| 15 |
+
*.npz filter=lfs diff=lfs merge=lfs -text
|
| 16 |
+
*.onnx filter=lfs diff=lfs merge=lfs -text
|
| 17 |
+
*.ot filter=lfs diff=lfs merge=lfs -text
|
| 18 |
+
*.parquet filter=lfs diff=lfs merge=lfs -text
|
| 19 |
+
*.pb filter=lfs diff=lfs merge=lfs -text
|
| 20 |
+
*.pickle filter=lfs diff=lfs merge=lfs -text
|
| 21 |
+
*.pkl filter=lfs diff=lfs merge=lfs -text
|
| 22 |
+
*.pt filter=lfs diff=lfs merge=lfs -text
|
| 23 |
+
*.pth filter=lfs diff=lfs merge=lfs -text
|
| 24 |
+
*.rar filter=lfs diff=lfs merge=lfs -text
|
| 25 |
+
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
| 26 |
+
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
| 27 |
+
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
| 28 |
+
*.tar filter=lfs diff=lfs merge=lfs -text
|
| 29 |
+
*.tflite filter=lfs diff=lfs merge=lfs -text
|
| 30 |
+
*.tgz filter=lfs diff=lfs merge=lfs -text
|
| 31 |
+
*.wasm filter=lfs diff=lfs merge=lfs -text
|
| 32 |
+
*.xz filter=lfs diff=lfs merge=lfs -text
|
| 33 |
+
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
+
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
+
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
Dockerfile
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
FROM python:3.11-slim
|
| 2 |
+
WORKDIR /app
|
| 3 |
+
COPY requirements.txt .
|
| 4 |
+
RUN pip install --no-cache-dir -r requirements.txt
|
| 5 |
+
COPY . .
|
| 6 |
+
EXPOSE 7860
|
| 7 |
+
ENV PYTHONUNBUFFERED=1
|
| 8 |
+
CMD ["python", "ml_api.py"]
|
README.md
CHANGED
|
@@ -1,10 +1,10 @@
|
|
| 1 |
-
---
|
| 2 |
-
title: Swat Ml Api
|
| 3 |
-
emoji: 💻
|
| 4 |
-
colorFrom: green
|
| 5 |
-
colorTo: yellow
|
| 6 |
-
sdk: docker
|
| 7 |
-
pinned: false
|
| 8 |
-
---
|
| 9 |
-
|
| 10 |
-
Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
|
|
|
|
| 1 |
+
---
|
| 2 |
+
title: Swat Ml Api
|
| 3 |
+
emoji: 💻
|
| 4 |
+
colorFrom: green
|
| 5 |
+
colorTo: yellow
|
| 6 |
+
sdk: docker
|
| 7 |
+
pinned: false
|
| 8 |
+
---
|
| 9 |
+
|
| 10 |
+
Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
|
alert_config.py
ADDED
|
@@ -0,0 +1,177 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Alert System Configuration
|
| 3 |
+
==========================
|
| 4 |
+
Store all alert settings and credentials here.
|
| 5 |
+
|
| 6 |
+
SECURITY: Never commit this file to Git!
|
| 7 |
+
Add to .gitignore: alert_config.py
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
# ============================================================================
|
| 11 |
+
# EMAIL CONFIGURATION (Gmail)
|
| 12 |
+
# ============================================================================
|
| 13 |
+
|
| 14 |
+
EMAIL_CONFIG = {
|
| 15 |
+
'enabled': False, # Set to False to disable email alerts
|
| 16 |
+
|
| 17 |
+
# Gmail SMTP settings
|
| 18 |
+
'smtp_server': 'smtp.gmail.com',
|
| 19 |
+
'smtp_port': 587,
|
| 20 |
+
|
| 21 |
+
# Your Gmail account
|
| 22 |
+
'sender_email': 'fypmlalerts@gmail.com', # CHANGE THIS
|
| 23 |
+
'sender_password': 'ahgm aryb wzbk opib', # CHANGE THIS (16-char app password)
|
| 24 |
+
|
| 25 |
+
# Who receives alerts
|
| 26 |
+
'recipient_emails': [
|
| 27 |
+
'ariadaikalam1234@gmail.com',
|
| 28 |
+
]
|
| 29 |
+
}
|
| 30 |
+
|
| 31 |
+
# ============================================================================
|
| 32 |
+
# SMS CONFIGURATION (Twilio)
|
| 33 |
+
# ============================================================================
|
| 34 |
+
|
| 35 |
+
SMS_CONFIG = {
|
| 36 |
+
'enabled': False, # Set to False to disable SMS alerts
|
| 37 |
+
|
| 38 |
+
# Twilio credentials (from https://console.twilio.com)
|
| 39 |
+
'twilio_account_sid': 'AC4392144e5e0fe6133a36928d5e8032db', # CHANGE THIS
|
| 40 |
+
'twilio_auth_token': 'dc97b81a11fc876f38ee3e61c2522ca1', # CHANGE THIS
|
| 41 |
+
'twilio_phone_number': '+14234583614', # CHANGE THIS
|
| 42 |
+
|
| 43 |
+
# Who receives SMS (must be verified on free tier)
|
| 44 |
+
'recipient_phones': [
|
| 45 |
+
'+916382043877', # CHANGE THIS
|
| 46 |
+
]
|
| 47 |
+
}
|
| 48 |
+
|
| 49 |
+
# ============================================================================
|
| 50 |
+
# PHONE CALL CONFIGURATION (Twilio)
|
| 51 |
+
# ============================================================================
|
| 52 |
+
|
| 53 |
+
CALL_CONFIG = {
|
| 54 |
+
'enabled': False, # Set to False to disable phone calls
|
| 55 |
+
|
| 56 |
+
# Twilio credentials (same as SMS)
|
| 57 |
+
'twilio_account_sid': SMS_CONFIG['twilio_account_sid'],
|
| 58 |
+
'twilio_auth_token': SMS_CONFIG['twilio_auth_token'],
|
| 59 |
+
'twilio_phone_number': SMS_CONFIG['twilio_phone_number'],
|
| 60 |
+
|
| 61 |
+
# Who receives calls (emergency contact only)
|
| 62 |
+
'recipient_phones': [
|
| 63 |
+
'+916382043877', # CHANGE THIS
|
| 64 |
+
],
|
| 65 |
+
|
| 66 |
+
# Only call if confidence is this high
|
| 67 |
+
'confidence_threshold': 0.90 # 90%
|
| 68 |
+
}
|
| 69 |
+
|
| 70 |
+
# ============================================================================
|
| 71 |
+
# ALERT BEHAVIOR
|
| 72 |
+
# ============================================================================
|
| 73 |
+
|
| 74 |
+
ALERT_BEHAVIOR = {
|
| 75 |
+
# Cooldown period (seconds) - prevents alert spam
|
| 76 |
+
# If alert sent at 10:00:00, next alert won't send until 10:05:00
|
| 77 |
+
'cooldown_seconds': 180, # 5 minutes
|
| 78 |
+
'degrading_persist_seconds': 30,
|
| 79 |
+
'faulted_persist_seconds': 15,
|
| 80 |
+
|
| 81 |
+
# Send alerts for these states
|
| 82 |
+
'alert_on_degrading': True, # Email only
|
| 83 |
+
'alert_on_faulted': True, # Email + SMS + Call (if high confidence)
|
| 84 |
+
|
| 85 |
+
# Test mode: Print instead of actually sending
|
| 86 |
+
'test_mode': False, # Set to False for production
|
| 87 |
+
}
|
| 88 |
+
|
| 89 |
+
# ============================================================================
|
| 90 |
+
# ALERT TEMPLATES
|
| 91 |
+
# ============================================================================
|
| 92 |
+
|
| 93 |
+
EMAIL_TEMPLATES = {
|
| 94 |
+
'subject': "⚠️ SWAT Alert: {component} - {state}",
|
| 95 |
+
|
| 96 |
+
'body_html': """
|
| 97 |
+
<html>
|
| 98 |
+
<head>
|
| 99 |
+
<style>
|
| 100 |
+
body {{ font-family: Arial, sans-serif; }}
|
| 101 |
+
.header {{ background: {color}; color: white; padding: 20px; border-radius: 8px; }}
|
| 102 |
+
.content {{ padding: 20px; }}
|
| 103 |
+
.metric {{ margin: 10px 0; }}
|
| 104 |
+
.label {{ font-weight: bold; }}
|
| 105 |
+
.footer {{ margin-top: 30px; color: #666; font-size: 12px; }}
|
| 106 |
+
</style>
|
| 107 |
+
</head>
|
| 108 |
+
<body>
|
| 109 |
+
<div class="header">
|
| 110 |
+
<h2>{icon} {alert_title}</h2>
|
| 111 |
+
</div>
|
| 112 |
+
<div class="content">
|
| 113 |
+
<div class="metric">
|
| 114 |
+
<span class="label">Component:</span> {component}
|
| 115 |
+
</div>
|
| 116 |
+
<div class="metric">
|
| 117 |
+
<span class="label">State:</span> {state}
|
| 118 |
+
</div>
|
| 119 |
+
<div class="metric">
|
| 120 |
+
<span class="label">Confidence:</span> {confidence}%
|
| 121 |
+
</div>
|
| 122 |
+
<div class="metric">
|
| 123 |
+
<span class="label">Timestamp:</span> {timestamp}
|
| 124 |
+
</div>
|
| 125 |
+
|
| 126 |
+
<h3>Top 3 Suspect Components:</h3>
|
| 127 |
+
<ul>
|
| 128 |
+
{top3_list}
|
| 129 |
+
</ul>
|
| 130 |
+
|
| 131 |
+
<h3>Recommended Actions:</h3>
|
| 132 |
+
<ol>
|
| 133 |
+
{actions_list}
|
| 134 |
+
</ol>
|
| 135 |
+
</div>
|
| 136 |
+
<div class="footer">
|
| 137 |
+
<em>This is an automated alert from SWAT Predictive Maintenance System</em>
|
| 138 |
+
</div>
|
| 139 |
+
</body>
|
| 140 |
+
</html>
|
| 141 |
+
"""
|
| 142 |
+
}
|
| 143 |
+
|
| 144 |
+
SMS_TEMPLATE = """
|
| 145 |
+
🚨 SWAT CRITICAL FAULT 🚨
|
| 146 |
+
|
| 147 |
+
Component: {component}
|
| 148 |
+
State: {state}
|
| 149 |
+
Confidence: {confidence}%
|
| 150 |
+
Time: {time}
|
| 151 |
+
|
| 152 |
+
IMMEDIATE ACTION REQUIRED
|
| 153 |
+
Check dashboard for details.
|
| 154 |
+
"""
|
| 155 |
+
|
| 156 |
+
CALL_SCRIPT = """
|
| 157 |
+
<Response>
|
| 158 |
+
<Say voice="alice">
|
| 159 |
+
Critical fault detected in SWAT system.
|
| 160 |
+
Component {component} has failed.
|
| 161 |
+
Confidence level: {confidence} percent.
|
| 162 |
+
Immediate maintenance required.
|
| 163 |
+
Please check the dashboard for details.
|
| 164 |
+
</Say>
|
| 165 |
+
<Pause length="2"/>
|
| 166 |
+
<Say voice="alice">
|
| 167 |
+
Repeating: Critical fault in component {component}.
|
| 168 |
+
Check the dashboard immediately.
|
| 169 |
+
</Say>
|
| 170 |
+
</Response>
|
| 171 |
+
"""
|
| 172 |
+
|
| 173 |
+
# ============================================================================
|
| 174 |
+
# SECURITY NOTE
|
| 175 |
+
# ============================================================================
|
| 176 |
+
# Add this file to .gitignore:
|
| 177 |
+
# echo "alert_config.py" >> .gitignore
|
alerts.py
ADDED
|
@@ -0,0 +1,417 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Alert System Implementation
|
| 3 |
+
===========================
|
| 4 |
+
Handles email, SMS, and phone call alerts.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
import smtplib
|
| 8 |
+
from email.mime.text import MIMEText
|
| 9 |
+
from email.mime.multipart import MIMEMultipart
|
| 10 |
+
from datetime import datetime
|
| 11 |
+
import time
|
| 12 |
+
|
| 13 |
+
# Import configuration
|
| 14 |
+
from alert_config import (
|
| 15 |
+
EMAIL_CONFIG, SMS_CONFIG, CALL_CONFIG,
|
| 16 |
+
ALERT_BEHAVIOR, EMAIL_TEMPLATES, SMS_TEMPLATE, CALL_SCRIPT
|
| 17 |
+
)
|
| 18 |
+
|
| 19 |
+
# Try to import Twilio
|
| 20 |
+
try:
|
| 21 |
+
from twilio.rest import Client
|
| 22 |
+
|
| 23 |
+
TWILIO_AVAILABLE = True
|
| 24 |
+
except ImportError:
|
| 25 |
+
TWILIO_AVAILABLE = False
|
| 26 |
+
print("⚠️ Twilio not installed. SMS/Calls disabled. Install: pip install twilio")
|
| 27 |
+
|
| 28 |
+
# ============================================================================
|
| 29 |
+
# COOLDOWN TRACKING
|
| 30 |
+
# ============================================================================
|
| 31 |
+
|
| 32 |
+
_last_alert_time = {} # Track last alert time per component
|
| 33 |
+
_state_started_at = {} # key: f"{component}_{state}" -> epoch seconds
|
| 34 |
+
|
| 35 |
+
def state_persisted_long_enough(component, state):
|
| 36 |
+
"""
|
| 37 |
+
Returns True only if the current state has been continuous long enough.
|
| 38 |
+
Resets other state timers for that component automatically.
|
| 39 |
+
"""
|
| 40 |
+
if not component or state in (None, "NORMAL", "UNKNOWN", "ERROR"):
|
| 41 |
+
return False
|
| 42 |
+
|
| 43 |
+
now = time.time()
|
| 44 |
+
|
| 45 |
+
# Reset timers for this component for other states (prevents stale timers)
|
| 46 |
+
for key in list(_state_started_at.keys()):
|
| 47 |
+
if key.startswith(component + "_") and key != f"{component}_{state}":
|
| 48 |
+
_state_started_at.pop(key, None)
|
| 49 |
+
|
| 50 |
+
k = f"{component}_{state}"
|
| 51 |
+
if k not in _state_started_at:
|
| 52 |
+
_state_started_at[k] = now
|
| 53 |
+
#print(f"🧪 {state} started: {component}")
|
| 54 |
+
return False
|
| 55 |
+
|
| 56 |
+
elapsed = now - _state_started_at[k]
|
| 57 |
+
|
| 58 |
+
if state == "DEGRADING":
|
| 59 |
+
required = ALERT_BEHAVIOR.get("degrading_persist_seconds", 60)
|
| 60 |
+
elif state == "FAULTED":
|
| 61 |
+
required = ALERT_BEHAVIOR.get("faulted_persist_seconds", 0)
|
| 62 |
+
else:
|
| 63 |
+
required = 0
|
| 64 |
+
|
| 65 |
+
if elapsed < required:
|
| 66 |
+
#print(f"🧪 Waiting persistence: {component} {state} {int(elapsed)}/{required}s")
|
| 67 |
+
return False
|
| 68 |
+
|
| 69 |
+
return True
|
| 70 |
+
|
| 71 |
+
def reset_component_timers(component):
|
| 72 |
+
if not component:
|
| 73 |
+
return
|
| 74 |
+
for key in list(_state_started_at.keys()):
|
| 75 |
+
if key.startswith(component + "_"):
|
| 76 |
+
_state_started_at.pop(key, None)
|
| 77 |
+
|
| 78 |
+
def check_cooldown(component, alert_type, state):
|
| 79 |
+
key = f"{component}_{state}_{alert_type}"
|
| 80 |
+
current_time = time.time()
|
| 81 |
+
|
| 82 |
+
if key in _last_alert_time:
|
| 83 |
+
elapsed = current_time - _last_alert_time[key]
|
| 84 |
+
if elapsed < ALERT_BEHAVIOR['cooldown_seconds']:
|
| 85 |
+
#print(f"⏳ Cooldown: {component} {state} ({alert_type}) - {int(elapsed)}s ago")
|
| 86 |
+
return False
|
| 87 |
+
|
| 88 |
+
_last_alert_time[key] = current_time
|
| 89 |
+
return True
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
# ============================================================================
|
| 93 |
+
# EMAIL ALERTS
|
| 94 |
+
# ============================================================================
|
| 95 |
+
|
| 96 |
+
def send_email_alert(prediction):
|
| 97 |
+
"""
|
| 98 |
+
Send email alert for DEGRADING or FAULTED state.
|
| 99 |
+
|
| 100 |
+
Args:
|
| 101 |
+
prediction (dict): ML prediction result
|
| 102 |
+
|
| 103 |
+
Returns:
|
| 104 |
+
bool: True if sent successfully
|
| 105 |
+
"""
|
| 106 |
+
if not EMAIL_CONFIG['enabled']:
|
| 107 |
+
#print("📧 Email alerts disabled in config")
|
| 108 |
+
return False
|
| 109 |
+
|
| 110 |
+
state = prediction['stage2']['state']
|
| 111 |
+
component = prediction['stage3']['component']
|
| 112 |
+
confidence = prediction['stage3']['confidence']
|
| 113 |
+
|
| 114 |
+
if not component:
|
| 115 |
+
return False
|
| 116 |
+
|
| 117 |
+
# Check cooldown
|
| 118 |
+
if not check_cooldown(component, 'EMAIL', state):
|
| 119 |
+
return False
|
| 120 |
+
|
| 121 |
+
# Test mode: Just print
|
| 122 |
+
if ALERT_BEHAVIOR['test_mode']:
|
| 123 |
+
print(f"📧 [TEST MODE] Would send email: {component} - {state} ({confidence * 100:.0f}%)")
|
| 124 |
+
return True
|
| 125 |
+
|
| 126 |
+
try:
|
| 127 |
+
# Create message
|
| 128 |
+
msg = MIMEMultipart('alternative')
|
| 129 |
+
|
| 130 |
+
# Subject
|
| 131 |
+
subject = EMAIL_TEMPLATES['subject'].format(
|
| 132 |
+
component=component,
|
| 133 |
+
state=state
|
| 134 |
+
)
|
| 135 |
+
msg['Subject'] = subject
|
| 136 |
+
msg['From'] = EMAIL_CONFIG['sender_email']
|
| 137 |
+
msg['To'] = ', '.join(EMAIL_CONFIG['recipient_emails'])
|
| 138 |
+
|
| 139 |
+
# Determine color and icon
|
| 140 |
+
if state == "FAULTED":
|
| 141 |
+
color = "#dc3545"
|
| 142 |
+
icon = "🔴"
|
| 143 |
+
alert_title = "CRITICAL FAULT DETECTED"
|
| 144 |
+
else:
|
| 145 |
+
color = "#ffc107"
|
| 146 |
+
icon = "🟡"
|
| 147 |
+
alert_title = "DEGRADATION WARNING"
|
| 148 |
+
|
| 149 |
+
# Build top 3 list
|
| 150 |
+
top3 = prediction['stage3'].get('top3', [])
|
| 151 |
+
top3_html = ''.join([
|
| 152 |
+
f"<li>{comp}: {conf * 100:.1f}%</li>"
|
| 153 |
+
for comp, conf in top3
|
| 154 |
+
])
|
| 155 |
+
|
| 156 |
+
# Build actions list
|
| 157 |
+
actions = prediction.get('actions', [])[:5] # Top 5 actions
|
| 158 |
+
actions_html = ''.join([
|
| 159 |
+
f"<li>{action}</li>"
|
| 160 |
+
for action in actions
|
| 161 |
+
])
|
| 162 |
+
|
| 163 |
+
# Format HTML body
|
| 164 |
+
html = EMAIL_TEMPLATES['body_html'].format(
|
| 165 |
+
color=color,
|
| 166 |
+
icon=icon,
|
| 167 |
+
alert_title=alert_title,
|
| 168 |
+
component=component,
|
| 169 |
+
state=state,
|
| 170 |
+
confidence=confidence * 100,
|
| 171 |
+
timestamp=prediction['timestamp'].strftime('%Y-%m-%d %H:%M:%S'),
|
| 172 |
+
top3_list=top3_html,
|
| 173 |
+
actions_list=actions_html
|
| 174 |
+
)
|
| 175 |
+
|
| 176 |
+
msg.attach(MIMEText(html, 'html'))
|
| 177 |
+
|
| 178 |
+
# Send email
|
| 179 |
+
with smtplib.SMTP(EMAIL_CONFIG['smtp_server'], EMAIL_CONFIG['smtp_port']) as server:
|
| 180 |
+
server.starttls()
|
| 181 |
+
server.login(EMAIL_CONFIG['sender_email'], EMAIL_CONFIG['sender_password'])
|
| 182 |
+
server.send_message(msg)
|
| 183 |
+
|
| 184 |
+
print(f"✅ Email sent: {component} - {state}")
|
| 185 |
+
return True
|
| 186 |
+
|
| 187 |
+
except Exception as e:
|
| 188 |
+
print(f"❌ Email failed: {e}")
|
| 189 |
+
return False
|
| 190 |
+
|
| 191 |
+
|
| 192 |
+
# ============================================================================
|
| 193 |
+
# SMS ALERTS
|
| 194 |
+
# ============================================================================
|
| 195 |
+
|
| 196 |
+
def send_sms_alert(prediction):
|
| 197 |
+
"""
|
| 198 |
+
Send SMS alert for FAULTED state.
|
| 199 |
+
|
| 200 |
+
Args:
|
| 201 |
+
prediction (dict): ML prediction result
|
| 202 |
+
|
| 203 |
+
Returns:
|
| 204 |
+
bool: True if sent successfully
|
| 205 |
+
"""
|
| 206 |
+
if not SMS_CONFIG['enabled']:
|
| 207 |
+
#print("📱 SMS alerts disabled in config")
|
| 208 |
+
return False
|
| 209 |
+
|
| 210 |
+
if not TWILIO_AVAILABLE:
|
| 211 |
+
print("❌ SMS failed: Twilio not installed")
|
| 212 |
+
return False
|
| 213 |
+
|
| 214 |
+
state = prediction['stage2']['state']
|
| 215 |
+
|
| 216 |
+
if state != 'FAULTED':
|
| 217 |
+
return False # Only SMS for faults
|
| 218 |
+
|
| 219 |
+
component = prediction['stage3']['component']
|
| 220 |
+
confidence = prediction['stage3']['confidence']
|
| 221 |
+
|
| 222 |
+
if not component:
|
| 223 |
+
return False
|
| 224 |
+
|
| 225 |
+
# Check cooldown
|
| 226 |
+
if not check_cooldown(component, 'SMS', state):
|
| 227 |
+
return False
|
| 228 |
+
|
| 229 |
+
# Test mode: Just print
|
| 230 |
+
if ALERT_BEHAVIOR['test_mode']:
|
| 231 |
+
print(f"📱 [TEST MODE] Would send SMS: {component} - FAULTED ({confidence * 100:.0f}%)")
|
| 232 |
+
return True
|
| 233 |
+
|
| 234 |
+
try:
|
| 235 |
+
client = Client(
|
| 236 |
+
SMS_CONFIG['twilio_account_sid'],
|
| 237 |
+
SMS_CONFIG['twilio_auth_token']
|
| 238 |
+
)
|
| 239 |
+
|
| 240 |
+
message_body = SMS_TEMPLATE.format(
|
| 241 |
+
component=component,
|
| 242 |
+
state=state,
|
| 243 |
+
confidence=confidence * 100,
|
| 244 |
+
time=prediction['timestamp'].strftime('%H:%M:%S')
|
| 245 |
+
)
|
| 246 |
+
|
| 247 |
+
for phone in SMS_CONFIG['recipient_phones']:
|
| 248 |
+
message = client.messages.create(
|
| 249 |
+
body=message_body,
|
| 250 |
+
from_=SMS_CONFIG['twilio_phone_number'],
|
| 251 |
+
to=phone
|
| 252 |
+
)
|
| 253 |
+
print(f"✅ SMS sent to {phone}: {message.sid}")
|
| 254 |
+
|
| 255 |
+
return True
|
| 256 |
+
|
| 257 |
+
except Exception as e:
|
| 258 |
+
print(f"❌ SMS failed: {e}")
|
| 259 |
+
return False
|
| 260 |
+
|
| 261 |
+
|
| 262 |
+
# ============================================================================
|
| 263 |
+
# PHONE CALL ALERTS
|
| 264 |
+
# ============================================================================
|
| 265 |
+
|
| 266 |
+
def send_phone_alert(prediction):
|
| 267 |
+
"""
|
| 268 |
+
Send phone call for FAULTED state with high confidence.
|
| 269 |
+
|
| 270 |
+
Args:
|
| 271 |
+
prediction (dict): ML prediction result
|
| 272 |
+
|
| 273 |
+
Returns:
|
| 274 |
+
bool: True if sent successfully
|
| 275 |
+
"""
|
| 276 |
+
if not CALL_CONFIG['enabled']:
|
| 277 |
+
#print("☎️ Phone alerts disabled in config")
|
| 278 |
+
return False
|
| 279 |
+
|
| 280 |
+
if not TWILIO_AVAILABLE:
|
| 281 |
+
print("❌ Call failed: Twilio not installed")
|
| 282 |
+
return False
|
| 283 |
+
|
| 284 |
+
state = prediction['stage2']['state']
|
| 285 |
+
confidence = prediction['stage3']['confidence']
|
| 286 |
+
|
| 287 |
+
if state != 'FAULTED' or confidence < CALL_CONFIG['confidence_threshold']:
|
| 288 |
+
return False
|
| 289 |
+
|
| 290 |
+
component = prediction['stage3']['component']
|
| 291 |
+
|
| 292 |
+
if not component:
|
| 293 |
+
return False
|
| 294 |
+
|
| 295 |
+
# Check cooldown
|
| 296 |
+
if not check_cooldown(component, 'CALL', state):
|
| 297 |
+
return False
|
| 298 |
+
|
| 299 |
+
# Test mode: Just print
|
| 300 |
+
if ALERT_BEHAVIOR['test_mode']:
|
| 301 |
+
print(f"☎️ [TEST MODE] Would call: {component} - FAULTED ({confidence * 100:.0f}%)")
|
| 302 |
+
return True
|
| 303 |
+
|
| 304 |
+
try:
|
| 305 |
+
client = Client(
|
| 306 |
+
CALL_CONFIG['twilio_account_sid'],
|
| 307 |
+
CALL_CONFIG['twilio_auth_token']
|
| 308 |
+
)
|
| 309 |
+
|
| 310 |
+
twiml = CALL_SCRIPT.format(
|
| 311 |
+
component=component,
|
| 312 |
+
confidence=int(confidence * 100)
|
| 313 |
+
)
|
| 314 |
+
|
| 315 |
+
for phone in CALL_CONFIG['recipient_phones']:
|
| 316 |
+
call = client.calls.create(
|
| 317 |
+
twiml=twiml,
|
| 318 |
+
from_=CALL_CONFIG['twilio_phone_number'],
|
| 319 |
+
to=phone
|
| 320 |
+
)
|
| 321 |
+
print(f"✅ Call initiated to {phone}: {call.sid}")
|
| 322 |
+
|
| 323 |
+
return True
|
| 324 |
+
|
| 325 |
+
except Exception as e:
|
| 326 |
+
print(f"❌ Call failed: {e}")
|
| 327 |
+
return False
|
| 328 |
+
|
| 329 |
+
|
| 330 |
+
# ============================================================================
|
| 331 |
+
# TRIGGER ALERTS
|
| 332 |
+
# ============================================================================
|
| 333 |
+
|
| 334 |
+
def trigger_alerts(prediction):
|
| 335 |
+
state = prediction['stage2']['state']
|
| 336 |
+
component = prediction['stage3'].get('component')
|
| 337 |
+
|
| 338 |
+
# If normal, reset timers and exit
|
| 339 |
+
if state in ('NORMAL', 'UNKNOWN', 'ERROR'):
|
| 340 |
+
if not component:
|
| 341 |
+
_state_started_at.clear()
|
| 342 |
+
# optionally also clear _last_alert_time for state keys if desired
|
| 343 |
+
else:
|
| 344 |
+
reset_component_timers(component)
|
| 345 |
+
|
| 346 |
+
return {'email': False, 'sms': False, 'call': False}
|
| 347 |
+
|
| 348 |
+
results = {'email': False, 'sms': False, 'call': False}
|
| 349 |
+
|
| 350 |
+
# ✅ Persistence gates for both DEGRADING and FAULTED
|
| 351 |
+
if state in ("DEGRADING", "FAULTED"):
|
| 352 |
+
if not state_persisted_long_enough(component, state):
|
| 353 |
+
return results # don't alert yet
|
| 354 |
+
|
| 355 |
+
# Email for DEGRADING or FAULTED
|
| 356 |
+
if state in ['DEGRADING', 'FAULTED']:
|
| 357 |
+
if ALERT_BEHAVIOR['alert_on_degrading'] or state == 'FAULTED':
|
| 358 |
+
results['email'] = send_email_alert(prediction)
|
| 359 |
+
|
| 360 |
+
# SMS for FAULTED only (after persistence gate)
|
| 361 |
+
if state == 'FAULTED' and ALERT_BEHAVIOR['alert_on_faulted']:
|
| 362 |
+
results['sms'] = send_sms_alert(prediction)
|
| 363 |
+
|
| 364 |
+
# Phone call for FAULTED only (after persistence gate + your confidence threshold)
|
| 365 |
+
if state == 'FAULTED' and ALERT_BEHAVIOR['alert_on_faulted']:
|
| 366 |
+
results['call'] = send_phone_alert(prediction)
|
| 367 |
+
|
| 368 |
+
return results
|
| 369 |
+
|
| 370 |
+
|
| 371 |
+
# ============================================================================
|
| 372 |
+
# TESTING
|
| 373 |
+
# ============================================================================
|
| 374 |
+
|
| 375 |
+
if __name__ == "__main__":
|
| 376 |
+
print("=" * 60)
|
| 377 |
+
print("ALERT SYSTEM TEST")
|
| 378 |
+
print("=" * 60)
|
| 379 |
+
|
| 380 |
+
# Mock prediction
|
| 381 |
+
test_prediction = {
|
| 382 |
+
'timestamp': datetime.now(),
|
| 383 |
+
'stage2': {
|
| 384 |
+
'state': 'FAULTED',
|
| 385 |
+
'confidence': 0.95
|
| 386 |
+
},
|
| 387 |
+
'stage3': {
|
| 388 |
+
'component': 'P302',
|
| 389 |
+
'confidence': 0.93,
|
| 390 |
+
'top3': [
|
| 391 |
+
('P302', 0.93),
|
| 392 |
+
('P402', 0.04),
|
| 393 |
+
('P501', 0.02)
|
| 394 |
+
]
|
| 395 |
+
},
|
| 396 |
+
'actions': [
|
| 397 |
+
'🔧 Inspect UF feed pump P302',
|
| 398 |
+
'📋 Check UF membrane differential pressure',
|
| 399 |
+
'💧 Consider membrane backwash or cleaning'
|
| 400 |
+
]
|
| 401 |
+
}
|
| 402 |
+
|
| 403 |
+
print("\n📧 Testing Email Alert...")
|
| 404 |
+
email_result = send_email_alert(test_prediction)
|
| 405 |
+
|
| 406 |
+
print("\n📱 Testing SMS Alert...")
|
| 407 |
+
sms_result = send_sms_alert(test_prediction)
|
| 408 |
+
|
| 409 |
+
print("\n☎️ Testing Phone Alert...")
|
| 410 |
+
call_result = send_phone_alert(test_prediction)
|
| 411 |
+
|
| 412 |
+
print("\n" + "=" * 60)
|
| 413 |
+
print("RESULTS:")
|
| 414 |
+
print("=" * 60)
|
| 415 |
+
print(f"Email: {'✅ Sent' if email_result else '❌ Failed'}")
|
| 416 |
+
print(f"SMS: {'✅ Sent' if sms_result else '❌ Failed'}")
|
| 417 |
+
print(f"Call: {'✅ Sent' if call_result else '❌ Failed'}")
|
ml_api.py
ADDED
|
@@ -0,0 +1,185 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
SWaT ML Inference API Service
|
| 3 |
+
Directly calls ml_inference.run_pipeline() and returns dashboard-friendly JSON
|
| 4 |
+
"""
|
| 5 |
+
|
| 6 |
+
from flask import Flask, request, jsonify
|
| 7 |
+
from flask_cors import CORS
|
| 8 |
+
import os
|
| 9 |
+
import sys
|
| 10 |
+
import time
|
| 11 |
+
import traceback
|
| 12 |
+
|
| 13 |
+
# Import from current folder
|
| 14 |
+
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
| 15 |
+
|
| 16 |
+
app = Flask(__name__)
|
| 17 |
+
CORS(app)
|
| 18 |
+
|
| 19 |
+
ML_AVAILABLE = False
|
| 20 |
+
ML_IMPORT_ERROR = None
|
| 21 |
+
ML_API_FILE = os.path.abspath(__file__)
|
| 22 |
+
ML_PID = os.getpid()
|
| 23 |
+
|
| 24 |
+
sensor_buffer = None
|
| 25 |
+
MODELS = None # cached models (already cached in your ml_inference too)
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
# -----------------------------------------------------------------------------
|
| 29 |
+
# Startup: load models ONCE + create one shared buffer
|
| 30 |
+
# -----------------------------------------------------------------------------
|
| 31 |
+
try:
|
| 32 |
+
from sensor_buffer import SensorBuffer
|
| 33 |
+
import ml_inference
|
| 34 |
+
|
| 35 |
+
t0 = time.perf_counter()
|
| 36 |
+
MODELS = ml_inference.load_models()
|
| 37 |
+
t1 = time.perf_counter()
|
| 38 |
+
|
| 39 |
+
if not MODELS or not MODELS.get("loaded"):
|
| 40 |
+
raise RuntimeError(MODELS.get("error", "Models not loaded"))
|
| 41 |
+
|
| 42 |
+
sensor_buffer = SensorBuffer(window_size=60, n_features=40)
|
| 43 |
+
ML_AVAILABLE = True
|
| 44 |
+
|
| 45 |
+
print(f"✅ Models loaded successfully in {(t1 - t0)*1000:.1f} ms")
|
| 46 |
+
print("ML Available: True")
|
| 47 |
+
|
| 48 |
+
except Exception as e:
|
| 49 |
+
ML_IMPORT_ERROR = f"{e}\n{traceback.format_exc()}"
|
| 50 |
+
ML_AVAILABLE = False
|
| 51 |
+
print(f"ML Import Error: {e}")
|
| 52 |
+
print(traceback.format_exc())
|
| 53 |
+
print("ML Available: False")
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
# -----------------------------------------------------------------------------
|
| 57 |
+
# Map ml_inference output -> what your web app expects
|
| 58 |
+
# -----------------------------------------------------------------------------
|
| 59 |
+
def to_dashboard_schema(result: dict):
|
| 60 |
+
stage1 = result.get("stage1", {}) or {}
|
| 61 |
+
stage2 = result.get("stage2", {}) or {}
|
| 62 |
+
stage3 = result.get("stage3", {}) or {}
|
| 63 |
+
|
| 64 |
+
bs = result.get("buffer_status", {}) or {}
|
| 65 |
+
buffer_status = {
|
| 66 |
+
"ready": bool(bs.get("ready", sensor_buffer.is_ready() if sensor_buffer else False)),
|
| 67 |
+
"size": int(bs.get("size", len(sensor_buffer) if sensor_buffer else 0)),
|
| 68 |
+
"capacity": int(getattr(sensor_buffer, "window_size", 60)) if sensor_buffer else 60,
|
| 69 |
+
"usingBuffer": bool(bs.get("using_buffer", False)),
|
| 70 |
+
}
|
| 71 |
+
|
| 72 |
+
return {
|
| 73 |
+
"success": True,
|
| 74 |
+
"stage1": {
|
| 75 |
+
"isAnomaly": bool(stage1.get("is_anomaly", False)),
|
| 76 |
+
"confidence": float(stage1.get("confidence", 0.0)),
|
| 77 |
+
"score": stage1.get("score", None),
|
| 78 |
+
"modelType": stage1.get("model_type", None),
|
| 79 |
+
},
|
| 80 |
+
"stage2": {
|
| 81 |
+
"state": stage2.get("state", "UNKNOWN"),
|
| 82 |
+
"confidence": float(stage2.get("confidence", 0.0)),
|
| 83 |
+
"modelType": stage2.get("model_type", None),
|
| 84 |
+
},
|
| 85 |
+
"stage3": {
|
| 86 |
+
"component": stage3.get("component", None),
|
| 87 |
+
"confidence": float(stage3.get("confidence", 0.0)),
|
| 88 |
+
"top3": [
|
| 89 |
+
{"component": c, "confidence": float(p)}
|
| 90 |
+
for (c, p) in (stage3.get("top3") or [])
|
| 91 |
+
],
|
| 92 |
+
"modelType": stage3.get("model_type", None),
|
| 93 |
+
},
|
| 94 |
+
"componentHealth": result.get("component_health", {}) or {},
|
| 95 |
+
"recommendedActions": result.get("actions", []) or [],
|
| 96 |
+
"alertsSent": result.get("alerts_sent", {}) or {},
|
| 97 |
+
"bufferStatus": buffer_status,
|
| 98 |
+
}
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
# -----------------------------------------------------------------------------
|
| 102 |
+
# Routes
|
| 103 |
+
# -----------------------------------------------------------------------------
|
| 104 |
+
@app.route("/health", methods=["GET"])
|
| 105 |
+
def health():
|
| 106 |
+
return jsonify({
|
| 107 |
+
"status": "ok",
|
| 108 |
+
"ml_available": bool(ML_AVAILABLE),
|
| 109 |
+
"pid": ML_PID,
|
| 110 |
+
"file": ML_API_FILE,
|
| 111 |
+
"import_error": ML_IMPORT_ERROR,
|
| 112 |
+
"buffer_size": len(sensor_buffer) if sensor_buffer else 0,
|
| 113 |
+
"buffer_ready": sensor_buffer.is_ready() if sensor_buffer else False,
|
| 114 |
+
})
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
@app.route("/api/inference", methods=["POST"])
|
| 118 |
+
def run_inference():
|
| 119 |
+
if not ML_AVAILABLE:
|
| 120 |
+
return jsonify({
|
| 121 |
+
"success": False,
|
| 122 |
+
"error": "ML components not available",
|
| 123 |
+
"import_error": ML_IMPORT_ERROR
|
| 124 |
+
}), 503
|
| 125 |
+
|
| 126 |
+
try:
|
| 127 |
+
data = request.get_json(silent=True) or {}
|
| 128 |
+
payload = data.get("payload")
|
| 129 |
+
|
| 130 |
+
if not isinstance(payload, dict):
|
| 131 |
+
return jsonify({"success": False, "error": "Missing payload in request"}), 400
|
| 132 |
+
|
| 133 |
+
# ✅ Directly call YOUR working pipeline
|
| 134 |
+
result = ml_inference.run_pipeline(payload, sensor_buffer=sensor_buffer)
|
| 135 |
+
|
| 136 |
+
if not result.get("success", False):
|
| 137 |
+
return jsonify({
|
| 138 |
+
"success": False,
|
| 139 |
+
"error": result.get("error", "Pipeline failed"),
|
| 140 |
+
"bufferStatus": {
|
| 141 |
+
"ready": sensor_buffer.is_ready() if sensor_buffer else False,
|
| 142 |
+
"size": len(sensor_buffer) if sensor_buffer else 0,
|
| 143 |
+
"capacity": getattr(sensor_buffer, "window_size", 60) if sensor_buffer else 60,
|
| 144 |
+
"usingBuffer": False,
|
| 145 |
+
}
|
| 146 |
+
}), 200
|
| 147 |
+
|
| 148 |
+
return jsonify(to_dashboard_schema(result)), 200
|
| 149 |
+
|
| 150 |
+
except Exception as e:
|
| 151 |
+
traceback.print_exc()
|
| 152 |
+
return jsonify({"success": False, "error": str(e)}), 500
|
| 153 |
+
|
| 154 |
+
|
| 155 |
+
@app.route("/api/buffer/reset", methods=["POST"])
|
| 156 |
+
def reset_buffer():
|
| 157 |
+
global sensor_buffer
|
| 158 |
+
if not ML_AVAILABLE:
|
| 159 |
+
return jsonify({"success": False, "error": "ML not available"}), 503
|
| 160 |
+
|
| 161 |
+
from sensor_buffer import SensorBuffer
|
| 162 |
+
sensor_buffer = SensorBuffer(window_size=60, n_features=40)
|
| 163 |
+
return jsonify({"success": True, "message": "Buffer reset"}), 200
|
| 164 |
+
|
| 165 |
+
|
| 166 |
+
@app.route("/api/buffer/status", methods=["GET"])
|
| 167 |
+
def buffer_status():
|
| 168 |
+
if not sensor_buffer:
|
| 169 |
+
return jsonify({"ready": False, "size": 0, "capacity": 60})
|
| 170 |
+
return jsonify({
|
| 171 |
+
"ready": sensor_buffer.is_ready(),
|
| 172 |
+
"size": len(sensor_buffer),
|
| 173 |
+
"capacity": sensor_buffer.window_size
|
| 174 |
+
})
|
| 175 |
+
|
| 176 |
+
|
| 177 |
+
if __name__ == "__main__":
|
| 178 |
+
print("=" * 60)
|
| 179 |
+
print("SWaT ML Inference API Service")
|
| 180 |
+
print("=" * 60)
|
| 181 |
+
print(f"ML Available: {ML_AVAILABLE}")
|
| 182 |
+
print("Starting server on http://localhost:5000")
|
| 183 |
+
print("=" * 60)
|
| 184 |
+
|
| 185 |
+
app.run(host="0.0.0.0", port=int(os.environ.get("PORT", 7860)))
|
ml_inference.py
ADDED
|
@@ -0,0 +1,813 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
ML INFERENCE ENGINE FOR SWAT DASHBOARD
|
| 3 |
+
======================================
|
| 4 |
+
Real-time predictive maintenance inference using trained models.
|
| 5 |
+
|
| 6 |
+
Implements 3-stage pipeline:
|
| 7 |
+
- Stage 1: Anomaly Detection (ANY model from training)
|
| 8 |
+
- Stage 2: State Classification (ANY model from training)
|
| 9 |
+
- Stage 3: Component Identification (ANY model from training)
|
| 10 |
+
|
| 11 |
+
Models are automatically loaded based on final_config.json from each stage.
|
| 12 |
+
Models are cached for fast inference.
|
| 13 |
+
"""
|
| 14 |
+
|
| 15 |
+
import numpy as np
|
| 16 |
+
import pandas as pd
|
| 17 |
+
import pickle
|
| 18 |
+
import json
|
| 19 |
+
import warnings
|
| 20 |
+
from sensor_buffer import SensorBuffer
|
| 21 |
+
from alerts import trigger_alerts
|
| 22 |
+
warnings.filterwarnings('ignore')
|
| 23 |
+
|
| 24 |
+
try:
|
| 25 |
+
import tensorflow as tf
|
| 26 |
+
from tensorflow import keras
|
| 27 |
+
TF_AVAILABLE = True
|
| 28 |
+
except ImportError:
|
| 29 |
+
TF_AVAILABLE = False
|
| 30 |
+
print("⚠️ TensorFlow not available. Install: pip install tensorflow")
|
| 31 |
+
|
| 32 |
+
try:
|
| 33 |
+
import streamlit as st
|
| 34 |
+
STREAMLIT_AVAILABLE = False
|
| 35 |
+
except ImportError:
|
| 36 |
+
STREAMLIT_AVAILABLE = False
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
# ============================================================================
|
| 40 |
+
# CONFIGURATION
|
| 41 |
+
# ============================================================================
|
| 42 |
+
|
| 43 |
+
# Model paths (adjust these to your local paths)
|
| 44 |
+
MODEL_DIR = r"D:\FYP - FINAL\SwatDashboard\PythonMlService\models"
|
| 45 |
+
DATA_DIR = r"D:\FYP - FINAL\SwatDashboard\PythonMlService\ml_data"
|
| 46 |
+
|
| 47 |
+
# Component names (must match training)
|
| 48 |
+
COMPONENTS = ['P101', 'P201', 'P203', 'P205', 'P302', 'P402', 'P403', 'P501', 'MV101', 'MV304']
|
| 49 |
+
STATE_NAMES = ['ANOMALY', 'DEGRADING', 'FAULTED']
|
| 50 |
+
|
| 51 |
+
# ============================================================================
|
| 52 |
+
# MODEL LOADING (CACHED & GENERAL)
|
| 53 |
+
# ============================================================================
|
| 54 |
+
|
| 55 |
+
if STREAMLIT_AVAILABLE:
|
| 56 |
+
@st.cache_resource
|
| 57 |
+
def load_models():
|
| 58 |
+
"""Load all trained models (cached for performance)"""
|
| 59 |
+
return _load_models_internal()
|
| 60 |
+
else:
|
| 61 |
+
_cached_models = None
|
| 62 |
+
def load_models():
|
| 63 |
+
global _cached_models
|
| 64 |
+
if _cached_models is None:
|
| 65 |
+
_cached_models = _load_models_internal()
|
| 66 |
+
return _cached_models
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
def _load_models_internal():
|
| 70 |
+
"""Internal model loading function - GENERAL VERSION"""
|
| 71 |
+
models = {}
|
| 72 |
+
|
| 73 |
+
try:
|
| 74 |
+
# Load scaler
|
| 75 |
+
scaler_path = f"{DATA_DIR}/scaler.pkl"
|
| 76 |
+
with open(scaler_path, 'rb') as f:
|
| 77 |
+
models['scaler'] = pickle.load(f)
|
| 78 |
+
|
| 79 |
+
# ====================================================================
|
| 80 |
+
# STAGE 1: Load best model based on config
|
| 81 |
+
# ====================================================================
|
| 82 |
+
stage1_dir = f"{MODEL_DIR}/stage1"
|
| 83 |
+
with open(f"{stage1_dir}/final_config.json", "r") as f:
|
| 84 |
+
config1 = json.load(f)
|
| 85 |
+
|
| 86 |
+
models['stage1_type'] = config1['best_model_type']
|
| 87 |
+
|
| 88 |
+
if models['stage1_type'] in ['autoencoder', 'dae']:
|
| 89 |
+
# Autoencoder or Denoising Autoencoder
|
| 90 |
+
if not TF_AVAILABLE:
|
| 91 |
+
raise ImportError("TensorFlow required for autoencoder models")
|
| 92 |
+
|
| 93 |
+
model_file = 'autoencoder.keras' if models['stage1_type'] == 'autoencoder' else 'denoising_autoencoder.keras'
|
| 94 |
+
threshold_file = 'autoencoder_threshold.pkl' if models['stage1_type'] == 'autoencoder' else 'dae_threshold.pkl'
|
| 95 |
+
|
| 96 |
+
models['stage1'] = keras.models.load_model(f"{stage1_dir}/{model_file}", compile=False)
|
| 97 |
+
with open(f"{stage1_dir}/{threshold_file}", 'rb') as f:
|
| 98 |
+
models['stage1_threshold'] = pickle.load(f)
|
| 99 |
+
|
| 100 |
+
elif models['stage1_type'] == 'lof':
|
| 101 |
+
# Local Outlier Factor
|
| 102 |
+
with open(f"{stage1_dir}/lof.pkl", 'rb') as f:
|
| 103 |
+
models['stage1'] = pickle.load(f)
|
| 104 |
+
with open(f"{stage1_dir}/lof_threshold.pkl", 'rb') as f:
|
| 105 |
+
models['stage1_threshold'] = pickle.load(f)
|
| 106 |
+
|
| 107 |
+
elif models['stage1_type'] == 'iforest':
|
| 108 |
+
# Isolation Forest
|
| 109 |
+
with open(f"{stage1_dir}/isolation_forest.pkl", 'rb') as f:
|
| 110 |
+
models['stage1'] = pickle.load(f)
|
| 111 |
+
models['stage1_threshold'] = None # Uses built-in threshold
|
| 112 |
+
|
| 113 |
+
else: # xgboost
|
| 114 |
+
with open(f"{stage1_dir}/xgboost.pkl", 'rb') as f:
|
| 115 |
+
models['stage1'] = pickle.load(f)
|
| 116 |
+
|
| 117 |
+
# ===== FORCE CPU (XGBoost 3.1.0 SAFE) =====
|
| 118 |
+
m = models['stage1']
|
| 119 |
+
|
| 120 |
+
# sklearn wrapper params
|
| 121 |
+
try:
|
| 122 |
+
m.set_params(
|
| 123 |
+
device="cpu", # 🔥 key line
|
| 124 |
+
tree_method="hist", # CPU-friendly
|
| 125 |
+
n_jobs=1 # avoid OpenMP fights
|
| 126 |
+
)
|
| 127 |
+
except Exception:
|
| 128 |
+
pass
|
| 129 |
+
|
| 130 |
+
# booster-level safety
|
| 131 |
+
try:
|
| 132 |
+
m.get_booster().set_param({"device": "cpu"})
|
| 133 |
+
except Exception:
|
| 134 |
+
pass
|
| 135 |
+
|
| 136 |
+
models['stage1_threshold'] = None
|
| 137 |
+
#print(" ✅ Stage 1 XGBoost loaded (CPU forced)")
|
| 138 |
+
|
| 139 |
+
# ====================================================================
|
| 140 |
+
# STAGE 2: Load best model based on config
|
| 141 |
+
# ====================================================================
|
| 142 |
+
stage2_dir = f"{MODEL_DIR}/stage2"
|
| 143 |
+
with open(f"{stage2_dir}/final_config.json", "r") as f:
|
| 144 |
+
config2 = json.load(f)
|
| 145 |
+
|
| 146 |
+
models['stage2_type'] = config2['best_model_type']
|
| 147 |
+
models['stage2_seq_length'] = config2.get('sequence_length', None)
|
| 148 |
+
|
| 149 |
+
if models['stage2_type'] in ['lstm', 'cnn']:
|
| 150 |
+
# Sequential models (LSTM or CNN)
|
| 151 |
+
if not TF_AVAILABLE:
|
| 152 |
+
raise ImportError("TensorFlow required for LSTM/CNN models")
|
| 153 |
+
|
| 154 |
+
models['stage2'] = keras.models.load_model(f"{stage2_dir}/{models['stage2_type']}.keras")
|
| 155 |
+
|
| 156 |
+
else: # xgboost
|
| 157 |
+
with open(f"{stage2_dir}/xgboost.pkl", 'rb') as f:
|
| 158 |
+
models['stage2'] = pickle.load(f)
|
| 159 |
+
|
| 160 |
+
# ====================================================================
|
| 161 |
+
# STAGE 3: Load best model based on config
|
| 162 |
+
# ====================================================================
|
| 163 |
+
stage3_dir = f"{MODEL_DIR}/stage3"
|
| 164 |
+
with open(f"{stage3_dir}/final_config.json", "r") as f:
|
| 165 |
+
config3 = json.load(f)
|
| 166 |
+
|
| 167 |
+
models['stage3_type'] = config3['best_model_type']
|
| 168 |
+
|
| 169 |
+
if models['stage3_type'] == 'mlp':
|
| 170 |
+
# Multi-Layer Perceptron
|
| 171 |
+
if not TF_AVAILABLE:
|
| 172 |
+
raise ImportError("TensorFlow required for MLP models")
|
| 173 |
+
|
| 174 |
+
models['stage3'] = keras.models.load_model(f"{stage3_dir}/mlp.keras")
|
| 175 |
+
|
| 176 |
+
elif models['stage3_type'] == 'lightgbm':
|
| 177 |
+
# LightGBM
|
| 178 |
+
with open(f"{stage3_dir}/lightgbm.pkl", 'rb') as f:
|
| 179 |
+
models['stage3'] = pickle.load(f)
|
| 180 |
+
|
| 181 |
+
else: # xgboost
|
| 182 |
+
with open(f"{stage3_dir}/xgboost.pkl", 'rb') as f:
|
| 183 |
+
models['stage3'] = pickle.load(f)
|
| 184 |
+
|
| 185 |
+
models['loaded'] = True
|
| 186 |
+
print(f"✅ Models loaded successfully!")
|
| 187 |
+
print(f" Stage 1: {config1['best_model']} ({models['stage1_type']})")
|
| 188 |
+
print(f" Stage 2: {config2['best_model']} ({models['stage2_type']})")
|
| 189 |
+
print(f" Stage 3: {config3['best_model']} ({models['stage3_type']})")
|
| 190 |
+
|
| 191 |
+
return models
|
| 192 |
+
|
| 193 |
+
except Exception as e:
|
| 194 |
+
print(f"⚠️ Error loading models: {e}")
|
| 195 |
+
models['loaded'] = False
|
| 196 |
+
models['error'] = str(e)
|
| 197 |
+
return models
|
| 198 |
+
|
| 199 |
+
|
| 200 |
+
# ============================================================================
|
| 201 |
+
# FEATURE EXTRACTION
|
| 202 |
+
# ============================================================================
|
| 203 |
+
|
| 204 |
+
def extract_features(payload):
|
| 205 |
+
"""
|
| 206 |
+
Extract 40 features from payload (matches training format).
|
| 207 |
+
|
| 208 |
+
Returns:
|
| 209 |
+
np.array: (1, 40) feature array
|
| 210 |
+
"""
|
| 211 |
+
features = []
|
| 212 |
+
|
| 213 |
+
# ========================================================================
|
| 214 |
+
# 16 SENSOR READINGS
|
| 215 |
+
# ========================================================================
|
| 216 |
+
sensors = [
|
| 217 |
+
'true_FIT101', 'true_FIT201', 'true_FIT301', 'true_FIT401', 'true_FIT501',
|
| 218 |
+
'true_LIT101', 'true_LIT301', 'true_LIT401',
|
| 219 |
+
'true_AIT201', 'true_AIT202', 'true_AIT203', 'true_AIT401', 'true_AIT402', 'true_AIT501',
|
| 220 |
+
'true_DPIT301', 'true_PIT501'
|
| 221 |
+
]
|
| 222 |
+
|
| 223 |
+
for sensor in sensors:
|
| 224 |
+
val = payload.get(sensor, 0)
|
| 225 |
+
try:
|
| 226 |
+
features.append(float(val))
|
| 227 |
+
except:
|
| 228 |
+
features.append(0.0)
|
| 229 |
+
|
| 230 |
+
# ========================================================================
|
| 231 |
+
# 24 MOTOR PHYSICS FEATURES
|
| 232 |
+
# ========================================================================
|
| 233 |
+
# These should match the motor physics features used in training
|
| 234 |
+
# Format: motor_temp, current, vibration for each component
|
| 235 |
+
components = ['P101', 'P201', 'P203', 'P205', 'P302', 'P402', 'P403', 'P501']
|
| 236 |
+
|
| 237 |
+
for comp in components:
|
| 238 |
+
# Get motor physics features from payload
|
| 239 |
+
temp_key = f'true_{comp}_motor_temp'
|
| 240 |
+
current_key = f'true_{comp}_current'
|
| 241 |
+
vib_key = f'true_{comp}_vibration'
|
| 242 |
+
|
| 243 |
+
# Extract values (use defaults if not present)
|
| 244 |
+
temp = payload.get(temp_key, 0.0)
|
| 245 |
+
current = payload.get(current_key, 0.0)
|
| 246 |
+
vib = payload.get(vib_key, 0.0)
|
| 247 |
+
|
| 248 |
+
try:
|
| 249 |
+
features.extend([float(temp), float(current), float(vib)])
|
| 250 |
+
except:
|
| 251 |
+
features.extend([0.0, 0.0, 0.0])
|
| 252 |
+
|
| 253 |
+
# Convert to numpy array
|
| 254 |
+
return np.array(features).reshape(1, -1)
|
| 255 |
+
|
| 256 |
+
|
| 257 |
+
# ============================================================================
|
| 258 |
+
# STAGE 1: ANOMALY DETECTION (GENERAL)
|
| 259 |
+
# ============================================================================
|
| 260 |
+
|
| 261 |
+
def predict_anomaly(features_scaled, models):
|
| 262 |
+
"""
|
| 263 |
+
Stage 1: Binary anomaly detection using ANY model type.
|
| 264 |
+
|
| 265 |
+
Returns:
|
| 266 |
+
tuple: (is_anomaly, confidence, score)
|
| 267 |
+
"""
|
| 268 |
+
if not models.get('loaded'):
|
| 269 |
+
return False, 0.0, 0.0
|
| 270 |
+
|
| 271 |
+
try:
|
| 272 |
+
model_type = models['stage1_type']
|
| 273 |
+
|
| 274 |
+
# ====================================================================
|
| 275 |
+
# AUTOENCODER / DENOISING AUTOENCODER
|
| 276 |
+
# ====================================================================
|
| 277 |
+
if model_type in ['autoencoder', 'dae']:
|
| 278 |
+
# Reconstruct input
|
| 279 |
+
reconstructed = models['stage1'].predict(features_scaled, verbose=0)
|
| 280 |
+
|
| 281 |
+
# Compute reconstruction error (MSE)
|
| 282 |
+
error = np.mean((features_scaled - reconstructed) ** 2)
|
| 283 |
+
|
| 284 |
+
# Compare to threshold
|
| 285 |
+
threshold = models['stage1_threshold']
|
| 286 |
+
is_anomaly = (error > threshold)
|
| 287 |
+
|
| 288 |
+
# Compute confidence
|
| 289 |
+
if is_anomaly:
|
| 290 |
+
confidence = min(1.0, (error / threshold - 1.0) * 2 + 0.5)
|
| 291 |
+
else:
|
| 292 |
+
confidence = min(1.0, (1.0 - error / threshold) + 0.5)
|
| 293 |
+
|
| 294 |
+
return bool(is_anomaly), float(confidence), float(error)
|
| 295 |
+
|
| 296 |
+
# ====================================================================
|
| 297 |
+
# LOCAL OUTLIER FACTOR
|
| 298 |
+
# ====================================================================
|
| 299 |
+
elif model_type == 'lof':
|
| 300 |
+
# Get anomaly score (higher = more anomalous)
|
| 301 |
+
score = -models['stage1'].decision_function(features_scaled)[0]
|
| 302 |
+
threshold = models['stage1_threshold']
|
| 303 |
+
|
| 304 |
+
is_anomaly = (score > threshold)
|
| 305 |
+
|
| 306 |
+
# Confidence based on distance from threshold
|
| 307 |
+
if is_anomaly:
|
| 308 |
+
confidence = min(1.0, (score / threshold - 1.0) + 0.5)
|
| 309 |
+
else:
|
| 310 |
+
confidence = min(1.0, (1.0 - score / threshold) + 0.5)
|
| 311 |
+
|
| 312 |
+
return bool(is_anomaly), float(confidence), float(score)
|
| 313 |
+
|
| 314 |
+
# ====================================================================
|
| 315 |
+
# ISOLATION FOREST
|
| 316 |
+
# ====================================================================
|
| 317 |
+
elif model_type == 'iforest':
|
| 318 |
+
# Predict (-1 = anomaly, 1 = normal)
|
| 319 |
+
pred = models['stage1'].predict(features_scaled)[0]
|
| 320 |
+
is_anomaly = (pred == -1)
|
| 321 |
+
|
| 322 |
+
# Get anomaly score for confidence
|
| 323 |
+
score = -models['stage1'].score_samples(features_scaled)[0]
|
| 324 |
+
confidence = min(1.0, abs(score) / 2.0)
|
| 325 |
+
|
| 326 |
+
return bool(is_anomaly), float(confidence), float(score)
|
| 327 |
+
|
| 328 |
+
# ====================================================================
|
| 329 |
+
# XGBOOST
|
| 330 |
+
# ====================================================================
|
| 331 |
+
else: # xgboost
|
| 332 |
+
import xgboost as xgb
|
| 333 |
+
import numpy as np
|
| 334 |
+
|
| 335 |
+
# force CPU
|
| 336 |
+
try:
|
| 337 |
+
models["stage1"].set_params(device="cpu", tree_method="hist", n_jobs=1)
|
| 338 |
+
except Exception:
|
| 339 |
+
pass
|
| 340 |
+
try:
|
| 341 |
+
models["stage1"].get_booster().set_param({"device": "cpu"})
|
| 342 |
+
except Exception:
|
| 343 |
+
pass
|
| 344 |
+
|
| 345 |
+
x = np.ascontiguousarray(features_scaled.astype(np.float32))
|
| 346 |
+
dm = xgb.DMatrix(x)
|
| 347 |
+
|
| 348 |
+
p1 = float(models["stage1"].get_booster().predict(dm)[0]) # binary:logistic => P(class=1)
|
| 349 |
+
is_anomaly = p1 > 0.5
|
| 350 |
+
confidence = p1 if is_anomaly else (1.0 - p1)
|
| 351 |
+
|
| 352 |
+
return bool(is_anomaly), float(confidence), float(p1)
|
| 353 |
+
|
| 354 |
+
except Exception as e:
|
| 355 |
+
print(f"Stage 1 error: {e}")
|
| 356 |
+
return False, 0.0, 0.0
|
| 357 |
+
|
| 358 |
+
|
| 359 |
+
# ============================================================================
|
| 360 |
+
# STAGE 2: STATE CLASSIFICATION (GENERAL)
|
| 361 |
+
# ============================================================================
|
| 362 |
+
|
| 363 |
+
def predict_state(features_scaled, models, is_anomaly, sensor_buffer=None):
|
| 364 |
+
"""
|
| 365 |
+
Stage 2: Multi-class state classification using ANY model type.
|
| 366 |
+
Only runs if Stage 1 detected anomaly.
|
| 367 |
+
|
| 368 |
+
Args:
|
| 369 |
+
features_scaled: Current sample features (scaled)
|
| 370 |
+
models: Loaded models dict
|
| 371 |
+
is_anomaly: Whether Stage 1 detected anomaly
|
| 372 |
+
sensor_buffer: Optional SensorBuffer for LSTM/CNN
|
| 373 |
+
|
| 374 |
+
Returns:
|
| 375 |
+
tuple: (state_name, confidence, state_probs)
|
| 376 |
+
"""
|
| 377 |
+
if not is_anomaly:
|
| 378 |
+
return "NORMAL", 1.0, [0.0, 0.0, 0.0]
|
| 379 |
+
|
| 380 |
+
if not models.get('loaded'):
|
| 381 |
+
return "UNKNOWN", 0.0, [0.0, 0.0, 0.0]
|
| 382 |
+
|
| 383 |
+
try:
|
| 384 |
+
model_type = models['stage2_type']
|
| 385 |
+
|
| 386 |
+
# ====================================================================
|
| 387 |
+
# LSTM / CNN (Sequential models)
|
| 388 |
+
# ====================================================================
|
| 389 |
+
if model_type in ['lstm', 'cnn']:
|
| 390 |
+
seq_length = models['stage2_seq_length']
|
| 391 |
+
|
| 392 |
+
# NEW: Try to use buffer if available
|
| 393 |
+
if sensor_buffer is not None and sensor_buffer.is_ready():
|
| 394 |
+
# Use real sequence from buffer
|
| 395 |
+
seq = sensor_buffer.get_sequence()
|
| 396 |
+
seq = seq.reshape(1, seq_length, -1)
|
| 397 |
+
use_buffer = True
|
| 398 |
+
else:
|
| 399 |
+
# Fallback: Repeat current sample (old hack)
|
| 400 |
+
seq = np.repeat(features_scaled, seq_length, axis=0)
|
| 401 |
+
seq = seq.reshape(1, seq_length, -1)
|
| 402 |
+
use_buffer = False
|
| 403 |
+
|
| 404 |
+
# Predict
|
| 405 |
+
probs = models['stage2'].predict(seq, verbose=0)[0]
|
| 406 |
+
|
| 407 |
+
# Get predicted class
|
| 408 |
+
state_idx = int(np.argmax(probs))
|
| 409 |
+
state_name = STATE_NAMES[state_idx]
|
| 410 |
+
confidence = float(probs[state_idx])
|
| 411 |
+
|
| 412 |
+
# Reduce confidence if using fallback hack
|
| 413 |
+
if not use_buffer:
|
| 414 |
+
confidence *= 0.85 # Penalty for not having real sequence
|
| 415 |
+
|
| 416 |
+
return state_name, confidence, probs.tolist()
|
| 417 |
+
|
| 418 |
+
# ====================================================================
|
| 419 |
+
# XGBOOST (Tabular model - doesn't need buffer)
|
| 420 |
+
# ====================================================================
|
| 421 |
+
else: # xgboost
|
| 422 |
+
# Predict probabilities
|
| 423 |
+
probs = models['stage2'].predict_proba(features_scaled)[0]
|
| 424 |
+
|
| 425 |
+
# Get predicted class
|
| 426 |
+
state_idx = int(np.argmax(probs))
|
| 427 |
+
state_name = STATE_NAMES[state_idx]
|
| 428 |
+
confidence = float(probs[state_idx])
|
| 429 |
+
|
| 430 |
+
return state_name, confidence, probs.tolist()
|
| 431 |
+
|
| 432 |
+
except Exception as e:
|
| 433 |
+
print(f"Stage 2 error: {e}")
|
| 434 |
+
return "UNKNOWN", 0.0, [0.0, 0.0, 0.0]
|
| 435 |
+
|
| 436 |
+
# ============================================================================
|
| 437 |
+
# STAGE 3: COMPONENT IDENTIFICATION (GENERAL)
|
| 438 |
+
# ============================================================================
|
| 439 |
+
|
| 440 |
+
def predict_component(features_scaled, models, state_name):
|
| 441 |
+
"""
|
| 442 |
+
Stage 3: Component identification using ANY model type.
|
| 443 |
+
Only runs if state is DEGRADING or FAULTED.
|
| 444 |
+
|
| 445 |
+
Returns:
|
| 446 |
+
tuple: (component, confidence, top3_components)
|
| 447 |
+
"""
|
| 448 |
+
if state_name == "NORMAL":
|
| 449 |
+
return None, 0.0, []
|
| 450 |
+
|
| 451 |
+
if not models.get('loaded'):
|
| 452 |
+
return None, 0.0, []
|
| 453 |
+
|
| 454 |
+
try:
|
| 455 |
+
model_type = models['stage3_type']
|
| 456 |
+
|
| 457 |
+
# ====================================================================
|
| 458 |
+
# MLP (Neural Network)
|
| 459 |
+
# ====================================================================
|
| 460 |
+
if model_type == 'mlp':
|
| 461 |
+
# Predict probabilities
|
| 462 |
+
probs = models['stage3'].predict(features_scaled, verbose=0)[0]
|
| 463 |
+
|
| 464 |
+
# Get top prediction
|
| 465 |
+
comp_idx = int(np.argmax(probs))
|
| 466 |
+
component = COMPONENTS[comp_idx]
|
| 467 |
+
confidence = float(probs[comp_idx])
|
| 468 |
+
|
| 469 |
+
# Get top 3
|
| 470 |
+
top3_idx = np.argsort(probs)[-3:][::-1]
|
| 471 |
+
top3 = [(COMPONENTS[i], float(probs[i])) for i in top3_idx]
|
| 472 |
+
|
| 473 |
+
return component, confidence, top3
|
| 474 |
+
|
| 475 |
+
# ====================================================================
|
| 476 |
+
# LIGHTGBM or XGBOOST
|
| 477 |
+
# ====================================================================
|
| 478 |
+
else: # lightgbm or xgboost
|
| 479 |
+
# Predict probabilities
|
| 480 |
+
probs = models['stage3'].predict_proba(features_scaled)[0]
|
| 481 |
+
|
| 482 |
+
# Get top prediction
|
| 483 |
+
comp_idx = int(np.argmax(probs))
|
| 484 |
+
component = COMPONENTS[comp_idx]
|
| 485 |
+
confidence = float(probs[comp_idx])
|
| 486 |
+
|
| 487 |
+
# Get top 3
|
| 488 |
+
top3_idx = np.argsort(probs)[-3:][::-1]
|
| 489 |
+
top3 = [(COMPONENTS[i], float(probs[i])) for i in top3_idx]
|
| 490 |
+
|
| 491 |
+
return component, confidence, top3
|
| 492 |
+
|
| 493 |
+
except Exception as e:
|
| 494 |
+
print(f"Stage 3 error: {e}")
|
| 495 |
+
return None, 0.0, []
|
| 496 |
+
|
| 497 |
+
|
| 498 |
+
# ============================================================================
|
| 499 |
+
# FULL PIPELINE
|
| 500 |
+
# ============================================================================
|
| 501 |
+
|
| 502 |
+
def run_pipeline(payload, sensor_buffer=None):
|
| 503 |
+
"""
|
| 504 |
+
Run complete 3-stage ML pipeline on payload.
|
| 505 |
+
|
| 506 |
+
Args:
|
| 507 |
+
payload (dict): Raw sensor/actuator data from dashboard
|
| 508 |
+
sensor_buffer (SensorBuffer): Optional buffer for LSTM/CNN models
|
| 509 |
+
|
| 510 |
+
Returns:
|
| 511 |
+
dict: Prediction results with all stages
|
| 512 |
+
"""
|
| 513 |
+
# Load models (cached)
|
| 514 |
+
models = load_models()
|
| 515 |
+
|
| 516 |
+
# Check if models loaded successfully
|
| 517 |
+
if not models.get('loaded'):
|
| 518 |
+
return {
|
| 519 |
+
'success': False,
|
| 520 |
+
'error': models.get('error', 'Models not loaded'),
|
| 521 |
+
'stage1': {'is_anomaly': False, 'confidence': 0.0},
|
| 522 |
+
'stage2': {'state': 'UNKNOWN', 'confidence': 0.0},
|
| 523 |
+
'stage3': {'component': None, 'confidence': 0.0}
|
| 524 |
+
}
|
| 525 |
+
|
| 526 |
+
try:
|
| 527 |
+
# Extract features
|
| 528 |
+
features = extract_features(payload)
|
| 529 |
+
|
| 530 |
+
# Scale features
|
| 531 |
+
features_scaled = models['scaler'].transform(features)
|
| 532 |
+
|
| 533 |
+
# NEW: Add to buffer if provided
|
| 534 |
+
if sensor_buffer is not None:
|
| 535 |
+
sensor_buffer.add_sample(features_scaled[0]) # Add 1D array
|
| 536 |
+
|
| 537 |
+
# Stage 1: Anomaly Detection
|
| 538 |
+
is_anomaly, s1_conf, score = predict_anomaly(features_scaled, models)
|
| 539 |
+
|
| 540 |
+
# Stage 2: State Classification (NOW WITH BUFFER)
|
| 541 |
+
state, s2_conf, state_probs = predict_state(
|
| 542 |
+
features_scaled,
|
| 543 |
+
models,
|
| 544 |
+
is_anomaly,
|
| 545 |
+
sensor_buffer=sensor_buffer # PASS BUFFER
|
| 546 |
+
)
|
| 547 |
+
|
| 548 |
+
# Stage 3: Component Identification
|
| 549 |
+
component, s3_conf, top3 = predict_component(features_scaled, models, state)
|
| 550 |
+
|
| 551 |
+
# Compile results
|
| 552 |
+
result = {
|
| 553 |
+
'success': True,
|
| 554 |
+
'timestamp': pd.Timestamp.now(),
|
| 555 |
+
'stage1': {
|
| 556 |
+
'is_anomaly': is_anomaly,
|
| 557 |
+
'confidence': s1_conf,
|
| 558 |
+
'score': score,
|
| 559 |
+
'model_type': models['stage1_type']
|
| 560 |
+
},
|
| 561 |
+
'stage2': {
|
| 562 |
+
'state': state,
|
| 563 |
+
'confidence': s2_conf,
|
| 564 |
+
'probabilities': {
|
| 565 |
+
'ANOMALY': state_probs[0] if len(state_probs) > 0 else 0.0,
|
| 566 |
+
'DEGRADING': state_probs[1] if len(state_probs) > 1 else 0.0,
|
| 567 |
+
'FAULTED': state_probs[2] if len(state_probs) > 2 else 0.0
|
| 568 |
+
},
|
| 569 |
+
'model_type': models['stage2_type']
|
| 570 |
+
},
|
| 571 |
+
'stage3': {
|
| 572 |
+
'component': component,
|
| 573 |
+
'confidence': s3_conf,
|
| 574 |
+
'top3': top3,
|
| 575 |
+
'model_type': models['stage3_type']
|
| 576 |
+
}
|
| 577 |
+
}
|
| 578 |
+
|
| 579 |
+
# NEW: Add buffer status to result
|
| 580 |
+
if sensor_buffer is not None:
|
| 581 |
+
result['buffer_status'] = {
|
| 582 |
+
'size': sensor_buffer.size(),
|
| 583 |
+
'ready': sensor_buffer.is_ready(),
|
| 584 |
+
'using_buffer': sensor_buffer.is_ready() and models['stage2_type'] in ['lstm', 'cnn']
|
| 585 |
+
}
|
| 586 |
+
# Build component health + recommended actions
|
| 587 |
+
component_health = get_component_health(result)
|
| 588 |
+
result['component_health'] = component_health # optional (for dashboard)
|
| 589 |
+
|
| 590 |
+
result['actions'] = get_recommended_actions(result, component_health)
|
| 591 |
+
|
| 592 |
+
# Now trigger alerts (email uses result['actions'])
|
| 593 |
+
alert_status = trigger_alerts(result)
|
| 594 |
+
result['alerts_sent'] = alert_status
|
| 595 |
+
return result
|
| 596 |
+
|
| 597 |
+
except Exception as e:
|
| 598 |
+
return {
|
| 599 |
+
'success': False,
|
| 600 |
+
'error': str(e),
|
| 601 |
+
'stage1': {'is_anomaly': False, 'confidence': 0.0},
|
| 602 |
+
'stage2': {'state': 'ERROR', 'confidence': 0.0},
|
| 603 |
+
'stage3': {'component': None, 'confidence': 0.0}
|
| 604 |
+
}
|
| 605 |
+
|
| 606 |
+
# ============================================================================
|
| 607 |
+
# COMPONENT HEALTH SUMMARY
|
| 608 |
+
# ============================================================================
|
| 609 |
+
|
| 610 |
+
def get_component_health(prediction):
|
| 611 |
+
"""
|
| 612 |
+
Generate health status for all components.
|
| 613 |
+
|
| 614 |
+
Returns:
|
| 615 |
+
dict: {component_name: status_dict}
|
| 616 |
+
"""
|
| 617 |
+
health = {}
|
| 618 |
+
|
| 619 |
+
state = prediction['stage2']['state']
|
| 620 |
+
identified_comp = prediction['stage3']['component']
|
| 621 |
+
top3 = prediction['stage3'].get('top3', [])
|
| 622 |
+
|
| 623 |
+
for comp in COMPONENTS:
|
| 624 |
+
if state == "NORMAL":
|
| 625 |
+
health[comp] = {
|
| 626 |
+
'status': 'NORMAL',
|
| 627 |
+
'icon': '🟢',
|
| 628 |
+
'confidence': 1.0,
|
| 629 |
+
'message': 'Operating normally'
|
| 630 |
+
}
|
| 631 |
+
elif comp == identified_comp:
|
| 632 |
+
# Primary suspect
|
| 633 |
+
conf = prediction['stage3']['confidence']
|
| 634 |
+
if state == "FAULTED":
|
| 635 |
+
health[comp] = {
|
| 636 |
+
'status': 'FAULTED',
|
| 637 |
+
'icon': '🔴',
|
| 638 |
+
'confidence': conf,
|
| 639 |
+
'message': f'FAULTED - Immediate action required'
|
| 640 |
+
}
|
| 641 |
+
elif state == "DEGRADING":
|
| 642 |
+
health[comp] = {
|
| 643 |
+
'status': 'DEGRADING',
|
| 644 |
+
'icon': '🟡',
|
| 645 |
+
'confidence': conf,
|
| 646 |
+
'message': f'Degrading - Schedule maintenance'
|
| 647 |
+
}
|
| 648 |
+
elif state == "ANOMALY":
|
| 649 |
+
health[comp] = {
|
| 650 |
+
'status': 'MONITOR',
|
| 651 |
+
'icon': '🟠',
|
| 652 |
+
'confidence': conf,
|
| 653 |
+
'message': 'Anomaly detected - Monitor closely'
|
| 654 |
+
}
|
| 655 |
+
elif any(comp == t[0] for t in top3):
|
| 656 |
+
# In top 3 - possible suspect
|
| 657 |
+
conf = next(t[1] for t in top3 if t[0] == comp)
|
| 658 |
+
health[comp] = {
|
| 659 |
+
'status': 'MONITOR',
|
| 660 |
+
'icon': '🟠',
|
| 661 |
+
'confidence': conf,
|
| 662 |
+
'message': f'Check (Top-3: {conf*100:.0f}%)'
|
| 663 |
+
}
|
| 664 |
+
else:
|
| 665 |
+
# Not identified
|
| 666 |
+
health[comp] = {
|
| 667 |
+
'status': 'NORMAL',
|
| 668 |
+
'icon': '🟢',
|
| 669 |
+
'confidence': 0.0,
|
| 670 |
+
'message': 'Operating normally'
|
| 671 |
+
}
|
| 672 |
+
|
| 673 |
+
return health
|
| 674 |
+
|
| 675 |
+
|
| 676 |
+
# ============================================================================
|
| 677 |
+
# RECOMMENDED ACTIONS
|
| 678 |
+
# ============================================================================
|
| 679 |
+
|
| 680 |
+
def get_recommended_actions(prediction, component_health):
|
| 681 |
+
"""
|
| 682 |
+
Generate maintenance recommendations based on predictions.
|
| 683 |
+
|
| 684 |
+
Returns:
|
| 685 |
+
list: List of action strings
|
| 686 |
+
"""
|
| 687 |
+
actions = []
|
| 688 |
+
|
| 689 |
+
state = prediction['stage2']['state']
|
| 690 |
+
component = prediction['stage3']['component']
|
| 691 |
+
|
| 692 |
+
if state == "NORMAL":
|
| 693 |
+
actions.append("✅ System operating normally - No action required")
|
| 694 |
+
return actions
|
| 695 |
+
else:
|
| 696 |
+
if state != "FAULTED" and state != "DEGRADING":
|
| 697 |
+
actions.append(f"⚠️ Suspected component: {component}")
|
| 698 |
+
|
| 699 |
+
if component:
|
| 700 |
+
# Component-specific actions
|
| 701 |
+
comp_actions = {
|
| 702 |
+
'P101': [
|
| 703 |
+
'🔧 Inspect intake pump P101 for wear',
|
| 704 |
+
'📋 Check pump vibration and bearing temperature',
|
| 705 |
+
'💧 Verify intake flow rate and pressure'
|
| 706 |
+
],
|
| 707 |
+
'P201': [
|
| 708 |
+
'🔧 Inspect NaCl dosing pump P201',
|
| 709 |
+
'📋 Check chemical feed lines for blockages',
|
| 710 |
+
'⚗️ Verify NaCl concentration and flow rate'
|
| 711 |
+
],
|
| 712 |
+
'P203': [
|
| 713 |
+
'🔧 Inspect HCl dosing pump P203',
|
| 714 |
+
'📋 Check acid feed system for leaks',
|
| 715 |
+
'⚗️ Calibrate pH sensor AIT202'
|
| 716 |
+
],
|
| 717 |
+
'P205': [
|
| 718 |
+
'🔧 Inspect NaOCl dosing pump P205',
|
| 719 |
+
'📋 Check chlorine feed system',
|
| 720 |
+
'⚗️ Verify ORP readings (AIT203)'
|
| 721 |
+
],
|
| 722 |
+
'P302': [
|
| 723 |
+
'🔧 Inspect UF feed pump P302',
|
| 724 |
+
'📋 Check UF membrane differential pressure',
|
| 725 |
+
'💧 Consider membrane backwash or cleaning'
|
| 726 |
+
],
|
| 727 |
+
'P402': [
|
| 728 |
+
'🔧 Inspect dechlorination feed pump P402',
|
| 729 |
+
'📋 Check UV system operation',
|
| 730 |
+
'⚗️ Verify ORP levels post-UV'
|
| 731 |
+
],
|
| 732 |
+
'P403': [
|
| 733 |
+
'🔧 Inspect NaHSO₄ dosing pump P403',
|
| 734 |
+
'📋 Check dechlorination efficiency',
|
| 735 |
+
'⚗️ Verify residual chlorine levels'
|
| 736 |
+
],
|
| 737 |
+
'P501': [
|
| 738 |
+
'🔧 Inspect RO feed pump P501',
|
| 739 |
+
'📋 Check RO membrane pressure and flow',
|
| 740 |
+
'💧 Monitor permeate quality (TDS)'
|
| 741 |
+
],
|
| 742 |
+
'MV101': [
|
| 743 |
+
'🔧 Inspect motorized valve MV101',
|
| 744 |
+
'📋 Check valve position and response',
|
| 745 |
+
'⚙️ Lubricate valve actuator'
|
| 746 |
+
],
|
| 747 |
+
'MV304': [
|
| 748 |
+
'🔧 Inspect motorized valve MV304',
|
| 749 |
+
'📋 Check UF backwash valve operation',
|
| 750 |
+
'⚙️ Verify valve seating'
|
| 751 |
+
]
|
| 752 |
+
}
|
| 753 |
+
if state == "FAULTED":
|
| 754 |
+
actions.append(f"🚨 URGENT: {component} has FAULTED")
|
| 755 |
+
actions.append("📞 Alert maintenance team")
|
| 756 |
+
actions.append(f"⏸️ Consider stopping {component} immediately")
|
| 757 |
+
# General actions based on state
|
| 758 |
+
if state == "DEGRADING":
|
| 759 |
+
actions.append(f"🚨 URGENT: {component} is DEGRADING")
|
| 760 |
+
actions.append("📅 Schedule maintenance within next 24-48 hours")
|
| 761 |
+
actions.append("📊 Increase monitoring frequency")
|
| 762 |
+
|
| 763 |
+
if component in comp_actions:
|
| 764 |
+
actions.extend(comp_actions[component][:3]) # Top 3 actions
|
| 765 |
+
|
| 766 |
+
|
| 767 |
+
return actions
|
| 768 |
+
|
| 769 |
+
|
| 770 |
+
# ============================================================================
|
| 771 |
+
# TESTING
|
| 772 |
+
# ============================================================================
|
| 773 |
+
|
| 774 |
+
if __name__ == "__main__":
|
| 775 |
+
print("\n" + "=" * 60)
|
| 776 |
+
print("TEST 1: Without Buffer (Old Way)")
|
| 777 |
+
print("=" * 60)
|
| 778 |
+
|
| 779 |
+
test_payload = {
|
| 780 |
+
'true_FIT101': 2.5, 'true_LIT101': 800.0,
|
| 781 |
+
'true_FIT201': 2.3, 'true_AIT201': 150.0, 'true_AIT202': 7.2, 'true_AIT203': 350.0,
|
| 782 |
+
'true_FIT301': 2.1, 'true_LIT301': 750.0, 'true_DPIT301': 0.15,
|
| 783 |
+
'true_FIT401': 2.0, 'true_LIT401': 700.0, 'true_AIT401': 85.0, 'true_AIT402': 280.0,
|
| 784 |
+
'true_FIT501': 1.8, 'true_PIT501': 5.2, 'true_AIT501': 6.8,
|
| 785 |
+
# Motor physics features
|
| 786 |
+
'true_P101_motor_temp': 45.0, 'true_P101_current': 3.5, 'true_P101_vibration': 1.2,
|
| 787 |
+
'true_P201_motor_temp': 42.0, 'true_P201_current': 2.8, 'true_P201_vibration': 0.9,
|
| 788 |
+
'true_P203_motor_temp': 40.0, 'true_P203_current': 2.5, 'true_P203_vibration': 0.8,
|
| 789 |
+
'true_P205_motor_temp': 43.0, 'true_P205_current': 3.0, 'true_P205_vibration': 1.0,
|
| 790 |
+
'true_P302_motor_temp': 46.0, 'true_P302_current': 3.8, 'true_P302_vibration': 1.3,
|
| 791 |
+
'true_P402_motor_temp': 44.0, 'true_P402_current': 3.2, 'true_P402_vibration': 1.1,
|
| 792 |
+
'true_P403_motor_temp': 41.0, 'true_P403_current': 2.7, 'true_P403_vibration': 0.9,
|
| 793 |
+
'true_P501_motor_temp': 48.0, 'true_P501_current': 4.0, 'true_P501_vibration': 1.4,
|
| 794 |
+
'true_MV101_motor_temp': 38.0, 'true_MV101_current': 1.5, 'true_MV101_vibration': 0.5,
|
| 795 |
+
'true_MV304_motor_temp': 39.0, 'true_MV304_current': 1.6, 'true_MV304_vibration': 0.6,
|
| 796 |
+
}
|
| 797 |
+
|
| 798 |
+
# Create buffer and add 60 samples
|
| 799 |
+
buffer = SensorBuffer(window_size=60, n_features=40)
|
| 800 |
+
|
| 801 |
+
for i in range(60):
|
| 802 |
+
result1 = run_pipeline(test_payload, sensor_buffer=buffer)
|
| 803 |
+
|
| 804 |
+
if result1['success']:
|
| 805 |
+
print(f"✅ Stage 1: {result1['stage1']['is_anomaly']}")
|
| 806 |
+
print(f"✅ Stage 2: {result1['stage2']['state']} (conf: {result1['stage2']['confidence']:.2f})")
|
| 807 |
+
print(f"✅ Stage 3: {result1['stage3']['component']}")
|
| 808 |
+
print(f"\n📊 Buffer Status:")
|
| 809 |
+
print(f" Size: {result1['buffer_status']['size']}/60")
|
| 810 |
+
print(f" Ready: {result1['buffer_status']['ready']}")
|
| 811 |
+
print(f" Using Buffer: {result1['buffer_status']['using_buffer']}")
|
| 812 |
+
else:
|
| 813 |
+
print(f"\n❌ Pipeline Error: {result1['error']}")
|
requirements.txt
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
flask==3.0.0
|
| 2 |
+
flask-cors==4.0.0
|
sensor_buffer.py
ADDED
|
@@ -0,0 +1,97 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Sensor Buffer for Rolling Window
|
| 3 |
+
=================================
|
| 4 |
+
Maintains last 60 samples for LSTM/CNN inference.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
import numpy as np
|
| 8 |
+
from collections import deque
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
class SensorBuffer:
|
| 12 |
+
"""
|
| 13 |
+
Rolling buffer that stores last N sensor readings.
|
| 14 |
+
|
| 15 |
+
Usage:
|
| 16 |
+
buffer = SensorBuffer(window_size=60, n_features=40)
|
| 17 |
+
|
| 18 |
+
# Add new sample
|
| 19 |
+
buffer.add_sample(features_scaled)
|
| 20 |
+
|
| 21 |
+
# Check if ready
|
| 22 |
+
if buffer.is_ready():
|
| 23 |
+
sequence = buffer.get_sequence()
|
| 24 |
+
# Use sequence for LSTM/CNN
|
| 25 |
+
"""
|
| 26 |
+
|
| 27 |
+
def __init__(self, window_size=60, n_features=40):
|
| 28 |
+
"""
|
| 29 |
+
Initialize buffer.
|
| 30 |
+
|
| 31 |
+
Args:
|
| 32 |
+
window_size (int): Number of samples to store (default: 60)
|
| 33 |
+
n_features (int): Number of features per sample (default: 40)
|
| 34 |
+
"""
|
| 35 |
+
self.window_size = window_size
|
| 36 |
+
self.n_features = n_features
|
| 37 |
+
|
| 38 |
+
# Use deque for efficient FIFO operations
|
| 39 |
+
self.buffer = deque(maxlen=window_size)
|
| 40 |
+
|
| 41 |
+
#print(f"✅ SensorBuffer initialized: window={window_size}, features={n_features}")
|
| 42 |
+
|
| 43 |
+
def add_sample(self, features):
|
| 44 |
+
"""
|
| 45 |
+
Add new sample to buffer.
|
| 46 |
+
|
| 47 |
+
Args:
|
| 48 |
+
features (np.array): (40,) array of scaled features
|
| 49 |
+
"""
|
| 50 |
+
# Convert to 1D array if needed
|
| 51 |
+
if features.ndim == 2:
|
| 52 |
+
features = features.flatten()
|
| 53 |
+
|
| 54 |
+
# Add to buffer (automatically removes oldest if full)
|
| 55 |
+
self.buffer.append(features)
|
| 56 |
+
|
| 57 |
+
def get_sequence(self):
|
| 58 |
+
"""
|
| 59 |
+
Get sequence for LSTM/CNN (60 x 40 array).
|
| 60 |
+
|
| 61 |
+
Returns:
|
| 62 |
+
np.array: (window_size, n_features) array
|
| 63 |
+
"""
|
| 64 |
+
# If buffer not full yet, pad with zeros
|
| 65 |
+
if len(self.buffer) < self.window_size:
|
| 66 |
+
# Pad beginning with zeros
|
| 67 |
+
padding_needed = self.window_size - len(self.buffer)
|
| 68 |
+
padding = [np.zeros(self.n_features)] * padding_needed
|
| 69 |
+
return np.array(padding + list(self.buffer))
|
| 70 |
+
|
| 71 |
+
# Buffer is full - return as numpy array
|
| 72 |
+
return np.array(list(self.buffer))
|
| 73 |
+
|
| 74 |
+
def is_ready(self):
|
| 75 |
+
"""
|
| 76 |
+
Check if buffer has enough samples for reliable prediction.
|
| 77 |
+
|
| 78 |
+
Returns:
|
| 79 |
+
bool: True if buffer has at least window_size samples
|
| 80 |
+
"""
|
| 81 |
+
return len(self.buffer) >= self.window_size
|
| 82 |
+
|
| 83 |
+
def size(self):
|
| 84 |
+
"""Current number of samples in buffer"""
|
| 85 |
+
return len(self.buffer)
|
| 86 |
+
|
| 87 |
+
def clear(self):
|
| 88 |
+
"""Clear all samples from buffer"""
|
| 89 |
+
self.buffer.clear()
|
| 90 |
+
|
| 91 |
+
def __len__(self):
|
| 92 |
+
return len(self.buffer)
|
| 93 |
+
|
| 94 |
+
def __repr__(self):
|
| 95 |
+
return f"SensorBuffer(size={len(self.buffer)}/{self.window_size}, features={self.n_features})"
|
| 96 |
+
|
| 97 |
+
|