Spaces:
Sleeping
Sleeping
Update main.py
Browse files
main.py
CHANGED
|
@@ -5,6 +5,7 @@ import requests
|
|
| 5 |
import random
|
| 6 |
import logging
|
| 7 |
import json
|
|
|
|
| 8 |
from datetime import datetime
|
| 9 |
from dotenv import load_dotenv
|
| 10 |
import threading
|
|
@@ -16,9 +17,6 @@ import uvicorn
|
|
| 16 |
# Task 4 & 5 dependencies (Sendgrid API & Logging)
|
| 17 |
from urllib.error import HTTPError
|
| 18 |
|
| 19 |
-
# Task 4 & 5 dependencies (Sendgrid API & Logging)
|
| 20 |
-
from urllib.error import HTTPError
|
| 21 |
-
|
| 22 |
# Load optional .env if present in same directory
|
| 23 |
load_dotenv()
|
| 24 |
|
|
@@ -64,8 +62,6 @@ STATE_FILE = "state.txt"
|
|
| 64 |
LAST_EVENT_ID = None
|
| 65 |
LAST_EVENT_CODE = None
|
| 66 |
SESSION_EXPIRED = False
|
| 67 |
-
EXPIRED_XSRF = None
|
| 68 |
-
EXPIRED_BIP = None
|
| 69 |
|
| 70 |
# ==============================================================================
|
| 71 |
# STATE MANAGEMENT (Task 1)
|
|
@@ -88,33 +84,6 @@ def save_state(event_id):
|
|
| 88 |
f.write(str(event_id))
|
| 89 |
except Exception as e:
|
| 90 |
logger.error(f"Failed to write state file: {e}")
|
| 91 |
-
# Environment values (Make sure to populate your .env file)
|
| 92 |
-
# We will reload these inside the loop so you can change them on the fly.
|
| 93 |
-
|
| 94 |
-
# Email settings (Loaded from .env)
|
| 95 |
-
# The email you are sending FROM and TO (can be the same)
|
| 96 |
-
EMAIL_ADDRESS = os.getenv("EMAIL_ADDRESS", "")
|
| 97 |
-
# Gmail App Password (NOT your regular password)
|
| 98 |
-
EMAIL_PASSWORD = os.getenv("EMAIL_PASSWORD", "")
|
| 99 |
-
# Notification recipient
|
| 100 |
-
EMAIL_RECIPIENT = os.getenv("EMAIL_RECIPIENT", EMAIL_ADDRESS)
|
| 101 |
-
|
| 102 |
-
# App check interval in seconds (default 60 secs = 1 min)
|
| 103 |
-
CHECK_INTERVAL_SECONDS = 60
|
| 104 |
-
|
| 105 |
-
BIP_API = "https://bip.bitsathy.ac.in/nova-api/student-activity-masters"
|
| 106 |
-
HEADERS = {
|
| 107 |
-
"accept": "application/json",
|
| 108 |
-
"x-requested-with": "XMLHttpRequest",
|
| 109 |
-
}
|
| 110 |
-
|
| 111 |
-
# State tracking for the loop
|
| 112 |
-
LAST_EVENT_ID = None
|
| 113 |
-
LAST_EVENT_CODE = None
|
| 114 |
-
SESSION_EXPIRED = False
|
| 115 |
-
EXPIRED_XSRF = None
|
| 116 |
-
EXPIRED_BIP = None
|
| 117 |
-
|
| 118 |
|
| 119 |
# ==============================================================================
|
| 120 |
# EMAIL NOTIFIER
|
|
@@ -195,7 +164,7 @@ def send_event_alerts(events):
|
|
| 195 |
if not events:
|
| 196 |
return
|
| 197 |
|
| 198 |
-
msg = f"π’ <b>{len(events)} New BIP Event Found!</b>
|
| 199 |
|
| 200 |
for i, ev in enumerate(events, 1):
|
| 201 |
# Format start date
|
|
@@ -227,6 +196,7 @@ def send_event_alerts(events):
|
|
| 227 |
# SCRAPER LOGIC
|
| 228 |
# ==============================================================================
|
| 229 |
def fetch_bip_events(xsrf_token, bip_session, page=1):
|
|
|
|
| 230 |
logger.debug(f"--> fetch_bip_events(page={page})")
|
| 231 |
|
| 232 |
cookies = {
|
|
@@ -245,7 +215,10 @@ def fetch_bip_events(xsrf_token, bip_session, page=1):
|
|
| 245 |
# Check for session expiration
|
| 246 |
if "text/html" in r.headers.get("Content-Type", "") or r.status_code in [401, 403]:
|
| 247 |
logger.warning(f"Session expired detected! Content-type: {r.headers.get('Content-Type')}, Status: {r.status_code}")
|
|
|
|
| 248 |
return None, "Session expired or invalid cookies."
|
|
|
|
|
|
|
| 249 |
|
| 250 |
r.raise_for_status()
|
| 251 |
|
|
@@ -314,64 +287,85 @@ def check_new_events(last_id, xsrf_token, bip_session):
|
|
| 314 |
# SCHEDULER ENGINE
|
| 315 |
# ==============================================================================
|
| 316 |
def process_tick():
|
| 317 |
-
global LAST_EVENT_ID, LAST_EVENT_CODE
|
| 318 |
logger.debug("--- process_tick starting ---")
|
| 319 |
|
| 320 |
-
|
| 321 |
-
|
| 322 |
-
|
| 323 |
-
|
| 324 |
-
|
| 325 |
-
if not xsrf or not bip:
|
| 326 |
-
logger.warning("Skipping check: Please configure XSRF_TOKEN and BIP_SESSION in the deployment environment.")
|
| 327 |
-
os._exit(1)
|
| 328 |
-
|
| 329 |
-
# Task 1: Load state if we just started
|
| 330 |
-
if LAST_EVENT_ID is None:
|
| 331 |
-
load_state()
|
| 332 |
-
|
| 333 |
-
new_events, err = check_new_events(LAST_EVENT_ID, xsrf, bip)
|
| 334 |
-
|
| 335 |
-
if err:
|
| 336 |
-
logger.error(f"Error scraping events: {err}")
|
| 337 |
-
send_email_message(
|
| 338 |
-
"β οΈ BIP Scraper Error",
|
| 339 |
-
"β οΈ <b>Scraper Error!</b><br><br>"
|
| 340 |
-
f"The notifier encountered an error and has paused checking. Error:<br>"
|
| 341 |
-
f"<code>{err}</code><br><br>"
|
| 342 |
-
"Please update the `XSRF_TOKEN` and `BIP_SESSION` variables in your Secret/Env configuration and restart the Space.",
|
| 343 |
-
is_html=True
|
| 344 |
-
)
|
| 345 |
-
logger.error("Notifier is shutting down completely because of the scraping error.")
|
| 346 |
-
os._exit(1)
|
| 347 |
|
| 348 |
-
|
| 349 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 350 |
if LAST_EVENT_ID is None:
|
| 351 |
-
|
| 352 |
-
|
| 353 |
-
|
| 354 |
-
|
| 355 |
-
|
| 356 |
-
|
| 357 |
send_email_message(
|
| 358 |
-
"
|
| 359 |
-
|
| 360 |
-
f"
|
| 361 |
-
f"
|
|
|
|
| 362 |
is_html=True
|
| 363 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 364 |
else:
|
| 365 |
-
|
| 366 |
-
LAST_EVENT_ID = new_events[0]["id"]
|
| 367 |
-
LAST_EVENT_CODE = new_events[0].get('event_code', LAST_EVENT_ID)
|
| 368 |
-
save_state(LAST_EVENT_ID)
|
| 369 |
|
| 370 |
-
|
| 371 |
-
|
| 372 |
-
|
| 373 |
-
|
| 374 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 375 |
|
| 376 |
def list_all_events():
|
| 377 |
"""Fetches the first page of events from BIP and prints them."""
|
|
@@ -437,6 +431,29 @@ def test_email_alert():
|
|
| 437 |
else:
|
| 438 |
logger.error("β Failed to send test message. Check your .env configuration.")
|
| 439 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 440 |
def start_loop():
|
| 441 |
logger.info("=" * 60)
|
| 442 |
logger.info("π BIP CLI Notifier Started")
|
|
@@ -457,6 +474,18 @@ def start_loop():
|
|
| 457 |
time.sleep(min(5, remaining))
|
| 458 |
except KeyboardInterrupt:
|
| 459 |
logger.info("\nπ Keyboard interrupt received.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 460 |
finally:
|
| 461 |
logger.info("Cleaning up resources...")
|
| 462 |
try:
|
|
@@ -547,18 +576,21 @@ if __name__ == "__main__":
|
|
| 547 |
parser.add_argument("--list-all", action="store_true", help="List all recent events and exit")
|
| 548 |
parser.add_argument("--latest", action="store_true", help="Print details of the latest event and exit")
|
| 549 |
parser.add_argument("--test-alert", action="store_true", help="Send a test message to Email and exit")
|
|
|
|
| 550 |
parser.add_argument("--run", action="store_true", help="Start the continuous monitoring loop (via FastAPI)")
|
| 551 |
-
|
| 552 |
# Task 8: Fix duplicate parsing
|
| 553 |
args = parser.parse_args()
|
| 554 |
logger.debug(f"Parsed arguments: {args}")
|
| 555 |
-
|
| 556 |
if args.list_all:
|
| 557 |
list_all_events()
|
| 558 |
elif args.latest:
|
| 559 |
get_latest_event()
|
| 560 |
elif args.test_alert:
|
| 561 |
test_email_alert()
|
|
|
|
|
|
|
| 562 |
elif args.run:
|
| 563 |
# Launch FastAPI which internally starts the loop
|
| 564 |
port = int(os.getenv("PORT", 7860))
|
|
@@ -566,4 +598,4 @@ if __name__ == "__main__":
|
|
| 566 |
else:
|
| 567 |
# Default behavior: run FastAPI
|
| 568 |
port = int(os.getenv("PORT", 7860))
|
| 569 |
-
uvicorn.run(app, host="0.0.0.0", port=port)
|
|
|
|
| 5 |
import random
|
| 6 |
import logging
|
| 7 |
import json
|
| 8 |
+
import traceback
|
| 9 |
from datetime import datetime
|
| 10 |
from dotenv import load_dotenv
|
| 11 |
import threading
|
|
|
|
| 17 |
# Task 4 & 5 dependencies (Sendgrid API & Logging)
|
| 18 |
from urllib.error import HTTPError
|
| 19 |
|
|
|
|
|
|
|
|
|
|
| 20 |
# Load optional .env if present in same directory
|
| 21 |
load_dotenv()
|
| 22 |
|
|
|
|
| 62 |
LAST_EVENT_ID = None
|
| 63 |
LAST_EVENT_CODE = None
|
| 64 |
SESSION_EXPIRED = False
|
|
|
|
|
|
|
| 65 |
|
| 66 |
# ==============================================================================
|
| 67 |
# STATE MANAGEMENT (Task 1)
|
|
|
|
| 84 |
f.write(str(event_id))
|
| 85 |
except Exception as e:
|
| 86 |
logger.error(f"Failed to write state file: {e}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 87 |
|
| 88 |
# ==============================================================================
|
| 89 |
# EMAIL NOTIFIER
|
|
|
|
| 164 |
if not events:
|
| 165 |
return
|
| 166 |
|
| 167 |
+
msg = f"π’ <b>{len(events)} New BIP Event Found!</b><br><br>"
|
| 168 |
|
| 169 |
for i, ev in enumerate(events, 1):
|
| 170 |
# Format start date
|
|
|
|
| 196 |
# SCRAPER LOGIC
|
| 197 |
# ==============================================================================
|
| 198 |
def fetch_bip_events(xsrf_token, bip_session, page=1):
|
| 199 |
+
global SESSION_EXPIRED
|
| 200 |
logger.debug(f"--> fetch_bip_events(page={page})")
|
| 201 |
|
| 202 |
cookies = {
|
|
|
|
| 215 |
# Check for session expiration
|
| 216 |
if "text/html" in r.headers.get("Content-Type", "") or r.status_code in [401, 403]:
|
| 217 |
logger.warning(f"Session expired detected! Content-type: {r.headers.get('Content-Type')}, Status: {r.status_code}")
|
| 218 |
+
SESSION_EXPIRED = True
|
| 219 |
return None, "Session expired or invalid cookies."
|
| 220 |
+
else:
|
| 221 |
+
SESSION_EXPIRED = False
|
| 222 |
|
| 223 |
r.raise_for_status()
|
| 224 |
|
|
|
|
| 287 |
# SCHEDULER ENGINE
|
| 288 |
# ==============================================================================
|
| 289 |
def process_tick():
|
| 290 |
+
global LAST_EVENT_ID, LAST_EVENT_CODE
|
| 291 |
logger.debug("--- process_tick starting ---")
|
| 292 |
|
| 293 |
+
try:
|
| 294 |
+
# Reload environment variables on every tick
|
| 295 |
+
load_dotenv(override=True)
|
| 296 |
+
xsrf = os.getenv("XSRF_TOKEN", "")
|
| 297 |
+
bip = os.getenv("BIP_SESSION", "")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 298 |
|
| 299 |
+
if not xsrf or not bip:
|
| 300 |
+
logger.warning("Skipping check: Please configure XSRF_TOKEN and BIP_SESSION in the deployment environment.")
|
| 301 |
+
send_email_message(
|
| 302 |
+
"β οΈ BIP Scraper Error",
|
| 303 |
+
"β οΈ <b>Deployment Configuration Error!</b><br><br>"
|
| 304 |
+
"The application was started without the required <code>XSRF_TOKEN</code> or <code>BIP_SESSION</code> secrets.<br><br>"
|
| 305 |
+
"Please configure these variables in your deployment settings to begin tracking.",
|
| 306 |
+
is_html=True
|
| 307 |
+
)
|
| 308 |
+
raise SystemExit(1)
|
| 309 |
+
|
| 310 |
+
# Task 1: Load state if we just started
|
| 311 |
if LAST_EVENT_ID is None:
|
| 312 |
+
load_state()
|
| 313 |
+
|
| 314 |
+
new_events, err = check_new_events(LAST_EVENT_ID, xsrf, bip)
|
| 315 |
+
|
| 316 |
+
if err:
|
| 317 |
+
logger.error(f"Error scraping events: {err}")
|
| 318 |
send_email_message(
|
| 319 |
+
"β οΈ BIP Scraper Error",
|
| 320 |
+
"β οΈ <b>Scraper Error!</b><br><br>"
|
| 321 |
+
f"The notifier encountered an error and has paused checking. Error:<br>"
|
| 322 |
+
f"<code>{err}</code><br><br>"
|
| 323 |
+
"Please update the `XSRF_TOKEN` and `BIP_SESSION` variables in your Secret/Env configuration and restart the Space.",
|
| 324 |
is_html=True
|
| 325 |
)
|
| 326 |
+
logger.error("Notifier is shutting down completely because of the scraping error.")
|
| 327 |
+
raise SystemExit(1)
|
| 328 |
+
|
| 329 |
+
if new_events:
|
| 330 |
+
# If LAST_EVENT_ID is None, it's the very first startup run. Set ID without alerting.
|
| 331 |
+
if LAST_EVENT_ID is None:
|
| 332 |
+
LAST_EVENT_ID = new_events[0]["id"]
|
| 333 |
+
LAST_EVENT_CODE = new_events[0].get('event_code', LAST_EVENT_ID)
|
| 334 |
+
save_state(LAST_EVENT_ID)
|
| 335 |
+
logger.info(f"EVENT ID : {LAST_EVENT_CODE} (Tracking started)")
|
| 336 |
+
|
| 337 |
+
# Send the startup notification
|
| 338 |
+
send_email_message(
|
| 339 |
+
"π BIP Notifier is Online!",
|
| 340 |
+
f"You are receiving this because the <b>BIP Auto Notifier</b> script has successfully started tracking on the cloud.<br><br>"
|
| 341 |
+
f"<b>Current Active Event:</b> {LAST_EVENT_CODE}<br>"
|
| 342 |
+
f"The script is now monitoring in the background. You will receive alerts for any newer events.",
|
| 343 |
+
is_html=True
|
| 344 |
+
)
|
| 345 |
+
else:
|
| 346 |
+
send_event_alerts(new_events)
|
| 347 |
+
LAST_EVENT_ID = new_events[0]["id"]
|
| 348 |
+
LAST_EVENT_CODE = new_events[0].get('event_code', LAST_EVENT_ID)
|
| 349 |
+
save_state(LAST_EVENT_ID)
|
| 350 |
+
|
| 351 |
+
for ev in new_events:
|
| 352 |
+
code = ev.get('event_code', ev['id'])
|
| 353 |
+
logger.info(f"π¨ NEW EVENT ID : {code} (Alert Sent!)")
|
| 354 |
else:
|
| 355 |
+
logger.info(f"EVENT ID : {LAST_EVENT_CODE}")
|
|
|
|
|
|
|
|
|
|
| 356 |
|
| 357 |
+
except Exception as e:
|
| 358 |
+
logger.error(f"CRITICAL EXCEPTION in process_tick: {e}")
|
| 359 |
+
error_details = traceback.format_exc()
|
| 360 |
+
logger.error(error_details)
|
| 361 |
+
send_email_message(
|
| 362 |
+
"π¨ CRITICAL: BIP Notifier Tick Crashed",
|
| 363 |
+
f"<b>The notifier encountered an unexpected exception during process_tick data parsing.</b><br><br>"
|
| 364 |
+
f"<b>Error Traceback:</b><br><pre>{error_details}</pre><br>"
|
| 365 |
+
f"The application has successfully caught this error, and is safely shutting down to prevent mail loops.",
|
| 366 |
+
is_html=True
|
| 367 |
+
)
|
| 368 |
+
raise SystemExit(1)
|
| 369 |
|
| 370 |
def list_all_events():
|
| 371 |
"""Fetches the first page of events from BIP and prints them."""
|
|
|
|
| 431 |
else:
|
| 432 |
logger.error("β Failed to send test message. Check your .env configuration.")
|
| 433 |
|
| 434 |
+
def test_real_event_alert():
|
| 435 |
+
"""Fetches the actual latest event from BIP and sends it as a test alert."""
|
| 436 |
+
logger.info("Fetching the real latest event to send as a test alert...")
|
| 437 |
+
|
| 438 |
+
load_dotenv(override=True)
|
| 439 |
+
xsrf = os.getenv("XSRF_TOKEN", "")
|
| 440 |
+
bip = os.getenv("BIP_SESSION", "")
|
| 441 |
+
|
| 442 |
+
data, err = fetch_bip_events(xsrf, bip, page=1)
|
| 443 |
+
if err:
|
| 444 |
+
logger.error(f"Error fetching real event: {err}")
|
| 445 |
+
return
|
| 446 |
+
|
| 447 |
+
resources = data.get("resources", [])
|
| 448 |
+
if not resources:
|
| 449 |
+
logger.warning("No events found to test with.")
|
| 450 |
+
return
|
| 451 |
+
|
| 452 |
+
ev = parse_event(resources[0])
|
| 453 |
+
logger.info(f"Triggering send_event_alerts with real event data: {ev.get('event_code')}")
|
| 454 |
+
send_event_alerts([ev])
|
| 455 |
+
logger.info("β
Real latest event alert sent successfully!")
|
| 456 |
+
|
| 457 |
def start_loop():
|
| 458 |
logger.info("=" * 60)
|
| 459 |
logger.info("π BIP CLI Notifier Started")
|
|
|
|
| 474 |
time.sleep(min(5, remaining))
|
| 475 |
except KeyboardInterrupt:
|
| 476 |
logger.info("\nπ Keyboard interrupt received.")
|
| 477 |
+
except Exception as e:
|
| 478 |
+
logger.error(f"FATAL SYSTEM ERROR in start_loop: {e}")
|
| 479 |
+
error_details = traceback.format_exc()
|
| 480 |
+
logger.error(error_details)
|
| 481 |
+
send_email_message(
|
| 482 |
+
"π¨ CRITICAL: BIP Notifier Crashed",
|
| 483 |
+
f"<b>The notifier encountered an unexpected fatal system error and has violently shut down.</b><br><br>"
|
| 484 |
+
f"<b>Error Traceback:</b><br><pre>{error_details}</pre><br>"
|
| 485 |
+
f"The application has been stopped to prevent instability. Please check your deployment logs.",
|
| 486 |
+
is_html=True
|
| 487 |
+
)
|
| 488 |
+
raise SystemExit(1)
|
| 489 |
finally:
|
| 490 |
logger.info("Cleaning up resources...")
|
| 491 |
try:
|
|
|
|
| 576 |
parser.add_argument("--list-all", action="store_true", help="List all recent events and exit")
|
| 577 |
parser.add_argument("--latest", action="store_true", help="Print details of the latest event and exit")
|
| 578 |
parser.add_argument("--test-alert", action="store_true", help="Send a test message to Email and exit")
|
| 579 |
+
parser.add_argument("--test-real-event", action="store_true", help="Fetch the actual latest event and send it as an alert")
|
| 580 |
parser.add_argument("--run", action="store_true", help="Start the continuous monitoring loop (via FastAPI)")
|
| 581 |
+
|
| 582 |
# Task 8: Fix duplicate parsing
|
| 583 |
args = parser.parse_args()
|
| 584 |
logger.debug(f"Parsed arguments: {args}")
|
| 585 |
+
|
| 586 |
if args.list_all:
|
| 587 |
list_all_events()
|
| 588 |
elif args.latest:
|
| 589 |
get_latest_event()
|
| 590 |
elif args.test_alert:
|
| 591 |
test_email_alert()
|
| 592 |
+
elif args.test_real_event:
|
| 593 |
+
test_real_event_alert()
|
| 594 |
elif args.run:
|
| 595 |
# Launch FastAPI which internally starts the loop
|
| 596 |
port = int(os.getenv("PORT", 7860))
|
|
|
|
| 598 |
else:
|
| 599 |
# Default behavior: run FastAPI
|
| 600 |
port = int(os.getenv("PORT", 7860))
|
| 601 |
+
uvicorn.run(app, host="0.0.0.0", port=port)
|