import os import json from datetime import datetime import hashlib # Simple JSON file storage for now - we'll add HF Datasets later ALERTS_FILE = "/tmp/gbv_alerts.json" def generate_alert_id(article_url, content): """Generate unique ID to prevent duplicates""" unique_string = f"{article_url}_{content[:100]}" return hashlib.md5(unique_string.encode()).hexdigest() def load_alerts(): """Load alerts from JSON file""" try: with open(ALERTS_FILE, 'r') as f: return json.load(f) except: return [] def save_alerts(alerts): """Save alerts to JSON file""" try: with open(ALERTS_FILE, 'w') as f: json.dump(alerts, f, indent=2) return True except: return False def alert_exists(alert_id): """Check if alert already exists""" alerts = load_alerts() for alert in alerts: if alert.get('alert_id') == alert_id: return True return False def save_news_alert( source_site, article_title, article_url, content, threat_level, locations, severity_tier, model_confidence=None, enhanced_locations=None, ner_locations=None, rule_locations=None, emotional_boost=None, sentiment_label=None ): try: # Generate unique ID alert_id = generate_alert_id(article_url, content) # Check for duplicates if alert_exists(alert_id): print(f"⏭️ Duplicate alert skipped: {article_title[:50]}...") return False alerts = load_alerts() new_alert = { "alert_id": alert_id, "source_site": source_site, "article_title": article_title, "article_url": article_url, "content": content[:2000], "threat_level": threat_level, "locations": locations or [], "severity_tier": severity_tier, "model_confidence": model_confidence, "enhanced_locations": enhanced_locations or [], "ner_locations": ner_locations or [], "rule_locations": rule_locations or [], "emotional_boost": emotional_boost, "sentiment_label": sentiment_label, "created_at": datetime.utcnow().isoformat() } alerts.append(new_alert) save_alerts(alerts) print(f"✅ Saved alert: {article_title[:50]}...") return True except Exception as e: print(f"❌ Save error: {e}") return False def save_twitter_alert(username, content, keyword_found, threat_level): try: alert_id = generate_alert_id(username, content) if alert_exists(alert_id): print(f"⏭️ Duplicate Twitter alert skipped: {username}") return False alerts = load_alerts() new_alert = { "alert_id": alert_id, "username": username, "content": content[:500], "keyword_found": keyword_found, "threat_level": threat_level, "type": "twitter", "created_at": datetime.utcnow().isoformat(), "source_site": "twitter.com", "article_title": f"Twitter threat from {username}", "article_url": f"https://twitter.com/{username}", "locations": [], "severity_tier": "HIGH" if threat_level > 70 else "MEDIUM" } alerts.append(new_alert) save_alerts(alerts) print(f"✅ Saved Twitter alert: {username}") return True except Exception as e: print(f"❌ Twitter save error: {e}") return False def get_recent_alerts(limit=10): """Get recent alerts sorted by date""" try: alerts = load_alerts() alerts.sort(key=lambda x: x.get('created_at', ''), reverse=True) return alerts[:limit] except: return [] def get_alerts_by_severity(severity_tier): try: alerts = load_alerts() filtered = [alert for alert in alerts if alert.get('severity_tier') == severity_tier] filtered.sort(key=lambda x: x.get('created_at', ''), reverse=True) return filtered except: return []