/** * useLiveData - Universal hook for live data fetching in widgets * * Features: * - Automatic polling with configurable intervals * - WebSocket subscriptions for real-time updates * - Source recommendations with user-triggerable fetch * - Caching and error handling * - Connection state tracking */ import { useState, useEffect, useCallback, useRef } from 'react'; // Dynamic API base - use environment variable or HuggingFace Spaces URL in production const getApiBase = () => { // Check for Vite environment variable first if (typeof import.meta !== 'undefined' && import.meta.env?.VITE_API_URL) { return `${import.meta.env.VITE_API_URL}/api`; } // Production default: HuggingFace Spaces backend if (typeof window !== 'undefined' && window.location.hostname !== 'localhost') { return 'https://kraft102-widgettdc-api.hf.space/api'; } // Development default return 'http://localhost:3001/api'; }; const getWsBase = () => { // Check for Vite environment variable first if (typeof import.meta !== 'undefined' && import.meta.env?.VITE_WS_URL) { return import.meta.env.VITE_WS_URL; } // Production default: HuggingFace Spaces backend if (typeof window !== 'undefined' && window.location.hostname !== 'localhost') { return 'wss://kraft102-widgettdc-api.hf.space'; } // Development default return 'ws://localhost:3001'; }; const API_BASE = getApiBase(); const WS_BASE = getWsBase(); // Source types that widgets can request export type DataSourceType = | 'hyperlog' // Real-time events | 'alerts' // System alerts | 'autonomousStats' // Autonomous agent stats | 'sourcesHealth' // Data source health | 'decisionHistory' // AI decisions log | 'knowledge' // Knowledge graph data | 'systemHealth' // System services health | 'metrics' // Platform metrics | 'osint' // External OSINT feeds (future) | 'threatIntel' // Threat intelligence (future) | 'email'; // Email inbox data // Recommended external sources that can be fetched on demand export interface RecommendedSource { id: string; name: string; description: string; category: 'osint' | 'threatIntel' | 'social' | 'internal'; endpoint?: string; requiresApproval: boolean; isAvailable: boolean; } interface UseLiveDataOptions { sourceType: DataSourceType; pollInterval?: number; // ms, 0 = no polling wsEvents?: string[]; // WebSocket event types to subscribe transform?: (data: any) => T; // Transform raw API data enabled?: boolean; // Enable/disable fetching } interface UseLiveDataResult { data: T | null; isLoading: boolean; error: string | null; lastUpdated: Date | null; refresh: () => Promise; connectionStatus: 'connected' | 'connecting' | 'disconnected'; recommendedSources: RecommendedSource[]; fetchRecommendedSource: (sourceId: string) => Promise; } // API endpoint mapping const sourceEndpoints: Record = { hyperlog: '/hyper/events', alerts: '/mcp/autonomous/alerts', autonomousStats: '/mcp/autonomous/stats', sourcesHealth: '/mcp/autonomous/sources', decisionHistory: '/mcp/autonomous/decisions', knowledge: '/knowledge/compile', systemHealth: '/health', metrics: '/sys/metrics', osint: '/acquisition/osint', threatIntel: '/acquisition/threats', email: '/email/inbox', }; // Recommended sources per data type const recommendedSourcesMap: Record = { hyperlog: [ { id: 'syslog', name: 'System Logs', description: 'Local system event logs', category: 'internal', requiresApproval: false, isAvailable: true }, { id: 'auditlog', name: 'Audit Trail', description: 'Security audit events', category: 'internal', requiresApproval: false, isAvailable: true }, ], alerts: [ { id: 'selfehealing', name: 'Self-Healing Alerts', description: 'Autonomous recovery events', category: 'internal', requiresApproval: false, isAvailable: true }, { id: 'pattern-alerts', name: 'Pattern Detection', description: 'AI-detected anomalies', category: 'internal', requiresApproval: false, isAvailable: true }, ], autonomousStats: [], sourcesHealth: [], decisionHistory: [], knowledge: [ { id: 'neo4j-graph', name: 'Knowledge Graph', description: 'Neo4j entity relationships', category: 'internal', requiresApproval: false, isAvailable: true }, { id: 'embeddings', name: 'Semantic Embeddings', description: 'Vector similarity search', category: 'internal', requiresApproval: false, isAvailable: true }, ], systemHealth: [], metrics: [ { id: 'prometheus', name: 'Prometheus Metrics', description: 'Time-series system metrics', category: 'internal', requiresApproval: false, isAvailable: false }, ], osint: [ { id: 'shodan', name: 'Shodan', description: 'Internet-connected device search', category: 'osint', requiresApproval: true, isAvailable: false }, { id: 'virustotal', name: 'VirusTotal', description: 'File and URL analysis', category: 'threatIntel', requiresApproval: true, isAvailable: false }, { id: 'abuseipdb', name: 'AbuseIPDB', description: 'IP reputation database', category: 'threatIntel', requiresApproval: true, isAvailable: false }, ], threatIntel: [ { id: 'otx', name: 'AlienVault OTX', description: 'Open Threat Exchange IOCs', category: 'threatIntel', requiresApproval: true, isAvailable: false }, { id: 'mitre', name: 'MITRE ATT&CK', description: 'Adversary tactics & techniques', category: 'threatIntel', requiresApproval: false, isAvailable: true }, ], email: [ { id: 'outlook', name: 'Outlook/Office365', description: 'Microsoft email via IMAP', category: 'internal', requiresApproval: false, isAvailable: true }, { id: 'gmail', name: 'Gmail', description: 'Google email via IMAP', category: 'internal', requiresApproval: false, isAvailable: true }, ], }; export function useLiveData(options: UseLiveDataOptions): UseLiveDataResult { const { sourceType, pollInterval = 30000, wsEvents = [], transform, enabled = true, } = options; const [data, setData] = useState(null); const [isLoading, setIsLoading] = useState(true); const [error, setError] = useState(null); const [lastUpdated, setLastUpdated] = useState(null); const [connectionStatus, setConnectionStatus] = useState<'connected' | 'connecting' | 'disconnected'>('connecting'); const [recommendedSources, setRecommendedSources] = useState( recommendedSourcesMap[sourceType] || [] ); const wsRef = useRef(null); const pollRef = useRef(null); // Fetch data from API const fetchData = useCallback(async () => { if (!enabled) return; const endpoint = sourceEndpoints[sourceType]; if (!endpoint) { setError(`Unknown source type: ${sourceType}`); setIsLoading(false); return; } try { const response = await fetch(`${API_BASE}${endpoint}`); if (!response.ok) { throw new Error(`HTTP ${response.status}: ${response.statusText}`); } const rawData = await response.json(); const transformedData = transform ? transform(rawData) : rawData; setData(transformedData); setError(null); setLastUpdated(new Date()); setConnectionStatus('connected'); } catch (err: any) { console.error(`[useLiveData] ${sourceType} fetch error:`, err); setError(err.message || 'Failed to fetch data'); setConnectionStatus('disconnected'); } finally { setIsLoading(false); } }, [enabled, sourceType, transform]); // Refresh function const refresh = useCallback(async () => { setIsLoading(true); await fetchData(); }, [fetchData]); // Fetch a recommended source (user-triggered) const fetchRecommendedSource = useCallback(async (sourceId: string) => { const source = recommendedSources.find(s => s.id === sourceId); if (!source) { console.warn(`Unknown source: ${sourceId}`); return; } try { // Call backend to enable/fetch this source const response = await fetch(`${API_BASE}/acquisition/enable-source`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ sourceId, sourceName: source.name, category: source.category, requiresApproval: source.requiresApproval, }), }); if (response.ok) { // Update source availability setRecommendedSources(prev => prev.map(s => s.id === sourceId ? { ...s, isAvailable: true } : s ) ); // Refresh main data await refresh(); } else { console.error(`Failed to enable source ${sourceId}`); } } catch (err) { console.error(`Error enabling source ${sourceId}:`, err); } }, [recommendedSources, refresh]); // Setup polling useEffect(() => { if (!enabled) return; fetchData(); if (pollInterval > 0) { pollRef.current = setInterval(fetchData, pollInterval); } return () => { if (pollRef.current) { clearInterval(pollRef.current); } }; }, [enabled, pollInterval, fetchData]); // Setup WebSocket subscription useEffect(() => { if (!enabled || wsEvents.length === 0) return; const wsUrl = `${WS_BASE}/mcp/ws`; try { wsRef.current = new WebSocket(wsUrl); wsRef.current.onopen = () => { setConnectionStatus('connected'); // Subscribe to events wsRef.current?.send(JSON.stringify({ type: 'subscribe', events: wsEvents, })); }; wsRef.current.onmessage = (event) => { try { const message = JSON.parse(event.data); if (wsEvents.includes(message.type)) { // Real-time update - refresh data fetchData(); } } catch (err) { console.warn('WebSocket message parse error:', err); } }; wsRef.current.onclose = () => { setConnectionStatus('disconnected'); }; wsRef.current.onerror = () => { setConnectionStatus('disconnected'); }; } catch (err) { console.error('WebSocket connection error:', err); setConnectionStatus('disconnected'); } return () => { if (wsRef.current) { wsRef.current.close(); } }; }, [enabled, wsEvents, fetchData]); return { data, isLoading, error, lastUpdated, refresh, connectionStatus, recommendedSources, fetchRecommendedSource, }; } // Specialized hooks for common data types export function useAlerts(pollInterval = 15000) { return useLiveData({ sourceType: 'alerts', pollInterval, wsEvents: ['system.alert', 'security.alert', 'failure:recorded'], transform: (data) => data.alerts || data.events || [], }); } export function useHyperEvents(pollInterval = 10000) { return useLiveData({ sourceType: 'hyperlog', pollInterval, wsEvents: ['agent.log', 'mcp.tool.executed'], transform: (data) => ({ events: data.events || [], metrics: data.metrics || {}, }), }); } export function useSystemMetrics(pollInterval = 30000) { return useLiveData({ sourceType: 'metrics', pollInterval, transform: (data) => ({ cpu: data.cpu || 0, memory: data.memory || 0, uptime: data.uptime || 0, networkIn: data.network?.incoming || 0, networkOut: data.network?.outgoing || 0, }), }); } export function useSourcesHealth(pollInterval = 20000) { return useLiveData({ sourceType: 'sourcesHealth', pollInterval, wsEvents: ['source:health'], transform: (data) => data.sources || [], }); } export function useDecisionHistory(pollInterval = 30000) { return useLiveData({ sourceType: 'decisionHistory', pollInterval, wsEvents: ['decision:made'], transform: (data) => data.decisions || [], }); } export function useEmailInbox(pollInterval = 60000) { return useLiveData({ sourceType: 'email', pollInterval, wsEvents: ['email:refresh'], transform: (data) => ({ emails: data.emails || [], unreadCount: data.unreadCount || 0, source: data.source || 'unknown', lastFetch: data.lastFetch || null, }), }); }