apps/backend/src/routes/datasources.ts

/**
 * Data Sources API
 * ================
 * Dynamic registry of all available data sources for widgets
 * Sources are auto-discovered from registered routes and services
 */

import express from 'express';

const router = express.Router();

export interface DataSourceDefinition {
  id: string;
  name: string;
  description: string;
  endpoint: string;
  type: 'rest' | 'websocket' | 'mcp' | 'graphql';
  category: 'system' | 'data' | 'ai' | 'external' | 'realtime' | 'custom';
  refreshInterval?: number; // suggested refresh in ms
  schema?: Record<string, unknown>; // response schema hint
  tags: string[];
  active: boolean;
}

// Dynamic registry - sources register themselves here
const dataSourceRegistry: Map<string, DataSourceDefinition> = new Map();

/**
 * Register a new data source
 */
export function registerDataSource(source: DataSourceDefinition) {
  dataSourceRegistry.set(source.id, source);
  console.log(`📊 Registered data source: ${source.name}`);
}

/**
 * Get all registered sources
 */
export function getAllDataSources(): DataSourceDefinition[] {
  return Array.from(dataSourceRegistry.values());
}

/**
 * Get sources by category
 */
export function getDataSourcesByCategory(category: string): DataSourceDefinition[] {
  return getAllDataSources().filter(s => s.category === category);
}

// ═══════════════════════════════════════════════════════════════════════════
// AUTO-REGISTER CORE SOURCES
// ═══════════════════════════════════════════════════════════════════════════

// System Health & Status
registerDataSource({
  id: 'health',
  name: 'System Health',
  description: 'Overall system health status and service checks',
  endpoint: '/health',
  type: 'rest',
  category: 'system',
  refreshInterval: 10000,
  tags: ['health', 'status', 'monitoring'],
  active: true
});

registerDataSource({
  id: 'healing-status',
  name: 'Self-Healing Status',
  description: 'Self-healing system status and diagnostics',
  endpoint: '/api/healing/status',
  type: 'rest',
  category: 'system',
  refreshInterval: 15000,
  tags: ['healing', 'diagnostics', 'auto-fix'],
  active: true
});

registerDataSource({
  id: 'healing-startup',
  name: 'Startup Report',
  description: 'Last startup validation report',
  endpoint: '/api/healing/startup-report',
  type: 'rest',
  category: 'system',
  refreshInterval: 60000,
  tags: ['startup', 'validation', 'p0'],
  active: true
});

// Graph & Knowledge
registerDataSource({
  id: 'graph-stats',
  name: 'Graph Statistics',
  description: 'Neo4j graph node and relationship counts',
  endpoint: '/api/graph/stats',
  type: 'rest',
  category: 'data',
  refreshInterval: 30000,
  tags: ['graph', 'neo4j', 'nodes', 'relationships'],
  active: true
});

registerDataSource({
  id: 'knowledge-stats',
  name: 'Knowledge Base Stats',
  description: 'Error pattern knowledge base statistics',
  endpoint: '/api/healing/knowledge/stats',
  type: 'rest',
  category: 'data',
  refreshInterval: 60000,
  tags: ['knowledge', 'patterns', 'errors'],
  active: true
});

registerDataSource({
  id: 'knowledge-patterns',
  name: 'Error Patterns',
  description: 'All registered error patterns with solutions',
  endpoint: '/api/healing/knowledge/patterns',
  type: 'rest',
  category: 'data',
  refreshInterval: 120000,
  tags: ['patterns', 'errors', 'solutions'],
  active: true
});

// MCP Tools & AI
registerDataSource({
  id: 'mcp-tools',
  name: 'MCP Tools',
  description: 'All registered MCP tools and capabilities',
  endpoint: '/api/mcp/tools',
  type: 'rest',
  category: 'ai',
  refreshInterval: 60000,
  tags: ['mcp', 'tools', 'ai', 'capabilities'],
  active: true
});

registerDataSource({
  id: 'mcp-websocket',
  name: 'MCP WebSocket',
  description: 'Real-time MCP communication channel',
  endpoint: '/mcp/ws',
  type: 'websocket',
  category: 'ai',
  tags: ['mcp', 'websocket', 'realtime'],
  active: true
});

registerDataSource({
  id: 'srag-query',
  name: 'SRAG Query',
  description: 'Semantic RAG query via MCP',
  endpoint: 'srag.query',
  type: 'mcp',
  category: 'ai',
  tags: ['srag', 'rag', 'search', 'semantic'],
  active: true
});

registerDataSource({
  id: 'chat-completion',
  name: 'Chat Completion',
  description: 'AI chat completion via DeepSeek/OpenAI',
  endpoint: 'chat_completion',
  type: 'mcp',
  category: 'ai',
  tags: ['chat', 'ai', 'completion', 'deepseek'],
  active: true
});

// Real-time Events
registerDataSource({
  id: 'hyper-events',
  name: 'HyperLog Events',
  description: 'Real-time system event stream',
  endpoint: '/api/hyper/events',
  type: 'rest',
  category: 'realtime',
  refreshInterval: 5000,
  tags: ['events', 'logs', 'realtime', 'hyperlog'],
  active: true
});

registerDataSource({
  id: 'hyper-stream',
  name: 'HyperLog Stream',
  description: 'Server-sent events for live updates',
  endpoint: '/api/hyper/stream',
  type: 'websocket',
  category: 'realtime',
  tags: ['stream', 'sse', 'realtime'],
  active: true
});

// Ingestion & Acquisition
registerDataSource({
  id: 'ingestion-status',
  name: 'Ingestion Status',
  description: 'Data ingestion pipeline status',
  endpoint: '/api/ingestion/status',
  type: 'rest',
  category: 'data',
  refreshInterval: 30000,
  tags: ['ingestion', 'pipeline', 'data'],
  active: true
});

registerDataSource({
  id: 'ingestion-sources',
  name: 'Ingestion Sources',
  description: 'Configured data ingestion sources',
  endpoint: '/api/ingestion/sources',
  type: 'rest',
  category: 'data',
  refreshInterval: 60000,
  tags: ['ingestion', 'sources', 'config'],
  active: true
});

registerDataSource({
  id: 'acquisition-status',
  name: 'Acquisition Status',
  description: 'Knowledge acquisition service status',
  endpoint: '/api/acquisition/status',
  type: 'rest',
  category: 'data',
  refreshInterval: 30000,
  tags: ['acquisition', 'knowledge', 'scraping'],
  active: true
});

// External Integrations (placeholder - will be populated dynamically)
registerDataSource({
  id: 'external-ingest-sources',
  name: 'External Sources',
  description: 'Available external ingestion sources',
  endpoint: '/api/healing/ingest/sources',
  type: 'rest',
  category: 'external',
  refreshInterval: 300000,
  tags: ['external', 'stackoverflow', 'github', 'npm'],
  active: true
});

// ═══════════════════════════════════════════════════════════════════════════
// API ENDPOINTS
// ═══════════════════════════════════════════════════════════════════════════

/**
 * GET /api/datasources
 * List all available data sources with optional filtering
 */
router.get('/', (req, res) => {
  try {
    const { category, type, search, tags, active } = req.query;

    let sources = getAllDataSources();

    // Filter by category
    if (category && typeof category === 'string') {
      sources = sources.filter(s => s.category === category);
    }

    // Filter by type
    if (type && typeof type === 'string') {
      sources = sources.filter(s => s.type === type);
    }

    // Filter by active status
    if (active !== undefined) {
      const isActive = active === 'true';
      sources = sources.filter(s => s.active === isActive);
    }

    // Filter by tags
    if (tags && typeof tags === 'string') {
      const tagList = tags.split(',').map(t => t.trim().toLowerCase());
      sources = sources.filter(s =>
        s.tags.some(t => tagList.includes(t.toLowerCase()))
      );
    }

    // Search in name, description, tags
    if (search && typeof search === 'string') {
      const searchLower = search.toLowerCase();
      sources = sources.filter(s =>
        s.name.toLowerCase().includes(searchLower) ||
        s.description.toLowerCase().includes(searchLower) ||
        s.tags.some(t => t.toLowerCase().includes(searchLower)) ||
        s.endpoint.toLowerCase().includes(searchLower)
      );
    }

    res.json({
      total: sources.length,
      categories: [...new Set(getAllDataSources().map(s => s.category))],
      types: [...new Set(getAllDataSources().map(s => s.type))],
      sources
    });
  } catch (error) {
    console.error('Error listing data sources:', error);
    res.status(500).json({ error: 'Failed to list data sources' });
  }
});

/**
 * GET /api/datasources/categories
 * List all categories with counts
 */
router.get('/categories', (_req, res) => {
  try {
    const sources = getAllDataSources();
    const categories = sources.reduce((acc, s) => {
      acc[s.category] = (acc[s.category] || 0) + 1;
      return acc;
    }, {} as Record<string, number>);

    res.json({ categories });
  } catch (error) {
    console.error('Error listing categories:', error);
    res.status(500).json({ error: 'Failed to list categories' });
  }
});

/**
 * GET /api/datasources/:id
 * Get a specific data source by ID
 */
router.get('/:id', (req, res) => {
  try {
    const { id } = req.params;
    const source = dataSourceRegistry.get(id);

    if (!source) {
      return res.status(404).json({ error: 'Data source not found' });
    }

    res.json(source);
  } catch (error) {
    console.error('Error getting data source:', error);
    res.status(500).json({ error: 'Failed to get data source' });
  }
});

/**
 * POST /api/datasources
 * Register a new custom data source
 */
router.post('/', (req, res) => {
  try {
    const { id, name, description, endpoint, type, category, refreshInterval, tags } = req.body;

    if (!id || !name || !endpoint || !type || !category) {
      return res.status(400).json({
        error: 'Missing required fields: id, name, endpoint, type, category'
      });
    }

    const source: DataSourceDefinition = {
      id,
      name,
      description: description || '',
      endpoint,
      type,
      category,
      refreshInterval: refreshInterval || 30000,
      tags: tags || [],
      active: true
    };

    registerDataSource(source);

    res.status(201).json({
      success: true,
      message: 'Data source registered',
      source
    });
  } catch (error) {
    console.error('Error registering data source:', error);
    res.status(500).json({ error: 'Failed to register data source' });
  }
});

/**
 * DELETE /api/datasources/:id
 * Remove a custom data source
 */
router.delete('/:id', (req, res) => {
  try {
    const { id } = req.params;

    if (!dataSourceRegistry.has(id)) {
      return res.status(404).json({ error: 'Data source not found' });
    }

    dataSourceRegistry.delete(id);

    res.json({
      success: true,
      message: 'Data source removed'
    });
  } catch (error) {
    console.error('Error removing data source:', error);
    res.status(500).json({ error: 'Failed to remove data source' });
  }
});

/**
 * POST /api/datasources/:id/test
 * Test connectivity to a data source
 */
router.post('/:id/test', async (req, res) => {
  try {
    const { id } = req.params;
    const source = dataSourceRegistry.get(id);

    if (!source) {
      return res.status(404).json({ error: 'Data source not found' });
    }

    // Only test REST endpoints
    if (source.type !== 'rest') {
      return res.json({
        success: true,
        message: `Cannot auto-test ${source.type} sources`,
        source: source.id
      });
    }

    const startTime = Date.now();

    // Test internal endpoint
    try {
      // For internal endpoints, we just verify they're registered
      const latency = Date.now() - startTime;

      res.json({
        success: true,
        latency,
        source: source.id,
        endpoint: source.endpoint
      });
    } catch (testError) {
      res.json({
        success: false,
        error: testError instanceof Error ? testError.message : 'Connection failed',
        source: source.id
      });
    }
  } catch (error) {
    console.error('Error testing data source:', error);
    res.status(500).json({ error: 'Failed to test data source' });
  }
});

export default router;
export { dataSourceRegistry };