apps/backend/src/services/NeuralChat/NeuralCortex.ts

/**

 * ╔═══════════════════════════════════════════════════════════════════════════════════════╗

 * ║                              NEURAL CORTEX                                            ║

 * ║═══════════════════════════════════════════════════════════════════════════════════════║

 * ║                                                                                       ║

 * ║   "If everything is connected, you can talk to anything and find all patterns"       ║

 * ║                                                          - CLAK, 2025                ║

 * ║                                                                                       ║

 * ║   ┌─────────────────────────────────────────────────────────────────────────────┐    ║

 * ║   │                   HYBRID NEURAL CORTEX ARCHITECTURE                         │    ║

 * ║   │                                                                             │    ║

 * ║   │    ┌──────────┐     ┌──────────┐     ┌──────────┐     ┌──────────┐        │    ║

 * ║   │    │  AGENTS  │────▶│   CHAT   │────▶│  HYBRID  │────▶│ PATTERNS │        │    ║

 * ║   │    └──────────┘     └──────────┘     │  SEARCH  │     └──────────┘        │    ║

 * ║   │         │                │           └─────┬────┘          │               │    ║

 * ║   │         ▼                ▼                 │               ▼               │    ║

 * ║   │    ┌─────────┐      ┌─────────┐      ┌─────▼─────┐    ┌─────────┐        │    ║

 * ║   │    │  NEO4J  │◀────▶│ PGVECTR │      │ SEMANTIC  │    │ INSIGHT │        │    ║

 * ║   │    │ (Graph) │ link │ (Dense) │      │ MATCHING  │    │ ENGINE  │        │    ║

 * ║   │    └─────────┘      └─────────┘      └───────────┘    └─────────┘        │    ║

 * ║   │                                                                             │    ║

 * ║   │    CAPABILITIES:                                                            │    ║

 * ║   │    • Hybrid Search (Vector Similarity + Graph Traversal)                    │    ║

 * ║   │    • Chat with agents & documents                                           │    ║

 * ║   │    • Auto-link messages to relevant nodes                                   │    ║

 * ║   │    • Discover patterns across conversations                                 │    ║

 * ║   │    • Surface insights from connected data                                   │    ║

 * ║   │                                                                             │    ║

 * ║   └─────────────────────────────────────────────────────────────────────────────┘    ║

 * ╚═══════════════════════════════════════════════════════════════════════════════════════╝

 */

import { neo4jAdapter } from '../../adapters/Neo4jAdapter.js';
import { neuralChatService } from './ChatService.js';
import { AgentId, ChatMessage } from './types.js';
import { getVectorStore } from '../../platform/vector/index.js';

export interface CortexQuery {
    type: 'chat' | 'search' | 'pattern' | 'insight' | 'history';
    query: string;
    context?: {
        channel?: string;
        agent?: AgentId;
        timeRange?: { from: string; to: string };
        nodeTypes?: string[];
    };
}

export interface CortexResult {
    type: 'message' | 'node' | 'pattern' | 'insight' | 'connection';
    data: any;
    relevance: number;
    source: string;
    connections?: { type: string; target: string }[];
}

export interface DiscoveredPattern {
    id: string;
    name: string;
    description: string;
    occurrences: number;
    confidence: number;
    relatedNodes: string[];
    discoveredAt: string;
    examples: string[];
}

class NeuralCortex {
    private static instance: NeuralCortex;

    public static getInstance(): NeuralCortex {
        if (!NeuralCortex.instance) {
            NeuralCortex.instance = new NeuralCortex();
        }
        return NeuralCortex.instance;
    }

    /**

     * Process a message and connect it to the knowledge graph AND vector store

     */
    async processMessage(message: ChatMessage): Promise<{
        entities: string[];
        concepts: string[];
        linkedNodes: string[];
        vectorStored: boolean;
    }> {
        const entities = this.extractEntities(message.body);
        const concepts = this.extractConcepts(message.body);

        // 1. Create message node in GRAPH (Neo4j)
        await neo4jAdapter.runQuery(`

            CREATE (m:Message {

                id: $id,

                from_agent: $from,

                channel: $channel,

                body: $body,

                timestamp: $timestamp,

                type: $type

            })

            WITH m

            

            // Link to sender agent

            MERGE (a:Agent {name: $from})

            MERGE (a)-[:SENT]->(m)

            

            // Link to channel

            MERGE (c:Channel {name: $channel})

            MERGE (m)-[:IN_CHANNEL]->(c)

        `, {
            id: message.id,
            from: message.from,
            channel: message.channel,
            body: message.body,
            timestamp: message.timestamp,
            type: message.type || 'chat'
        });

        // 2. Link to mentioned entities in GRAPH
        const linkedNodes: string[] = [];
        for (const entity of entities) {
            const linked = await this.linkToEntity(message.id, entity);
            if (linked) linkedNodes.push(linked);
        }

        // 3. Link to concepts in GRAPH
        for (const concept of concepts) {
            await this.linkToConcept(message.id, concept);
        }

        // 4. Store embedding in VECTOR STORE (pgvector)
        // This enables "fuzzy" semantic search later
        let vectorStored = false;
        try {
            const vectorStore = await getVectorStore();
            await vectorStore.upsert({
                id: message.id,
                content: message.body,
                metadata: {
                    type: 'message',
                    channel: message.channel,
                    from: message.from,
                    timestamp: message.timestamp,
                    entities: entities,
                    concepts: concepts
                },
                namespace: 'neural_chat'
            });
            vectorStored = true;
        } catch (error) {
            console.warn('Failed to store vector for message:', error);
        }

        return { entities, concepts, linkedNodes, vectorStored };
    }

    /**

     * Extract entities from message (agents, files, components, etc.)

     */
    private extractEntities(text: string): string[] {
        const entities: string[] = [];

        // @mentions
        const mentions = text.match(/@(\w+)/g);
        if (mentions) entities.push(...mentions.map(m => m.slice(1)));

        // File paths
        const files = text.match(/[\w-]+\.(ts|js|tsx|jsx|json|md|py|yaml|yml|sql)/gi);
        if (files) entities.push(...files);

        // Component/Class names (PascalCase)
        const components = text.match(/\b[A-Z][a-zA-Z]+(?:Widget|Service|Controller|Adapter|Component|Provider|Handler)\b/g);
        if (components) entities.push(...components);

        // URLs
        const urls = text.match(/https?:\/\/[^\s]+/g);
        if (urls) entities.push(...urls);

        return [...new Set(entities)];
    }

    /**

     * Extract concepts from message (topics, actions, technologies)

     */
    private extractConcepts(text: string): string[] {
        const concepts: string[] = [];
        const textLower = text.toLowerCase();

        // Technologies
        const techs = ['neo4j', 'react', 'typescript', 'docker', 'kubernetes', 'api', 'websocket', 'mcp', 'graphql', 'rest', 'postgresql', 'redis', 'vector', 'pgvector', 'rag'];
        techs.forEach(t => { if (textLower.includes(t)) concepts.push(t); });

        // Actions
        const actions = ['deploy', 'review', 'test', 'refactor', 'implement', 'fix', 'create', 'delete', 'update', 'analyze', 'research', 'architect'];
        actions.forEach(a => { if (textLower.includes(a)) concepts.push(a); });

        // Domains
        const domains = ['security', 'performance', 'architecture', 'authentication', 'authorization', 'database', 'frontend', 'backend', 'infrastructure', 'ai', 'agents'];
        domains.forEach(d => { if (textLower.includes(d)) concepts.push(d); });

        return [...new Set(concepts)];
    }

    /**

     * Link message to existing entity in graph

     */
    private async linkToEntity(messageId: string, entity: string): Promise<string | null> {
        try {
            const result = await neo4jAdapter.runQuery(`

                MATCH (m:Message {id: $messageId})

                MATCH (e) WHERE e.name = $entity OR e.path CONTAINS $entity OR e.id = $entity

                MERGE (m)-[:MENTIONS]->(e)

                RETURN e.name as linked

            `, { messageId, entity });

            return result[0]?.linked || null;
        } catch {
            return null;
        }
    }

    /**

     * Link message to concept (create if not exists)

     */
    private async linkToConcept(messageId: string, concept: string): Promise<void> {
        try {
            await neo4jAdapter.runQuery(`

                MATCH (m:Message {id: $messageId})

                MERGE (c:Concept {name: $concept})

                ON CREATE SET c.created_at = datetime()

                MERGE (m)-[:RELATES_TO]->(c)

                WITH c

                SET c.mention_count = coalesce(c.mention_count, 0) + 1

            `, { messageId, concept });
        } catch (error) {
            console.warn('Failed to link concept:', error);
        }
    }

    /**

     * Query the cortex in natural language using Hybrid Strategy

     */
    async query(input: CortexQuery): Promise<CortexResult[]> {
        const results: CortexResult[] = [];

        switch (input.type) {
            case 'search':
                return await this.hybridSearch(input.query, input.context);

            case 'pattern':
                return await this.findPatterns(input.query, input.context);

            case 'insight':
                return await this.generateInsights(input.query, input.context);

            case 'history':
                return await this.getDecisionHistory(input.query, input.context);

            case 'chat':
            default:
                // 1. Search Chat History (Vector + Keyword)
                const chatResults = await this.searchMessages(input.query);
                // 2. Search Knowledge Graph (Keyword/Hybrid)
                const graphResults = await this.hybridSearch(input.query, input.context);

                return [...chatResults, ...graphResults].sort((a, b) => b.relevance - a.relevance);
        }
    }

    /**

     * Search messages using Vector Similarity (Semantic) + Graph (Keyword)

     */
    private async searchMessages(query: string): Promise<CortexResult[]> {
        try {
            const results: CortexResult[] = [];

            // A. Semantic Search (Vector)
            try {
                const vectorStore = await getVectorStore();
                const vectorResults = await vectorStore.search({
                    text: query,
                    limit: 10,
                    namespace: 'neural_chat'
                });

                results.push(...vectorResults.map(r => ({
                    type: 'message' as const,
                    data: {
                        id: r.id,
                        body: r.content,
                        metadata: r.metadata,
                        from: r.metadata?.from,
                        timestamp: r.metadata?.timestamp
                    },
                    relevance: r.similarity,
                    source: 'vector_memory'
                })));
            } catch (err) {
                console.warn('Vector search failed, falling back to graph only', err);
            }

            // B. Keyword Search (Graph) - if vector search didn't yield enough
            if (results.length < 5) {
                const graphResults = await neo4jAdapter.runQuery(`

                    MATCH (m:Message)

                    WHERE toLower(m.body) CONTAINS toLower($query)

                    OPTIONAL MATCH (a:Agent)-[:SENT]->(m)

                    RETURN m, a.name as agent

                    ORDER BY m.timestamp DESC

                    LIMIT 10

                `, { query });

                const existingIds = new Set(results.map(r => r.data.id));

                for (const r of graphResults) {
                    if (!existingIds.has(r.m.properties.id)) {
                        results.push({
                            type: 'message' as const,
                            data: {
                                id: r.m.properties.id,
                                body: r.m.properties.body,
                                from: r.agent,
                                timestamp: r.m.properties.timestamp
                            },
                            relevance: this.calculateRelevance(query, r.m.properties.body),
                            source: 'graph_memory'
                        });
                    }
                }
            }

            return results;
        } catch {
            return [];
        }
    }

    /**

     * Hybrid Search: Vector -> Graph Entry -> Traversal

     */
    private async hybridSearch(query: string, context?: CortexQuery['context']): Promise<CortexResult[]> {
        try {
            const nodeTypes = context?.nodeTypes?.join('|') || 'File|Component|Service|Document|Concept|Agent';
            const results: CortexResult[] = [];

            // 1. Vector Search (Find conceptually related nodes)
            // Assuming we have documents/nodes in 'knowledge' namespace
            try {
                const vectorStore = await getVectorStore();
                const vectorResults = await vectorStore.search({
                    text: query,
                    limit: 10,
                    namespace: 'knowledge' // Search acquired knowledge too
                });

                results.push(...vectorResults.map(r => ({
                    type: 'node' as const,
                    data: {
                        name: r.metadata?.title || r.id,
                        description: r.content.substring(0, 200) + '...',
                        labels: [r.metadata?.type || 'Unknown']
                    },
                    relevance: r.similarity,
                    source: 'semantic_search'
                })));

            } catch (e) { /* ignore vector error */ }

            // 2. Graph Search (Exact/Fuzzy string match)
            const graphResults = await neo4jAdapter.runQuery(`

                MATCH (n)

                WHERE any(label IN labels(n) WHERE label IN split($nodeTypes, '|'))

                AND (

                    toLower(n.name) CONTAINS toLower($query) OR

                    toLower(coalesce(n.description, '')) CONTAINS toLower($query) OR

                    toLower(coalesce(n.path, '')) CONTAINS toLower($query)

                )

                OPTIONAL MATCH (n)-[r]-(connected)

                RETURN n, labels(n) as types, 

                       collect(DISTINCT {type: type(r), target: coalesce(connected.name, connected.id)}) as connections

                LIMIT 20

            `, { query, nodeTypes });

            // Merge results (simple dedup by name)
            const existingNames = new Set(results.map(r => r.data.name));

            for (const r of graphResults) {
                const name = r.n.properties.name || r.n.properties.id;
                if (!existingNames.has(name)) {
                    results.push({
                        type: 'node' as const,
                        data: {
                            name: name,
                            labels: r.types,
                            properties: r.n.properties
                        },
                        relevance: this.calculateRelevance(query, name),
                        source: 'knowledge_graph',
                        connections: r.connections.filter((c: any) => c.target)
                    });
                }
            }

            return results;
        } catch {
            return [];
        }
    }

    /**

     * Find patterns across conversations and code

     */
    private async findPatterns(query: string, context?: CortexQuery['context']): Promise<CortexResult[]> {
        try {
            // Find frequently co-occurring concepts
            const conceptPatterns = await neo4jAdapter.runQuery(`

                MATCH (m:Message)-[:RELATES_TO]->(c1:Concept)

                MATCH (m)-[:RELATES_TO]->(c2:Concept)

                WHERE c1 <> c2 AND id(c1) < id(c2)

                WITH c1, c2, count(m) as cooccurrences

                WHERE cooccurrences >= 2

                RETURN c1.name as concept1, c2.name as concept2, cooccurrences

                ORDER BY cooccurrences DESC

                LIMIT 10

            `);

            // Find agent collaboration patterns
            const collabPatterns = await neo4jAdapter.runQuery(`

                MATCH (a1:Agent)-[:SENT]->(m1:Message)-[:IN_CHANNEL]->(ch:Channel)<-[:IN_CHANNEL]-(m2:Message)<-[:SENT]-(a2:Agent)

                WHERE a1 <> a2 AND m1.timestamp < m2.timestamp

                AND duration.between(datetime(m1.timestamp), datetime(m2.timestamp)).minutes < 30

                WITH a1, a2, count(*) as interactions

                RETURN a1.name as agent1, a2.name as agent2, interactions

                ORDER BY interactions DESC

            `);

            // Find decision patterns (messages with action words followed by changes)
            const decisionPatterns = await neo4jAdapter.runQuery(`

                MATCH (m:Message)

                WHERE any(word IN ['besluttet', 'approved', 'godkendt', 'implement', 'deploy', 'fix'] 

                         WHERE toLower(m.body) CONTAINS word)

                OPTIONAL MATCH (m)-[:MENTIONS]->(e)

                RETURN m.body as decision, m.from_agent as by, m.timestamp as when, 

                       collect(e.name) as affected

                ORDER BY m.timestamp DESC

                LIMIT 10

            `);

            const patterns: CortexResult[] = [];

            if (conceptPatterns.length > 0) {
                patterns.push({
                    type: 'pattern',
                    data: {
                        name: 'Concept Relationships',
                        description: 'Frequently discussed together',
                        items: conceptPatterns.map((p: any) => `${p.concept1} ↔ ${p.concept2} (${p.cooccurrences}x)`)
                    },
                    relevance: 0.9,
                    source: 'pattern_analysis'
                });
            }

            if (collabPatterns.length > 0) {
                patterns.push({
                    type: 'pattern',
                    data: {
                        name: 'Agent Collaboration',
                        description: 'Who works together most',
                        items: collabPatterns.map((p: any) => `${p.agent1} ↔ ${p.agent2} (${p.interactions} interactions)`)
                    },
                    relevance: 0.85,
                    source: 'pattern_analysis'
                });
            }

            if (decisionPatterns.length > 0) {
                patterns.push({
                    type: 'pattern',
                    data: {
                        name: 'Recent Decisions',
                        description: 'Actions taken by the team',
                        items: decisionPatterns.map((p: any) => ({
                            decision: p.decision.substring(0, 100) + '...',
                            by: p.by,
                            when: p.when,
                            affected: p.affected
                        }))
                    },
                    relevance: 0.95,
                    source: 'pattern_analysis'
                });
            }

            return patterns;
        } catch (error) {
            console.warn('Pattern finding failed:', error);
            return [];
        }
    }

    /**

     * Generate insights from the knowledge graph

     */
    private async generateInsights(query: string, context?: CortexQuery['context']): Promise<CortexResult[]> {
        try {
            const insights: CortexResult[] = [];

            // Most active areas
            const activeAreas = await neo4jAdapter.runQuery(`

                MATCH (c:Concept)<-[:RELATES_TO]-(m:Message)

                WITH c.name as concept, count(m) as activity

                ORDER BY activity DESC

                LIMIT 5

                RETURN concept, activity

            `);

            // Knowledge gaps (mentioned but not documented)
            const gaps = await neo4jAdapter.runQuery(`

                MATCH (m:Message)-[:MENTIONS]->(name)

                WHERE NOT (name)-[:DOCUMENTED_IN]->(:Document)

                AND NOT name:Agent

                WITH name.name as entity, count(m) as mentions

                WHERE mentions >= 2

                RETURN entity, mentions

                ORDER BY mentions DESC

                LIMIT 5

            `);

            // Cross-cutting concerns (concepts that touch many areas)
            const crossCutting = await neo4jAdapter.runQuery(`

                MATCH (c:Concept)<-[:RELATES_TO]-(m:Message)-[:IN_CHANNEL]->(ch:Channel)

                WITH c.name as concept, count(DISTINCT ch) as channels

                WHERE channels >= 2

                RETURN concept, channels

                ORDER BY channels DESC

                LIMIT 5

            `);

            if (activeAreas.length > 0) {
                insights.push({
                    type: 'insight',
                    data: {
                        title: '🔥 Hottest Topics',
                        description: 'Most discussed areas right now',
                        items: activeAreas.map((a: any) => `${a.concept}: ${a.activity} mentions`)
                    },
                    relevance: 0.9,
                    source: 'insight_engine'
                });
            }

            if (crossCutting.length > 0) {
                insights.push({
                    type: 'insight',
                    data: {
                        title: '🔗 Cross-Cutting Concerns',
                        description: 'Topics that span multiple channels',
                        items: crossCutting.map((c: any) => `${c.concept}: ${c.channels} channels`)
                    },
                    relevance: 0.85,
                    source: 'insight_engine'
                });
            }

            return insights;
        } catch {
            return [];
        }
    }

    /**

     * Calculate relevance score (0-1) for a result

     */
    private calculateRelevance(query: string, text: string): number {
        if (!text) return 0;
        const queryTerms = query.toLowerCase().split(' ');
        const textLower = text.toLowerCase();

        let matches = 0;
        for (const term of queryTerms) {
            if (textLower.includes(term)) matches++;
        }

        return matches / queryTerms.length;
    }

    /**

     * Retrieve decision history

     */
    private async getDecisionHistory(query: string, context?: CortexQuery['context']): Promise<CortexResult[]> {
        try {
            const history = await neo4jAdapter.runQuery(`

                MATCH (m:Message)

                WHERE any(word IN ['decision', 'approved', 'rejected', 'selected', 'chose'] 

                         WHERE toLower(m.body) CONTAINS word)

                AND toLower(m.body) CONTAINS toLower($query)

                RETURN m, m.from_agent as agent

                ORDER BY m.timestamp DESC

                LIMIT 20

             `, { query });

            return history.map((h: any) => ({
                type: 'message',
                data: {
                    id: h.m.properties.id,
                    body: h.m.properties.body,
                    agent: h.agent,
                    timestamp: h.m.properties.timestamp
                },
                relevance: 1,
                source: 'history'
            }));
        } catch {
            return [];
        }
    }
}

// Singleton export
export const neuralCortex = new NeuralCortex();