apps/backend/src/services/NeuralCompiler.ts

// 🧠 THE BRAIN: NeuralCompiler.ts
// Ansvarlig for at æde filer og gøre dem til struktureret viden.
// Renamed from KnowledgeCompiler to avoid conflict with existing service.

import fs from 'fs/promises';
import path from 'path';
import crypto from 'crypto';
import chokidar from 'chokidar';
import { vectorService } from './VectorService';

// Conditional imports for optional services
let neo4jService: any = null;
let metricsService: any = null;

export interface CompiledDocument {
  id: string;
  name: string;
  path: string;
  extension: string;
  size: number;
  content: string;
  embedding: number[];
  lastModified: Date;
  tags: string[];
}

export class NeuralCompiler {
  private static instance: NeuralCompiler;
  private watcher: chokidar.FSWatcher | null = null;
  private isProcessing = false;
  private documentCache = new Map<string, CompiledDocument>();
  private eventQueue: { type: string; path: string }[] = [];
  private processingInterval: NodeJS.Timeout | null = null;

  private constructor() {
    console.log('📚 [KNOWLEDGE] Compiler Initialized');
    this.initServices();
  }

  private async initServices() {
    try {
      const neo4j = await import('./Neo4jService').catch(() => null);
      if (neo4j) neo4jService = neo4j.neo4jService;
      
      const metrics = await import('./MetricsService').catch(() => null);
      if (metrics) metricsService = metrics.metricsService;
    } catch {
      console.log('📚 [KNOWLEDGE] Running in standalone mode');
    }
  }

  public static getInstance(): NeuralCompiler {
    if (!NeuralCompiler.instance) {
      NeuralCompiler.instance = new NeuralCompiler();
    }
    return NeuralCompiler.instance;
  }

  public async startWatching(dirPath: string): Promise<void> {
    // Ensure directory exists
    await fs.mkdir(dirPath, { recursive: true }).catch(() => {});
    
    console.log(`👁️ [KNOWLEDGE] Watching directory: ${dirPath}`);
    
    this.watcher = chokidar.watch(dirPath, {
      ignored: /(^|[\/\\])\../, // Ignore dotfiles
      persistent: true,
      depth: 5,
      awaitWriteFinish: {
        stabilityThreshold: 1000,
        pollInterval: 100
      }
    });

    this.watcher
      .on('add', p => this.queueEvent('ADD', p))
      .on('change', p => this.queueEvent('MODIFY', p))
      .on('unlink', p => this.queueEvent('DELETE', p))
      .on('error', err => console.error('👁️ [KNOWLEDGE] Watcher error:', err));

    // Process queue every 500ms to batch events
    this.processingInterval = setInterval(() => this.processQueue(), 500);
  }

  private queueEvent(type: string, filePath: string) {
    this.eventQueue.push({ type, path: filePath });
  }

  private async processQueue() {
    if (this.isProcessing || this.eventQueue.length === 0) return;
    
    this.isProcessing = true;
    const events = [...this.eventQueue];
    this.eventQueue = [];

    for (const event of events) {
      await this.handleFileEvent(event.type as 'ADD' | 'MODIFY' | 'DELETE', event.path);
    }
    
    this.isProcessing = false;
  }

  private async handleFileEvent(
    eventType: 'ADD' | 'MODIFY' | 'DELETE', 
    filePath: string
  ): Promise<void> {
    try {
      const fileId = this.generateFileId(filePath);
      const filename = path.basename(filePath);
      const ext = path.extname(filePath).toLowerCase();

      // Skip unsupported files
      const supportedExts = ['.txt', '.md', '.json', '.ts', '.js', '.py', '.html', '.css', '.yaml', '.yml', '.xml', '.csv'];
      if (!supportedExts.includes(ext)) return;

      if (eventType === 'DELETE') {
        this.documentCache.delete(fileId);
        
        if (neo4jService) {
          await neo4jService.write(
            `MATCH (f:File {id: $id}) DETACH DELETE f`, 
            { id: fileId }
          );
        }
        console.log(`🗑️ [KNOWLEDGE] Forgot file: ${filename}`);
        return;
      }

      // 1. Read Content
      const content = await fs.readFile(filePath, 'utf-8');
      
      // 2. Generate Vector Embedding (Cognitive Dark Matter)
      const textForEmbedding = content.substring(0, 2000); // Limit for speed
      const embedding = await vectorService.embedText(textForEmbedding);

      // 3. Create compiled document
      const doc: CompiledDocument = {
        id: fileId,
        name: filename,
        path: filePath,
        extension: ext,
        size: content.length,
        content: content.substring(0, 5000), // Store first 5KB
        embedding,
        lastModified: new Date(),
        tags: this.extractTags(content, ext)
      };

      // 4. Cache locally
      this.documentCache.set(fileId, doc);

      // 5. Ingest into Neo4j if available
      if (neo4jService) {
        await neo4jService.write(`
          MERGE (f:File {id: $id})
          SET f.name = $name,
              f.path = $path,
              f.extension = $ext,
              f.size = $size,
              f.lastModified = datetime(),
              f.contentPreview = $preview
          
          // Link to Directory
          MERGE (d:Directory {path: $dirPath})
          MERGE (f)-[:LOCATED_IN]->(d)
          
          // Auto-Tagging based on extension
          MERGE (t:Tag {name: $ext})
          MERGE (f)-[:TAGGED]->(t)
        `, {
          id: fileId,
          name: filename,
          path: filePath,
          dirPath: path.dirname(filePath),
          ext: ext,
          size: content.length,
          preview: content.substring(0, 500)
        });
      }

      console.log(`✨ [KNOWLEDGE] Assimilated: ${filename} (${eventType})`);
      
      if (metricsService) {
        metricsService.incrementCounter('knowledge_files_ingested');
      }
    } catch (error) {
      console.error(`📚 [KNOWLEDGE] Error processing ${filePath}:`, error);
    }
  }

  private extractTags(content: string, ext: string): string[] {
    const tags: string[] = [ext];
    
    // Extract hashtags
    const hashtags = content.match(/#\w+/g) || [];
    tags.push(...hashtags.map(t => t.toLowerCase()));
    
    // Detect language/framework
    if (content.includes('import React')) tags.push('react');
    if (content.includes('from fastapi')) tags.push('fastapi');
    if (content.includes('async function')) tags.push('async');
    if (content.includes('class ')) tags.push('oop');
    
    return [...new Set(tags)];
  }

  private generateFileId(filePath: string): string {
    return crypto.createHash('md5').update(filePath).digest('hex');
  }

  // Public API for querying
  public async searchSimilar(query: string, topK: number = 5): Promise<CompiledDocument[]> {
    const items = Array.from(this.documentCache.values()).map(doc => ({
      id: doc.id,
      embedding: doc.embedding
    }));

    const results = await vectorService.findSimilar(query, items, topK);
    
    return results
      .map(r => this.documentCache.get(r.id))
      .filter((d): d is CompiledDocument => d !== undefined);
  }

  public getDocument(id: string): CompiledDocument | undefined {
    return this.documentCache.get(id);
  }

  public getAllDocuments(): CompiledDocument[] {
    return Array.from(this.documentCache.values());
  }

  public getStats() {
    return {
      totalDocuments: this.documentCache.size,
      queueLength: this.eventQueue.length,
      isProcessing: this.isProcessing
    };
  }

  public async stop(): Promise<void> {
    if (this.watcher) {
      await this.watcher.close();
      this.watcher = null;
    }
    if (this.processingInterval) {
      clearInterval(this.processingInterval);
      this.processingInterval = null;
    }
    console.log('📚 [NEURAL] Compiler Stopped');
  }
}

export const neuralCompiler = NeuralCompiler.getInstance();