backend/server.js

import express from 'express';
import multer from 'multer';
import cors from 'cors';
import fs from 'fs-extra';
import path from 'path';
import { v4 as uuidv4 } from 'uuid';
import { fileURLToPath } from 'url';
import PDFParser from 'pdf2json';
import HuggingFaceAIService from './huggingface-ai.js';

// Initialize Hugging Face AI Service with environment variable
// Using Llama-2-13b-chat-hf model via new Inference Providers API
const HF_API_KEY = process.env.HF_TOKEN || process.env.HUGGINGFACE_API_TOKEN || process.env.HUGGING_FACE_API_KEY;
let hfAI = null;

console.log('🔑 Environment Check:');
console.log('   HF_TOKEN exists:', !!process.env.HF_TOKEN);
console.log('   HF_TOKEN length:', process.env.HF_TOKEN ? process.env.HF_TOKEN.length : 0);
console.log('   HF_TOKEN starts with:', process.env.HF_TOKEN ? process.env.HF_TOKEN.substring(0, 6) + '...' : 'N/A');

if (HF_API_KEY) {
  try {
    // Initialize with Llama-2-13b-chat-hf model
    hfAI = new HuggingFaceAIService(HF_API_KEY, 'meta-llama/Llama-2-13b-chat-hf');
    console.log('✅ Hugging Face AI Service initialized successfully');
    console.log('📡 Using Inference Providers API (router.huggingface.co)');
    console.log('🤖 Model: meta-llama/Llama-2-13b-chat-hf');
  } catch (error) {
    console.error('❌ Failed to initialize Hugging Face AI:', error.message);
    console.error('💡 Make sure you have:');
    console.error('   1. Accepted model terms at: https://huggingface.co/meta-llama/Llama-2-13b-chat-hf');
    console.error('   2. Set HF_TOKEN environment variable with your API key');
  }
} else {
  console.warn('⚠️  HF_TOKEN not set - AI features will use fallback');
  console.warn('💡 Set HF_TOKEN environment variable to enable AI-powered responses');
  console.warn('   Get your token at: https://huggingface.co/settings/tokens');
  console.warn('📝 Available env vars:', Object.keys(process.env).filter(k => k.includes('HF') || k.includes('TOKEN')).join(', '));
}

// Health check endpoint
const addHealthCheck = (app) => {
  app.get('/api/health', (req, res) => {
    const healthInfo = {
      status: 'healthy',
      ai_service: hfAI ? 'connected' : 'fallback',
      model: hfAI ? 'meta-llama/Llama-2-13b-chat-hf' : 'none',
      api_endpoint: 'Inference Providers API (router.huggingface.co)',
      timestamp: new Date().toISOString()
    };
    
    console.log('💚 Health check:', healthInfo);
    res.status(200).json(healthInfo);
  });
};

const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);

const app = express();
const PORT = process.env.PORT || 5001;

// Enhanced CORS for Hugging Face Spaces
app.use(cors({
  origin: function (origin, callback) {
    // Allow requests with no origin (like mobile apps, Postman, curl)
    if (!origin) return callback(null, true);
    
    // Allow localhost and HF Spaces
    const allowedOrigins = [
      'http://localhost:3000',
      'http://localhost:5001',
      'http://localhost:7860',
      'http://127.0.0.1:7860',
      'https://huggingface.co'
    ];
    
    // Allow any .hf.space domain
    if (origin.includes('.hf.space') || allowedOrigins.includes(origin)) {
      callback(null, true);
    } else {
      callback(null, true); // Allow all for now to debug
    }
  },
  credentials: true,
  methods: ['GET', 'POST', 'PUT', 'DELETE', 'OPTIONS'],
  allowedHeaders: ['Content-Type', 'Authorization']
}));

app.use(express.json({ limit: '50mb' }));
app.use(express.urlencoded({ extended: true, limit: '50mb' }));

// Add health check
addHealthCheck(app);

// Async initialization function
async function initializeServer() {
  try {
    // Storage configuration - use temp directory on HF Spaces
    const isCIEnvironment = process.env.CI || process.env.SPACE_ID || process.env.SPACE_AUTHOR_NAME;
    const baseDir = isCIEnvironment ? '/tmp' : __dirname;
    
    const storageDir = path.join(baseDir, 'storage');
    const uploadsDir = path.join(storageDir, 'current-rfps');
    const previousRFPsDir = path.join(storageDir, 'previous-rfps');
    const metadataPath = path.join(storageDir, 'metadata.json');

    console.log('📁 Storage configuration:');
    console.log('   CI Environment:', !!isCIEnvironment);
    console.log('   Base directory:', baseDir);
    console.log('   Storage directory:', storageDir);

    // Ensure directories exist with error handling
    try {
      await fs.ensureDir(storageDir);
      await fs.ensureDir(uploadsDir);
      await fs.ensureDir(previousRFPsDir);
      console.log('✅ Storage directories created successfully');
    } catch (dirError) {
      console.error('❌ Failed to create storage directories:', dirError.message);
      console.log('💡 Attempting to use /tmp fallback...');
      
      // Fallback to /tmp if permission denied
      const fallbackStorageDir = path.join('/tmp', 'rfp-storage');
      const fallbackUploadsDir = path.join(fallbackStorageDir, 'current-rfps');
      const fallbackPreviousRFPsDir = path.join(fallbackStorageDir, 'previous-rfps');
      const fallbackMetadataPath = path.join(fallbackStorageDir, 'metadata.json');
      
      await fs.ensureDir(fallbackStorageDir);
      await fs.ensureDir(fallbackUploadsDir);
      await fs.ensureDir(fallbackPreviousRFPsDir);
      
      console.log('✅ Using fallback storage at /tmp/rfp-storage');
      
      // Update paths to use fallback
      Object.assign(global, {
        storageDir: fallbackStorageDir,
        uploadsDir: fallbackUploadsDir,
        previousRFPsDir: fallbackPreviousRFPsDir,
        metadataPath: fallbackMetadataPath
      });
      
      return;
    }
    
    // Store paths globally for use in routes
    Object.assign(global, {
      storageDir,
      uploadsDir,
      previousRFPsDir,
      metadataPath
    });

    // Initialize metadata if it doesn't exist
    if (!await fs.pathExists(global.metadataPath)) {
      await fs.writeJson(global.metadataPath, {
        documents: [],
        previousRFPs: [],
        questions: [],
        lastUpdated: new Date().toISOString()
      });
      console.log('✅ Metadata file initialized');
    }

    // Configure multer for file uploads
    const storage = multer.diskStorage({
      destination: (req, file, cb) => {
        cb(null, global.uploadsDir);
      },
      filename: (req, file, cb) => {
        const uniqueSuffix = uuidv4();
        const ext = path.extname(file.originalname);
        const name = path.basename(file.originalname, ext);
        cb(null, `${name}_${uniqueSuffix}${ext}`);
      }
    });

    const upload = multer({ 
      storage,
      limits: {
        fileSize: 50 * 1024 * 1024 // 50MB limit
      },
      fileFilter: (req, file, cb) => {
        const allowedTypes = ['.pdf', '.docx', '.doc', '.txt'];
        const ext = path.extname(file.originalname).toLowerCase();
        if (allowedTypes.includes(ext)) {
          cb(null, true);
        } else {
          cb(new Error('Only PDF, DOCX, DOC, and TXT files are allowed'), false);
        }
      }
    });

    // Enhanced AI Generation with better error handling
    const generateAIAnswer = async (question, referenceText = "", previousAnswers = []) => {
      console.log('🤖 Generating AI answer for question:', question.substring(0, 100) + '...');
      
      if (!hfAI) {
        console.log('📝 Using fallback answer generation');
        return {
          answer: `**Professional Response Required**



This question requires detailed analysis and expertise. Based on industry best practices and our organization's capabilities:



**Key Points to Address:**

• ${question}



**Recommended Approach:**

1. Comprehensive analysis of requirements

2. Detailed technical specifications

3. Implementation timeline and methodology

4. Quality assurance measures

5. Risk mitigation strategies



**Next Steps:**

Please review and customize this response with specific details about your organization's capabilities, experience, and proposed solutions.



*This response was generated using fallback mode. Connect your Hugging Face API token for enhanced AI-powered answers.*`,
          confidence: 0.7,
          citations: referenceText ? [`Reference document pages 1-3`] : []
        };
      }

      try {
        const result = await hfAI.generateAnswer(question, referenceText, previousAnswers);
        console.log('✅ AI answer generated successfully');
        return result;
      } catch (error) {
        console.error('❌ AI generation failed:', error.message);
        return {
          answer: `**Professional Response Framework**



Question: ${question}



**Analysis Required:**

This question requires detailed consideration of the following aspects:

- Technical requirements and specifications

- Implementation methodology and timeline

- Resource allocation and team expertise

- Quality assurance and testing protocols

- Risk assessment and mitigation strategies



**Response Template:**

1. **Understanding of Requirements:** [Detailed analysis of what is being asked]

2. **Proposed Solution:** [Specific approach and methodology]

3. **Implementation Plan:** [Timeline, milestones, and deliverables]

4. **Team & Expertise:** [Relevant experience and qualifications]

5. **Quality Measures:** [Testing, validation, and success criteria]



**Note:** Please customize this framework with your organization's specific capabilities, experience, and proposed solutions.



*AI service temporarily unavailable. Please review and enhance this response manually.*`,
          confidence: 0.6,
          citations: []
        };
      }
    };

    // Upload previous RFP endpoint
    app.post('/api/upload-previous-rfp', upload.single('file'), async (req, res) => {
      try {
        if (!req.file) {
          return res.status(400).json({ error: 'No file uploaded' });
        }

        console.log('📄 Processing previous RFP:', req.file.originalname);
        console.log('📁 Category:', req.body.category || 'general');

        // Move file to previous-rfps directory
        const targetPath = path.join(global.previousRFPsDir, req.file.filename);
        await fs.move(req.file.path, targetPath, { overwrite: true });

        const metadata = await fs.readJson(global.metadataPath);
        const fileId = uuidv4();
        
        const documentInfo = {
          id: fileId,
          filename: req.file.filename,
          originalName: req.file.originalname,
          category: req.body.category || 'general',
          path: targetPath,
          size: req.file.size,
          uploadedAt: new Date().toISOString(),
          type: 'previous-rfp'
        };

        // Extract text from the document
        let extractedText = '';
        try {
          if (path.extname(req.file.originalname).toLowerCase() === '.pdf') {
            console.log('📖 Extracting text from PDF...');
            extractedText = await extractTextFromPDF(targetPath);
          } else {
            console.log('📄 Reading text file...');
            extractedText = await fs.readFile(targetPath, 'utf-8');
          }

          documentInfo.extractedText = extractedText;
          documentInfo.textLength = extractedText.length;
          console.log(`✅ Extracted ${extractedText.length} characters`);

        } catch (extractError) {
          console.error('❌ Text extraction failed:', extractError.message);
          documentInfo.extractionError = extractError.message;
        }

        // Update metadata
        if (!metadata.previousRFPs) {
          metadata.previousRFPs = [];
        }
        metadata.previousRFPs.push(documentInfo);
        metadata.lastUpdated = new Date().toISOString();
        
        await fs.writeJson(global.metadataPath, metadata, { spaces: 2 });

        console.log('✅ Previous RFP processed and metadata updated');

        res.json({
          success: true,
          document: documentInfo
        });

      } catch (error) {
        console.error('❌ Upload processing failed:', error);
        res.status(500).json({ 
          error: 'Failed to process uploaded file',
          details: error.message 
        });
      }
    });

    // Get previous RFPs endpoint
    app.get('/api/previous-rfps', async (req, res) => {
      try {
        const metadata = await fs.readJson(global.metadataPath);
        
        // Transform backend format to frontend format with validation
        const documents = (metadata.previousRFPs || []).map(doc => {
          // Handle legacy/malformed documents
          const originalName = doc.originalName || doc.filename || 'Unknown Document';
          const fileExt = path.extname(originalName).substring(1).toUpperCase() || 'FILE';
          const uploadDate = doc.uploadedAt || doc.uploadDate || new Date().toISOString();
          
          return {
            id: doc.id || uuidv4(),
            name: originalName,
            category: doc.category || 'general',
            fileType: fileExt,
            uploadDate: uploadDate,
            size: doc.size || 0,
            contentLength: doc.textLength || doc.contentLength || 0,
            processed: !!doc.extractedText
          };
        });
        
        res.json({
          success: true,
          documents: documents,
          total: documents.length
        });
      } catch (error) {
        console.error('❌ Failed to fetch previous RFPs:', error);
        res.status(500).json({ 
          success: false,
          error: 'Failed to fetch previous RFPs',
          details: error.message 
        });
      }
    });

    // Get document content endpoint
    app.get('/api/document/:id/content', async (req, res) => {
      try {
        const { id } = req.params;
        const metadata = await fs.readJson(global.metadataPath);
        
        // Find document in previousRFPs or documents
        const document = [...(metadata.previousRFPs || []), ...(metadata.documents || [])]
          .find(doc => doc.id === id);

        if (!document) {
          return res.status(404).json({ success: false, error: 'Document not found' });
        }

        if (document.extractedText) {
          // Handle legacy/malformed documents
          const originalName = document.originalName || document.filename || 'Unknown Document';
          const fileExt = path.extname(originalName).substring(1).toUpperCase() || 'FILE';
          const uploadDate = document.uploadedAt || document.uploadDate || new Date().toISOString();
          
          res.json({
            success: true,
            document: {
              id: document.id || uuidv4(),
              name: originalName,
              category: document.category || 'general',
              fileType: fileExt,
              uploadDate: uploadDate,
              size: document.size || 0,
              contentLength: document.textLength || document.contentLength || 0
            },
            content: document.extractedText
          });
        } else {
          res.status(404).json({ success: false, error: 'No extracted text available' });
        }
      } catch (error) {
        console.error('❌ Failed to fetch document content:', error);
        res.status(500).json({ 
          success: false,
          error: 'Failed to fetch document content',
          details: error.message 
        });
      }
    });

    // Delete document endpoint
    app.delete('/api/document/:id', async (req, res) => {
      try {
        const { id } = req.params;
        const metadata = await fs.readJson(global.metadataPath);
        
        // Find and remove document
        const previousRFPs = metadata.previousRFPs || [];
        const docIndex = previousRFPs.findIndex(doc => doc.id === id);
        
        if (docIndex === -1) {
          return res.status(404).json({ error: 'Document not found' });
        }

        const document = previousRFPs[docIndex];
        
        // Delete file if it exists
        try {
          if (await fs.pathExists(document.path)) {
            await fs.remove(document.path);
          }
        } catch (fileError) {
          console.warn('⚠️  Could not delete file:', fileError.message);
        }

        // Remove from metadata
        previousRFPs.splice(docIndex, 1);
        metadata.previousRFPs = previousRFPs;
        metadata.lastUpdated = new Date().toISOString();
        
        await fs.writeJson(global.metadataPath, metadata, { spaces: 2 });

        console.log('✅ Document deleted:', document.originalName);

        res.json({
          success: true,
          message: 'Document deleted successfully'
        });

      } catch (error) {
        console.error('❌ Failed to delete document:', error);
        res.status(500).json({ 
          error: 'Failed to delete document',
          details: error.message 
        });
      }
    });

    // Upload current RFP endpoint
    app.post('/api/upload-current-rfp', upload.single('file'), async (req, res) => {
      try {
        if (!req.file) {
          return res.status(400).json({ error: 'No file uploaded' });
        }

        console.log('📄 Processing uploaded file:', req.file.originalname);

        const metadata = await fs.readJson(global.metadataPath);
        const fileId = uuidv4();
        
        const documentInfo = {
          id: fileId,
          filename: req.file.filename,
          originalName: req.file.originalname,
          path: req.file.path,
          size: req.file.size,
          uploadedAt: new Date().toISOString(),
          type: 'current-rfp'
        };

        // Extract text from the document
        let extractedText = '';
        let questions = [];

        try {
          if (path.extname(req.file.originalname).toLowerCase() === '.pdf') {
            console.log('📖 Extracting text from PDF...');
            extractedText = await extractTextFromPDF(req.file.path);
          } else {
            console.log('📄 Reading text file...');
            extractedText = await fs.readFile(req.file.path, 'utf-8');
          }

          // Extract questions from the text
          questions = extractQuestionsFromText(extractedText);
          console.log(`✅ Extracted ${questions.length} questions from document`);

          documentInfo.extractedText = extractedText;
          documentInfo.questionsFound = questions.length;

        } catch (extractError) {
          console.error('❌ Text extraction failed:', extractError.message);
          documentInfo.extractionError = extractError.message;
        }

        // Update metadata
        metadata.documents.push(documentInfo);
        metadata.questions = questions;
        metadata.lastUpdated = new Date().toISOString();
        
        await fs.writeJson(global.metadataPath, metadata, { spaces: 2 });

        console.log('✅ File processed and metadata updated');

        res.json({
          success: true,
          document: documentInfo,
          questions: questions,
          totalQuestions: questions.length
        });

      } catch (error) {
        console.error('❌ Upload processing failed:', error);
        res.status(500).json({ 
          error: 'Failed to process uploaded file',
          details: error.message 
        });
      }
    });

    // Get questions endpoint
    app.get('/api/questions', async (req, res) => {
      try {
        const metadata = await fs.readJson(global.metadataPath);
        res.json({
          questions: metadata.questions || [],
          total: metadata.questions?.length || 0
        });
      } catch (error) {
        console.error('❌ Failed to fetch questions:', error);
        res.status(500).json({ error: 'Failed to fetch questions' });
      }
    });

    // Add questions endpoint
    app.post('/api/add-questions', async (req, res) => {
      try {
        const { questions: newQuestions } = req.body;

        if (!Array.isArray(newQuestions)) {
          return res.status(400).json({ error: 'Questions must be an array' });
        }

        const metadata = await fs.readJson(global.metadataPath);
        
        // Add new questions with unique IDs
        const questionsWithIds = newQuestions.map(q => ({
          id: uuidv4(),
          text: q.text || q,
          addedAt: new Date().toISOString(),
          source: 'manual'
        }));

        metadata.questions = [...(metadata.questions || []), ...questionsWithIds];
        metadata.lastUpdated = new Date().toISOString();

        await fs.writeJson(global.metadataPath, metadata, { spaces: 2 });

        console.log(`✅ Added ${newQuestions.length} new questions`);

        res.json({
          success: true,
          added: questionsWithIds.length,
          total: metadata.questions.length
        });

      } catch (error) {
        console.error('❌ Failed to add questions:', error);
        res.status(500).json({ error: 'Failed to add questions' });
      }
    });

    // Format answer endpoint - Clean up and extract relevant content
    app.post('/api/format-answer', async (req, res) => {
      try {
        const { question, rawAnswer, context } = req.body;

        if (!question || !rawAnswer) {
          return res.status(400).json({ error: 'Question and rawAnswer are required' });
        }

        console.log('✂️ Formatting answer for question:', question.substring(0, 100) + '...');
        console.log('📏 Raw answer length:', rawAnswer.length);

        // Extract relevant content based on question keywords
        const formattedAnswer = extractRelevantContent(question, rawAnswer);

        res.json({
          success: true,
          formattedAnswer: formattedAnswer,
          originalAnswer: rawAnswer
        });

      } catch (error) {
        console.error('❌ Answer formatting failed:', error);
        res.status(500).json({ 
          success: false,
          error: 'Failed to format answer',
          details: error.message,
          formattedAnswer: rawAnswer, // Fallback to original
          originalAnswer: rawAnswer
        });
      }
    });

    // Helper function to extract relevant content
    function extractRelevantContent(question, rawText) {
      console.log('🔍 Extracting relevant content for question:', question.substring(0, 100));
      console.log('📄 Raw text length:', rawText.length);
      
      // Remove common PDF artifacts and headers
      let cleaned = rawText
        .replace(/EXECUTIVE SUMMAR\s*Y/gi, '')
        .replace(/Company Backgr\s*ound/gi, '')
        .replace(/Page \d+/gi, '')
        .replace(/T A B [A-Z]:/gi, '')
        .replace(/\f/g, '\n')
        .replace(/\r\n/g, '\n')
        .replace(/_{10,}/g, '') // Remove long underscores
        .replace(/\s{2,}/g, ' ') // Collapse multiple spaces
        .replace(/\n{3,}/g, '\n\n');

      // Extract key concepts from question
      const questionLower = question.toLowerCase();
      const keyPhrases = [];
      
      // Add important phrases
      if (questionLower.includes('organization') || questionLower.includes('vendor')) {
        keyPhrases.push('sedna consulting', 'company', 'organization', 'business enterprise', 'founded', 'employees');
      }
      if (questionLower.includes('work done') || questionLower.includes('types of work')) {
        keyPhrases.push('services', 'expertise', 'solutions', 'application', 'development', 'staffing');
      }
      if (questionLower.includes('business focus') || questionLower.includes('specialty')) {
        keyPhrases.push('specialty', 'focus', 'niche', 'expertise', 'government sector');
      }
      if (questionLower.includes('employees') || questionLower.includes('job category')) {
        keyPhrases.push('total employees', 'staff', 'professionals', 'years', 'founded');
      }

      // Find sentences that contain these key phrases
      const sentences = cleaned.split(/[.!?]+\s+/);
      const relevantSentences = [];
      
      sentences.forEach(sentence => {
        const sentenceLower = sentence.toLowerCase();
        const matchCount = keyPhrases.filter(phrase => sentenceLower.includes(phrase)).length;
        
        if (matchCount > 0) {
          relevantSentences.push({
            text: sentence.trim(),
            score: matchCount
          });
        }
      });

      // Sort by relevance and take top sentences
      relevantSentences.sort((a, b) => b.score - a.score);
      const topSentences = relevantSentences.slice(0, 10).map(s => s.text);

      // Build answer from relevant sentences
      let result = topSentences.join('. ') + '.';

      // If result is too short, add contextual paragraphs
      if (result.length < 200) {
        const paragraphs = cleaned.split(/\n\n+/);
        const relevantParas = paragraphs
          .filter(p => keyPhrases.some(phrase => p.toLowerCase().includes(phrase)))
          .slice(0, 3);
        result = relevantParas.join('\n\n');
      }

      // Limit to reasonable length
      if (result.length > 1500) {
        result = result.substring(0, 1500);
        const lastPeriod = result.lastIndexOf('.');
        if (lastPeriod > 1000) {
          result = result.substring(0, lastPeriod + 1);
        }
        result += '\n\n[Content focused on key points]';
      }

      // Final cleanup
      result = result
        .trim()
        .replace(/\s+/g, ' ')
        .replace(/\s([.,!?;:])/g, '$1')
        .replace(/\.\s+/g, '.\n\n')
        .replace(/\n{3,}/g, '\n\n');

      console.log('✅ Extracted content length:', result.length);
      return result || 'Unable to extract relevant content. Please review the original document.';
    }

    // Generate AI answer endpoint
    app.post('/api/generate-answer', async (req, res) => {
      try {
        const { questionId, question, referenceText = "", previousAnswers = [] } = req.body;

        if (!question) {
          return res.status(400).json({ error: 'Question is required' });
        }

        console.log('🤖 Generating answer for question ID:', questionId);

        const aiResult = await generateAIAnswer(question, referenceText, previousAnswers);

        res.json({
          success: true,
          questionId,
          answer: aiResult.answer,
          confidence: aiResult.confidence,
          citations: aiResult.citations || [],
          generatedAt: new Date().toISOString(),
          aiService: hfAI ? 'huggingface' : 'fallback'
        });

      } catch (error) {
        console.error('❌ AI answer generation failed:', error);
        res.status(500).json({ 
          error: 'Failed to generate AI answer',
          details: error.message 
        });
      }
    });

    // Utility function to extract text from PDF
    async function extractTextFromPDF(filePath) {
      return new Promise((resolve, reject) => {
        const pdfParser = new PDFParser();
        
        pdfParser.on('pdfParser_dataError', (errData) => {
          reject(new Error('PDF parsing failed: ' + errData.parserError));
        });
        
        pdfParser.on('pdfParser_dataReady', (pdfData) => {
          try {
            let text = '';
            
            if (pdfData.Pages) {
              pdfData.Pages.forEach(page => {
                if (page.Texts) {
                  page.Texts.forEach(textItem => {
                    if (textItem.R) {
                      textItem.R.forEach(textRun => {
                        if (textRun.T) {
                          const decoded = decodeURIComponent(textRun.T);
                          // Add space only if it doesn't already end with space
                          // and next char won't be punctuation
                          if (decoded && decoded.trim()) {
                            text += decoded;
                            // Add space intelligently - not after every single character
                            if (!decoded.match(/[\s\-]$/) && decoded.length > 1) {
                              text += ' ';
                            }
                          }
                        }
                      });
                    }
                  });
                }
                text += '\n';
              });
            }
            
            // Post-processing: fix common PDF extraction issues
            text = text
              .replace(/\s{3,}/g, '  ') // Reduce excessive spacing
              .replace(/([a-z])\s+([a-z])/g, (match, p1, p2) => {
                // Remove space between lowercase letters (likely same word)
                return p1 + p2;
              })
              .trim();
            
            resolve(text);
          } catch (error) {
            reject(new Error('Failed to extract text from PDF: ' + error.message));
          }
        });
        
        pdfParser.loadPDF(filePath);
      });
    }

    // Utility function to extract questions from text
    function extractQuestionsFromText(text) {
      if (!text) return [];

      const questions = [];
      const lines = text.split('\n');
      
      // Common question patterns
      const questionPatterns = [
        /^\d+\.\s*(.+\?)\s*$/i,
        /^[a-z]\)\s*(.+\?)\s*$/i,
        /^\d+\)\s*(.+\?)\s*$/i,
        /^Question\s*\d*:?\s*(.+\?)\s*$/i,
        /^Q\d*:?\s*(.+\?)\s*$/i,
        /(.{10,}(?:what|how|why|when|where|who|describe|explain|provide|list|identify|specify).{10,}\?)/i
      ];

      let questionCounter = 1;

      lines.forEach((line, index) => {
        const trimmedLine = line.trim();
        
        if (trimmedLine.length < 10) return;

        for (const pattern of questionPatterns) {
          const match = trimmedLine.match(pattern);
          if (match) {
            const questionText = match[1] || match[0];
            
            if (questionText.length > 20 && questionText.length < 1000) {
              questions.push({
                id: uuidv4(),
                text: questionText.trim(),
                extractedAt: new Date().toISOString(),
                source: 'extracted',
                order: questionCounter++,
                lineNumber: index + 1
              });
            }
            break;
          }
        }
      });

      return questions;
    }

    // Start server
    app.listen(PORT, '0.0.0.0', () => {
      console.log(`🚀 SEDNA RFP Backend Server running on port ${PORT}`);
      console.log(`🌐 Health check: http://localhost:${PORT}/api/health`);
      console.log(`🤖 AI Service: ${hfAI ? 'Connected to Hugging Face' : 'Using fallback mode'}`);
    });

  } catch (error) {
    console.error('❌ Failed to initialize server:', error);
    process.exit(1);
  }
}

// Initialize the server
initializeServer();