src/utils/documentParser.ts

// Document parser utility for PDF and Word documents

export interface ParsedDocument {
  content: string
  metadata: {
    fileName: string
    fileSize: number
    pageCount?: number
    extractedAt: Date
  }
  citations: Array<{
    text: string
    pageNumber: number
    context: string
  }>
}

export async function parseDocument(file: File): Promise<ParsedDocument> {
  const fileType = file.type
  
  if (fileType === 'application/pdf') {
    return await parsePDF(file)
  } else if (fileType.includes('word') || file.name.endsWith('.docx') || file.name.endsWith('.doc')) {
    return await parseWord(file)
  } else {
    throw new Error(`Unsupported file type: ${fileType}`)
  }
}

async function parsePDF(file: File): Promise<ParsedDocument> {
  try {
    // For now, we'll use a simple text extraction
    // In a real implementation, you'd use pdf.js or similar
    const arrayBuffer = await file.arrayBuffer()
    
    // Simulated PDF parsing - replace with actual PDF.js implementation
    const content = await simulatedPDFExtraction(arrayBuffer)
    
    return {
      content,
      metadata: {
        fileName: file.name,
        fileSize: file.size,
        extractedAt: new Date()
      },
      citations: extractCitations(content)
    }
  } catch (error) {
    console.error('Error parsing PDF:', error)
    throw new Error('Failed to parse PDF document')
  }
}

async function parseWord(file: File): Promise<ParsedDocument> {
  try {
    // For now, we'll use a simple text extraction
    // In a real implementation, you'd use mammoth.js or similar
    const arrayBuffer = await file.arrayBuffer()
    
    // Simulated Word parsing - replace with actual mammoth.js implementation
    const content = await simulatedWordExtraction(arrayBuffer)
    
    return {
      content,
      metadata: {
        fileName: file.name,
        fileSize: file.size,
        extractedAt: new Date()
      },
      citations: extractCitations(content)
    }
  } catch (error) {
    console.error('Error parsing Word document:', error)
    throw new Error('Failed to parse Word document')
  }
}

// Simulated extraction functions - replace with real implementations
async function simulatedPDFExtraction(arrayBuffer: ArrayBuffer): Promise<string> {
  // This is a placeholder - in real implementation, use pdf.js
  await new Promise(resolve => setTimeout(resolve, 1000)) // Simulate processing time
  
  return `Sample extracted content from PDF document.

  

Question 1: What is your company's experience with similar projects?

Answer: Our company has extensive experience with similar projects spanning over 15 years. We have successfully completed more than 200 projects of similar scope and complexity.



Question 2: What is your proposed timeline?

Answer: We propose a 6-month timeline for project completion, with key milestones at 2-month intervals.



Question 3: What is your team composition?

Answer: Our team consists of 5 senior consultants, 3 project managers, and 2 technical specialists, all with relevant industry experience.`
}

async function simulatedWordExtraction(arrayBuffer: ArrayBuffer): Promise<string> {
  // This is a placeholder - in real implementation, use mammoth.js
  await new Promise(resolve => setTimeout(resolve, 800)) // Simulate processing time
  
  return `Sample extracted content from Word document.

  

1. Company Overview

Our consulting firm specializes in digital transformation and has been operating for over 10 years.



2. Technical Capabilities

We have expertise in cloud migration, data analytics, and process optimization.



3. Previous Client Success Stories

- Client A: 40% efficiency improvement

- Client B: $2M cost savings

- Client C: 6-month accelerated delivery`
}

function extractCitations(content: string): Array<{ text: string; pageNumber: number; context: string }> {
  const citations: Array<{ text: string; pageNumber: number; context: string }> = []
  const lines = content.split('\n')
  
  lines.forEach((line, index) => {
    if (line.toLowerCase().includes('answer:') || line.toLowerCase().includes('response:')) {
      citations.push({
        text: line.trim(),
        pageNumber: Math.floor(index / 10) + 1, // Rough page estimation
        context: lines.slice(Math.max(0, index - 1), index + 2).join('\n')
      })
    }
  })
  
  return citations
}

export function searchPreviousAnswers(query: string, documents: ParsedDocument[]): Array<{
  answer: string
  source: string
  pageNumber: number
  relevanceScore: number
}> {
  const results: Array<{
    answer: string
    source: string
    pageNumber: number
    relevanceScore: number
  }> = []
  
  documents.forEach(doc => {
    doc.citations.forEach(citation => {
      const relevanceScore = calculateEnhancedRelevance(query, citation.text, citation.context)
      if (relevanceScore > 0.2) { // Lower threshold for better recall
        results.push({
          answer: citation.text,
          source: doc.metadata.fileName,
          pageNumber: citation.pageNumber,
          relevanceScore
        })
      }
    })
  })
  
  return results.sort((a, b) => b.relevanceScore - a.relevanceScore)
}

function calculateRelevance(query: string, text: string): number {
  const queryWords = query.toLowerCase().split(/\s+/)
  const textWords = text.toLowerCase().split(/\s+/)
  
  let matches = 0
  queryWords.forEach(queryWord => {
    if (textWords.some(textWord => textWord.includes(queryWord) || queryWord.includes(textWord))) {
      matches++
    }
  })
  
  return matches / queryWords.length
}

/**

 * Enhanced relevance calculation with multiple scoring factors

 */
function calculateEnhancedRelevance(query: string, text: string, context?: string): number {
  const queryLower = query.toLowerCase();
  const textLower = text.toLowerCase();
  const contextLower = context?.toLowerCase() || '';
  
  // 1. Exact phrase matching (highest weight)
  let exactPhraseScore = 0;
  const queryPhrases = extractPhrases(queryLower);
  queryPhrases.forEach(phrase => {
    if (textLower.includes(phrase)) {
      exactPhraseScore += 0.3;
    } else if (contextLower.includes(phrase)) {
      exactPhraseScore += 0.15;
    }
  });
  
  // 2. Keyword matching with partial match support
  const queryWords = extractMeaningfulWords(queryLower);
  const textWords = extractMeaningfulWords(textLower);
  const contextWords = extractMeaningfulWords(contextLower);
  
  let keywordScore = 0;
  queryWords.forEach(queryWord => {
    // Check for exact matches
    if (textWords.includes(queryWord)) {
      keywordScore += 0.15;
    } else if (contextWords.includes(queryWord)) {
      keywordScore += 0.08;
    } else {
      // Check for partial matches (word contains or is contained by)
      const partialMatch = textWords.some(textWord => 
        textWord.includes(queryWord) || queryWord.includes(textWord)
      );
      if (partialMatch) {
        keywordScore += 0.10;
      }
    }
  });
  
  // Normalize keyword score
  keywordScore = Math.min(keywordScore / queryWords.length, 1.0);
  
  // 3. Semantic similarity (question patterns)
  let semanticScore = 0;
  if (containsQuestionPattern(queryLower) && containsAnswerPattern(textLower)) {
    semanticScore += 0.2;
  }
  
  // 4. Length and quality bonus
  let qualityScore = 0;
  if (text.length > 100 && text.length < 2000) {
    qualityScore += 0.1; // Good length for an answer
  }
  if (text.split(/[.!?]/).length > 2) {
    qualityScore += 0.05; // Multiple sentences
  }
  
  // 5. Synonym and related term matching
  let synonymScore = calculateSynonymMatch(queryLower, textLower);
  
  // Combine scores with weights
  const finalScore = (
    exactPhraseScore * 0.35 +
    keywordScore * 0.30 +
    semanticScore * 0.15 +
    qualityScore * 0.10 +
    synonymScore * 0.10
  );
  
  return Math.min(finalScore, 1.0);
}

/**

 * Extract meaningful phrases (2-3 word combinations)

 */
function extractPhrases(text: string): string[] {
  const words = text.split(/\s+/).filter(w => w.length > 2);
  const phrases: string[] = [];
  
  // Extract 2-word phrases
  for (let i = 0; i < words.length - 1; i++) {
    phrases.push(`${words[i]} ${words[i + 1]}`);
  }
  
  // Extract 3-word phrases for longer queries
  if (words.length > 5) {
    for (let i = 0; i < words.length - 2; i++) {
      phrases.push(`${words[i]} ${words[i + 1]} ${words[i + 2]}`);
    }
  }
  
  return phrases;
}

/**

 * Extract meaningful words (filter stop words)

 */
function extractMeaningfulWords(text: string): string[] {
  const stopWords = new Set([
    'the', 'a', 'an', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for',
    'of', 'with', 'by', 'from', 'as', 'is', 'was', 'are', 'were', 'be',
    'been', 'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would',
    'could', 'should', 'may', 'might', 'can', 'this', 'that', 'these',
    'those', 'i', 'you', 'he', 'she', 'it', 'we', 'they', 'what', 'which',
    'who', 'when', 'where', 'why', 'how'
  ]);
  
  return text
    .toLowerCase()
    .replace(/[^\w\s]/g, ' ')
    .split(/\s+/)
    .filter(word => word.length > 2 && !stopWords.has(word));
}

/**

 * Check if text contains question patterns

 */
function containsQuestionPattern(text: string): boolean {
  const questionWords = ['what', 'how', 'why', 'when', 'where', 'who', 'describe', 'explain', 'provide', 'list'];
  return questionWords.some(word => text.includes(word)) || text.includes('?');
}

/**

 * Check if text contains answer patterns

 */
function containsAnswerPattern(text: string): boolean {
  const answerPatterns = ['answer:', 'response:', 'we provide', 'we offer', 'our company', 'our approach'];
  return answerPatterns.some(pattern => text.includes(pattern));
}

/**

 * Calculate synonym and related term matching

 */
function calculateSynonymMatch(query: string, text: string): number {
  const synonymMap = new Map<string, string[]>([
    ['experience', ['background', 'expertise', 'history', 'track record', 'portfolio', 'projects']],
    ['approach', ['methodology', 'process', 'strategy', 'method', 'framework', 'technique']],
    ['team', ['staff', 'personnel', 'resources', 'employees', 'workforce', 'people']],
    ['cost', ['price', 'budget', 'pricing', 'fee', 'rate', 'expense', 'investment']],
    ['timeline', ['schedule', 'duration', 'timeframe', 'deadline', 'delivery', 'time']],
    ['quality', ['excellence', 'standards', 'assurance', 'reliable', 'reliability']],
    ['services', ['offerings', 'solutions', 'capabilities', 'products']],
    ['financial', ['fiscal', 'monetary', 'revenue', 'budget', 'funding']],
    ['implementation', ['deployment', 'installation', 'rollout', 'execution']],
    ['support', ['maintenance', 'assistance', 'help', 'service']],
    ['company', ['organization', 'firm', 'business', 'enterprise']],
    ['client', ['customer', 'account', 'partner']],
    ['project', ['initiative', 'program', 'engagement', 'effort']],
    ['technical', ['technology', 'tech', 'it', 'system']],
    ['management', ['oversight', 'administration', 'governance', 'control']]
  ]);
  
  const queryWords = extractMeaningfulWords(query);
  const textWords = extractMeaningfulWords(text);
  
  let matches = 0;
  queryWords.forEach(qWord => {
    if (synonymMap.has(qWord)) {
      const synonyms = synonymMap.get(qWord) || [];
      if (synonyms.some(syn => textWords.includes(syn))) {
        matches++;
      }
    }
  });
  
  return queryWords.length > 0 ? matches / queryWords.length : 0;
}