src/eval/test-runner.ts

/**
 * PRIX AI Evaluation System - Test Runner
 * Simulates PRs locally and runs the full review pipeline
 */

import * as fs from 'fs'
import * as path from 'path'
import {execSync} from 'child_process'
import {als, PRIXContext} from '../context'
import {setCommenterContext} from '../commenter'
import {octokit, setOctokit} from '../octokit'
import {run as reviewPipeline} from '../review'
import {Options} from '../options'
import {Prompts} from '../prompts'
import {MockGitHubLayer, suppressGitHubCalls} from './mock-github'
import {
  TestCase,
  ExpectedResult,
  TestOutput,
  DiffFile,
  MockPRContext
} from './types'

export interface TestRunnerOptions {
  testsDir: string
  outputDir: string
  verbose?: boolean
  skipAutoPR?: boolean
  mockAI?: boolean
}

export class TestRunner {
  private options: TestRunnerOptions
  private currentTest: string | null = null

  constructor(options: TestRunnerOptions) {
    this.options = {
      verbose: false,
      skipAutoPR: true,
      mockAI: false,
      ...options
    }

    // Ensure output directory exists
    if (!fs.existsSync(this.options.outputDir)) {
      fs.mkdirSync(this.options.outputDir, {recursive: true})
    }
  }

  /**
   * Load all test cases from the tests directory
   */
  loadTestCases(): TestCase[] {
    const tests: TestCase[] = []
    const categories = fs
      .readdirSync(this.options.testsDir)
      .filter(f =>
        fs.statSync(path.join(this.options.testsDir, f)).isDirectory()
      )

    for (const category of categories) {
      const categoryPath = path.join(this.options.testsDir, category)
      const testDirs = fs
        .readdirSync(categoryPath)
        .filter(f => fs.statSync(path.join(categoryPath, f)).isDirectory())

      for (const testDir of testDirs) {
        const testPath = path.join(categoryPath, testDir)
        const expectedPath = path.join(testPath, 'expected.json')

        if (!fs.existsSync(expectedPath)) {
          console.warn(`⚠️  Skipping ${testDir}: no expected.json found`)
          continue
        }

        const expected: ExpectedResult = JSON.parse(
          fs.readFileSync(expectedPath, 'utf8')
        )

        tests.push({
          name: testDir,
          path: testPath,
          category,
          basePath: path.join(testPath, 'base'),
          headPath: path.join(testPath, 'head'),
          expected
        })
      }
    }

    return tests
  }

  /**
   * Run a single test case
   */
  async runTestCase(testCase: TestCase): Promise<TestOutput> {
    const startTime = Date.now()
    this.currentTest = testCase.name

    console.log(`\n🧪 Running test: ${testCase.category}/${testCase.name}`)

    try {
      // 1. Load base and head files
      const files = this.loadDiffFiles(testCase)

      if (files.length === 0) {
        throw new Error('No files found in base/head directories')
      }

      // 2. Generate diffs
      const diffFiles = this.generateDiffs(files)

      // 3. Set up mock context
      const mockContext: MockPRContext = {
        number: 1,
        title: `[EVAL] ${testCase.name}`,
        body: 'Test PR for evaluation',
        base: {ref: testCase.basePath, sha: 'base-sha'},
        head: {ref: testCase.headPath, sha: 'head-sha'},
        files: diffFiles
      }

      // 4. Create mock GitHub layer
      const logFile = path.join(this.options.outputDir, `${testCase.name}.log`)
      const mockLayer = new MockGitHubLayer(mockContext, {logFile})
      const probotContext = mockLayer.createMockProbotContext()

      // 5. Suppress actual GitHub calls
      suppressGitHubCalls()

      // 6. Run ALS context with working directory
      const mockOctokit = mockLayer.createMockOctokit()
      const store: PRIXContext = {
        probotContext,
        octokit: mockOctokit,
        repo: {owner: 'mock-owner', repo: 'mock-repo'},
        workingDir: testCase.headPath
      }

      let output: TestOutput = {
        detectedIssues: false,
        autoPRTriggered: false,
        suggestions: [],
        comments: [],
        errors: [],
        executionTime: 0,
        changedLines: 0
      }

      // 7. Run inside ALS
      await als.run(store, async () => {
        // Set up context for commenter
        setCommenterContext(probotContext)
        setOctokit(probotContext.octokit)

        // Create options and prompts - Options constructor takes individual parameters
        const options = new Options(
          this.options.verbose || false, // debug
          false, // disableReview
          true, // disableReleaseNotes
          '0', // maxFiles
          false, // reviewSimpleChanges
          false, // reviewCommentLGTM
          null, // pathFilters
          '', // systemMessage
          'openai/gpt-oss-120b:free', // lightModel
          'openai/gpt-oss-120b:free', // heavyModel
          '0.0', // modelTemperature
          '3', // retries
          '120000', // timeoutMS
          '6', // concurrencyLimit
          '6', // githubConcurrencyLimit
          'https://openrouter.ai/api/v1', // apiBaseUrl
          'en-US', // language
          false, // createRemedyPR
          !this.options.skipAutoPR, // enableAutoPR
          false // strictMode
        )

        const prompts = new Prompts()

        try {
          // Run the actual review pipeline
          await reviewPipeline(probotContext, options, prompts)

          // Collect results from mock layer
          const comments = mockLayer.getComments()
          const prs = mockLayer.getCreatedPRs()

          // Calculate changed lines
          const changedLines = diffFiles.reduce((sum, f) => {
            return (
              sum +
              f.diff
                .split('\n')
                .filter(l => l.startsWith('+') || l.startsWith('-')).length
            )
          }, 0)

          // Parse suggestions from comments
          const suggestions = this.parseSuggestionsFromComments(comments)

          output = {
            detectedIssues: comments.length > 0 || suggestions.length > 0,
            issueType: this.inferIssueType(comments),
            confidence: this.inferConfidence(comments),
            autoPRTriggered: prs.length > 0,
            suggestions,
            comments,
            errors: [],
            executionTime: Date.now() - startTime,
            changedLines
          }
        } catch (err: any) {
          output.errors.push(err.message)
          console.error(`❌ Pipeline error: ${err.message}`)
        }
      })

      return output
    } catch (err: any) {
      return {
        detectedIssues: false,
        autoPRTriggered: false,
        suggestions: [],
        comments: [],
        errors: [err.message],
        executionTime: Date.now() - startTime,
        changedLines: 0
      }
    }
  }

  /**
   * Load files from base and head directories
   */
  private loadDiffFiles(
    testCase: TestCase
  ): Array<{filename: string; base: string; head: string}> {
    const files: Array<{filename: string; base: string; head: string}> = []

    if (!fs.existsSync(testCase.headPath)) {
      return files
    }

    const headFiles = this.getAllFiles(testCase.headPath)

    for (const relPath of headFiles) {
      const headFile = path.join(testCase.headPath, relPath)
      const baseFile = path.join(testCase.basePath, relPath)

      const headContent = fs.readFileSync(headFile, 'utf8')
      const baseContent = fs.existsSync(baseFile)
        ? fs.readFileSync(baseFile, 'utf8')
        : ''

      files.push({
        filename: relPath,
        base: baseContent,
        head: headContent
      })
    }

    return files
  }

  /**
   * Get all files recursively from a directory
   */
  private getAllFiles(dir: string, basePath: string = ''): string[] {
    const files: string[] = []
    const items = fs.readdirSync(dir)

    for (const item of items) {
      const fullPath = path.join(dir, item)
      const relPath = basePath ? path.join(basePath, item) : item

      if (fs.statSync(fullPath).isDirectory()) {
        files.push(...this.getAllFiles(fullPath, relPath))
      } else {
        files.push(relPath)
      }
    }

    return files
  }

  /**
   * Generate unified diffs for files
   */
  private generateDiffs(
    files: Array<{filename: string; base: string; head: string}>
  ): DiffFile[] {
    return files.map(f => {
      const diff = this.createUnifiedDiff(f.filename, f.base, f.head)
      return {
        filename: f.filename,
        baseContent: f.base,
        headContent: f.head,
        diff
      }
    })
  }

  /**
   * Create unified diff format
   */
  private createUnifiedDiff(
    filename: string,
    base: string,
    head: string
  ): string {
    const baseLines = base.split('\n')
    const headLines = head.split('\n')

    let diff = `--- a/${filename}\n+++ b/${filename}\n`

    // Simple line-by-line diff (for testing purposes)
    const maxLen = Math.max(baseLines.length, headLines.length)

    for (let i = 0; i < maxLen; i++) {
      const baseLine = baseLines[i] || ''
      const headLine = headLines[i] || ''

      if (baseLine !== headLine) {
        if (baseLine !== undefined) {
          diff += `-${baseLine}\n`
        }
        if (headLine !== undefined) {
          diff += `+${headLine}\n`
        }
      } else {
        diff += ` ${baseLine}\n`
      }
    }

    return diff
  }

  /**
   * Parse suggestions from review comments
   */
  private parseSuggestionsFromComments(comments: any[]): any[] {
    const suggestions: any[] = []

    for (const comment of comments) {
      // Extract suggestion blocks from comment body
      const suggestionMatch = comment.body.match(
        /```suggestion\n([\s\S]*?)\n```/
      )
      if (suggestionMatch) {
        suggestions.push({
          filename: comment.path,
          startLine: comment.line || 1,
          endLine: comment.line || 1,
          suggestion: suggestionMatch[1].trim()
        })
      }
    }

    return suggestions
  }

  /**
   * Infer issue type from comments
   */
  private inferIssueType(comments: any[]): any {
    const types = ['security', 'bug', 'performance', 'style']
    const text = comments
      .map(c => c.body)
      .join(' ')
      .toLowerCase()

    for (const type of types) {
      if (text.includes(type)) return type
    }
    return 'style'
  }

  /**
   * Infer confidence from comments
   */
  private inferConfidence(comments: any[]): any {
    const text = comments
      .map(c => c.body)
      .join(' ')
      .toLowerCase()

    if (text.includes('high confidence') || text.includes('critical'))
      return 'high'
    if (text.includes('low confidence') || text.includes('minor')) return 'low'
    return 'medium'
  }
}

/**
 * Utility function to run a single test by path
 */
export async function runTestCase(
  testPath: string,
  options?: Partial<TestRunnerOptions>
): Promise<TestOutput> {
  const expectedPath = path.join(testPath, 'expected.json')

  if (!fs.existsSync(expectedPath)) {
    throw new Error(`No expected.json found at ${testPath}`)
  }

  const expected = JSON.parse(fs.readFileSync(expectedPath, 'utf8'))

  const testCase: TestCase = {
    name: path.basename(testPath),
    path: testPath,
    category: path.basename(path.dirname(testPath)),
    basePath: path.join(testPath, 'base'),
    headPath: path.join(testPath, 'head'),
    expected
  }

  const runner = new TestRunner({
    testsDir: path.dirname(path.dirname(testPath)),
    outputDir: path.join(testPath, '..', '..', 'output'),
    ...options
  })

  return runner.runTestCase(testCase)
}