src/eval/cli.ts

#!/usr/bin/env node
/**

 * PRIX AI Evaluation System - CLI

 * Command-line interface for running evaluations

 */

import * as fs from 'fs'
import * as path from 'path'
import {TestRunner, runTestCase} from './test-runner'
import {Evaluator} from './evaluator'
import {MetricsAggregator} from './metrics'
import {TestResult} from './types'

interface CLIOptions {
  command: 'eval' | 'single' | 'compare' | 'baseline' | 'report'
  testPath?: string
  testsDir: string
  outputDir: string
  verbose: boolean
  format: 'text' | 'json' | 'csv' | 'markdown'
  failOnRegression: boolean
}

const DEFAULT_TESTS_DIR = path.join(process.cwd(), 'tests')
const DEFAULT_OUTPUT_DIR = path.join(process.cwd(), 'eval-output')
const BASELINE_PATH = path.join(DEFAULT_OUTPUT_DIR, 'baseline_metrics.json')
const HISTORY_PATH = path.join(DEFAULT_OUTPUT_DIR, 'metrics_history.json')

function parseArgs(): CLIOptions {
  const args = process.argv.slice(2)

  const options: CLIOptions = {
    command: 'eval',
    testsDir: DEFAULT_TESTS_DIR,
    outputDir: DEFAULT_OUTPUT_DIR,
    verbose: false,
    format: 'text',
    failOnRegression: false
  }

  for (let i = 0; i < args.length; i++) {
    const arg = args[i]

    switch (arg) {
      case 'eval':
      case 'single':
      case 'compare':
      case 'baseline':
      case 'report':
        options.command = arg
        break
      case '--test':
      case '-t':
        options.testPath = args[++i]
        break
      case '--tests-dir':
      case '-d':
        options.testsDir = args[++i]
        break
      case '--output':
      case '-o':
        options.outputDir = args[++i]
        break
      case '--verbose':
      case '-v':
        options.verbose = true
        break
      case '--format':
      case '-f':
        options.format = args[++i] as any
        break
      case '--fail-on-regression':
        options.failOnRegression = true
        break
      case '--help':
      case '-h':
        printHelp()
        process.exit(0)
    }
  }

  return options
}

function printHelp(): void {
  console.log(`

PRIX AI Evaluation System



Usage: npm run test:eval [command] [options]



Commands:

  eval       Run all test cases (default)

  single     Run a single test case (--test required)

  compare    Compare current results with baseline

  baseline   Save current results as new baseline

  report     Generate report from last run



Options:

  -t, --test <path>        Path to specific test case

  -d, --tests-dir <path>   Directory containing tests (default: ./tests)

  -o, --output <path>      Output directory (default: ./eval-output)

  -v, --verbose            Enable verbose logging

  -f, --format <format>    Output format: text, json, csv, markdown

  --fail-on-regression     Exit with error if regression detected

  -h, --help               Show this help



Examples:

  npm run test:eval                    # Run all tests

  npm run test:eval single -t tests/bugs/mutation  # Run single test

  npm run test:eval compare            # Compare with baseline

  npm run test:eval baseline           # Save new baseline

  npm run test:eval -f json            # Output as JSON

`)
}

async function runAllTests(options: CLIOptions): Promise<void> {
  console.log('🚀 PRIX AI Evaluation System')
  console.log(`📁 Tests directory: ${options.testsDir}`)
  console.log(`📁 Output directory: ${options.outputDir}`)
  console.log('')

  const runner = new TestRunner({
    testsDir: options.testsDir,
    outputDir: options.outputDir,
    verbose: options.verbose,
    skipAutoPR: true
  })

  const testCases = runner.loadTestCases()

  if (testCases.length === 0) {
    console.error('❌ No test cases found!')
    process.exit(1)
  }

  console.log(`📋 Found ${testCases.length} test case(s)`)
  console.log('')

  const evaluator = new Evaluator()
  const results: TestResult[] = []

  for (const testCase of testCases) {
    const output = await runner.runTestCase(testCase)
    const result = evaluator.evaluate(testCase, output)
    results.push(result)

    // Print individual result
    if (options.format === 'text') {
      console.log(evaluator.generateReport(result))
    }
  }

  // Aggregate metrics
  const metrics = new MetricsAggregator({
    baselinePath: path.join(options.outputDir, 'baseline_metrics.json'),
    historyPath: path.join(options.outputDir, 'metrics_history.json')
  })

  const summary = metrics.aggregate(results)
  const regression = metrics.detectRegression(results)

  // Save results
  const resultsPath = path.join(options.outputDir, 'last_results.json')
  fs.writeFileSync(resultsPath, JSON.stringify({results, summary}, null, 2))

  // Append to history
  metrics.appendHistory(results)

  // Generate and print report
  if (options.format === 'text') {
    console.log(metrics.generateReport(summary, regression))
  } else {
    console.log(metrics.exportMetrics(summary, options.format))
  }

  // Exit with error if regression detected and flag is set
  if (options.failOnRegression && regression.hasRegression) {
    console.error('\n❌ REGRESSION DETECTED - Exiting with error')
    process.exit(1)
  }

  // Exit with error if any tests failed
  if (summary.failedTests > 0) {
    process.exit(1)
  }
}

async function runSingleTest(options: CLIOptions): Promise<void> {
  if (!options.testPath) {
    console.error('❌ --test path is required for single test mode')
    process.exit(1)
  }

  const fullPath = path.resolve(options.testPath)

  if (!fs.existsSync(fullPath)) {
    console.error(`❌ Test path not found: ${fullPath}`)
    process.exit(1)
  }

  console.log(`🧪 Running single test: ${fullPath}`)

  const output = await runTestCase(fullPath, {
    verbose: options.verbose,
    skipAutoPR: true
  })

  const expectedPath = path.join(fullPath, 'expected.json')
  const expected = JSON.parse(fs.readFileSync(expectedPath, 'utf8'))

  const evaluator = new Evaluator()

  const testCase = {
    name: path.basename(fullPath),
    path: fullPath,
    category: path.basename(path.dirname(fullPath)),
    basePath: path.join(fullPath, 'base'),
    headPath: path.join(fullPath, 'head'),
    expected
  }

  const result = evaluator.evaluate(testCase, output)

  if (options.format === 'text') {
    console.log(evaluator.generateReport(result))
  } else {
    console.log(JSON.stringify(result, null, 2))
  }

  if (!result.passed) {
    process.exit(1)
  }
}

async function compareWithBaseline(options: CLIOptions): Promise<void> {
  const resultsPath = path.join(options.outputDir, 'last_results.json')

  if (!fs.existsSync(resultsPath)) {
    console.error('❌ No results found. Run tests first.')
    process.exit(1)
  }

  const {results} = JSON.parse(fs.readFileSync(resultsPath, 'utf8'))

  const metrics = new MetricsAggregator({
    baselinePath: path.join(options.outputDir, 'baseline_metrics.json'),
    historyPath: path.join(options.outputDir, 'metrics_history.json')
  })

  const regression = metrics.detectRegression(results)

  console.log('📊 Regression Analysis')
  console.log('='.repeat(50))
  console.log(
    `Status: ${

      regression.hasRegression ? '❌ REGRESSION DETECTED' : '✅ No Regression'

    }`
  )
  console.log(`Accuracy Delta: ${(regression.accuracyDelta * 100).toFixed(2)}%`)

  if (regression.newFailures.length > 0) {
    console.log('\nNew Failures:')
    for (const test of regression.newFailures) {
      console.log(`  ❌ ${test}`)
    }
  }

  if (regression.fixedTests.length > 0) {
    console.log('\nFixed Tests:')
    for (const test of regression.fixedTests) {
      console.log(`  ✅ ${test}`)
    }
  }

  if (options.failOnRegression && regression.hasRegression) {
    process.exit(1)
  }
}

async function saveBaseline(options: CLIOptions): Promise<void> {
  const resultsPath = path.join(options.outputDir, 'last_results.json')

  if (!fs.existsSync(resultsPath)) {
    console.error('❌ No results found. Run tests first.')
    process.exit(1)
  }

  const {results} = JSON.parse(fs.readFileSync(resultsPath, 'utf8'))

  const metrics = new MetricsAggregator({
    baselinePath: path.join(options.outputDir, 'baseline_metrics.json'),
    historyPath: path.join(options.outputDir, 'metrics_history.json')
  })

  metrics.saveBaseline(results)
  console.log('✅ Baseline saved successfully')
}

async function generateReport(options: CLIOptions): Promise<void> {
  const resultsPath = path.join(options.outputDir, 'last_results.json')

  if (!fs.existsSync(resultsPath)) {
    console.error('❌ No results found. Run tests first.')
    process.exit(1)
  }

  const {summary} = JSON.parse(fs.readFileSync(resultsPath, 'utf8'))

  const metrics = new MetricsAggregator({
    baselinePath: path.join(options.outputDir, 'baseline_metrics.json'),
    historyPath: path.join(options.outputDir, 'metrics_history.json')
  })

  const regression = metrics.detectRegression([])

  if (options.format === 'text') {
    console.log(metrics.generateReport(summary, regression))
  } else {
    console.log(metrics.exportMetrics(summary, options.format))
  }
}

async function main(): Promise<void> {
  const options = parseArgs()

  try {
    switch (options.command) {
      case 'eval':
        await runAllTests(options)
        break
      case 'single':
        await runSingleTest(options)
        break
      case 'compare':
        await compareWithBaseline(options)
        break
      case 'baseline':
        await saveBaseline(options)
        break
      case 'report':
        await generateReport(options)
        break
    }
  } catch (err: any) {
    console.error(`❌ Error: ${err.message}`)
    if (options.verbose) {
      console.error(err.stack)
    }
    process.exit(1)
  }
}

main()