/** * PRIX AI Evaluation System - Test Runner * Simulates PRs locally and runs the full review pipeline */ import * as fs from 'fs' import * as path from 'path' import {execSync} from 'child_process' import {als, PRIXContext} from '../context' import {setCommenterContext} from '../commenter' import {octokit, setOctokit} from '../octokit' import {run as reviewPipeline} from '../review' import {Options} from '../options' import {Prompts} from '../prompts' import {MockGitHubLayer, suppressGitHubCalls} from './mock-github' import { TestCase, ExpectedResult, TestOutput, DiffFile, MockPRContext } from './types' export interface TestRunnerOptions { testsDir: string outputDir: string verbose?: boolean skipAutoPR?: boolean mockAI?: boolean } export class TestRunner { private options: TestRunnerOptions private currentTest: string | null = null constructor(options: TestRunnerOptions) { this.options = { verbose: false, skipAutoPR: true, mockAI: false, ...options } // Ensure output directory exists if (!fs.existsSync(this.options.outputDir)) { fs.mkdirSync(this.options.outputDir, {recursive: true}) } } /** * Load all test cases from the tests directory */ loadTestCases(): TestCase[] { const tests: TestCase[] = [] const categories = fs .readdirSync(this.options.testsDir) .filter(f => fs.statSync(path.join(this.options.testsDir, f)).isDirectory() ) for (const category of categories) { const categoryPath = path.join(this.options.testsDir, category) const testDirs = fs .readdirSync(categoryPath) .filter(f => fs.statSync(path.join(categoryPath, f)).isDirectory()) for (const testDir of testDirs) { const testPath = path.join(categoryPath, testDir) const expectedPath = path.join(testPath, 'expected.json') if (!fs.existsSync(expectedPath)) { console.warn(`⚠️ Skipping ${testDir}: no expected.json found`) continue } const expected: ExpectedResult = JSON.parse( fs.readFileSync(expectedPath, 'utf8') ) tests.push({ name: testDir, path: testPath, category, basePath: path.join(testPath, 'base'), headPath: path.join(testPath, 'head'), expected }) } } return tests } /** * Run a single test case */ async runTestCase(testCase: TestCase): Promise { const startTime = Date.now() this.currentTest = testCase.name console.log(`\n🧪 Running test: ${testCase.category}/${testCase.name}`) try { // 1. Load base and head files const files = this.loadDiffFiles(testCase) if (files.length === 0) { throw new Error('No files found in base/head directories') } // 2. Generate diffs const diffFiles = this.generateDiffs(files) // 3. Set up mock context const mockContext: MockPRContext = { number: 1, title: `[EVAL] ${testCase.name}`, body: 'Test PR for evaluation', base: {ref: testCase.basePath, sha: 'base-sha'}, head: {ref: testCase.headPath, sha: 'head-sha'}, files: diffFiles } // 4. Create mock GitHub layer const logFile = path.join(this.options.outputDir, `${testCase.name}.log`) const mockLayer = new MockGitHubLayer(mockContext, {logFile}) const probotContext = mockLayer.createMockProbotContext() // 5. Suppress actual GitHub calls suppressGitHubCalls() // 6. Run ALS context with working directory const mockOctokit = mockLayer.createMockOctokit() const store: PRIXContext = { probotContext, octokit: mockOctokit, repo: {owner: 'mock-owner', repo: 'mock-repo'}, workingDir: testCase.headPath } let output: TestOutput = { detectedIssues: false, autoPRTriggered: false, suggestions: [], comments: [], errors: [], executionTime: 0, changedLines: 0 } // 7. Run inside ALS await als.run(store, async () => { // Set up context for commenter setCommenterContext(probotContext) setOctokit(probotContext.octokit) // Create options and prompts - Options constructor takes individual parameters const options = new Options( this.options.verbose || false, // debug false, // disableReview true, // disableReleaseNotes '0', // maxFiles false, // reviewSimpleChanges false, // reviewCommentLGTM null, // pathFilters '', // systemMessage 'openai/gpt-oss-120b:free', // lightModel 'openai/gpt-oss-120b:free', // heavyModel '0.0', // modelTemperature '3', // retries '120000', // timeoutMS '6', // concurrencyLimit '6', // githubConcurrencyLimit 'https://openrouter.ai/api/v1', // apiBaseUrl 'en-US', // language false, // createRemedyPR !this.options.skipAutoPR, // enableAutoPR false // strictMode ) const prompts = new Prompts() try { // Run the actual review pipeline await reviewPipeline(probotContext, options, prompts) // Collect results from mock layer const comments = mockLayer.getComments() const prs = mockLayer.getCreatedPRs() // Calculate changed lines const changedLines = diffFiles.reduce((sum, f) => { return ( sum + f.diff .split('\n') .filter(l => l.startsWith('+') || l.startsWith('-')).length ) }, 0) // Parse suggestions from comments const suggestions = this.parseSuggestionsFromComments(comments) output = { detectedIssues: comments.length > 0 || suggestions.length > 0, issueType: this.inferIssueType(comments), confidence: this.inferConfidence(comments), autoPRTriggered: prs.length > 0, suggestions, comments, errors: [], executionTime: Date.now() - startTime, changedLines } } catch (err: any) { output.errors.push(err.message) console.error(`❌ Pipeline error: ${err.message}`) } }) return output } catch (err: any) { return { detectedIssues: false, autoPRTriggered: false, suggestions: [], comments: [], errors: [err.message], executionTime: Date.now() - startTime, changedLines: 0 } } } /** * Load files from base and head directories */ private loadDiffFiles( testCase: TestCase ): Array<{filename: string; base: string; head: string}> { const files: Array<{filename: string; base: string; head: string}> = [] if (!fs.existsSync(testCase.headPath)) { return files } const headFiles = this.getAllFiles(testCase.headPath) for (const relPath of headFiles) { const headFile = path.join(testCase.headPath, relPath) const baseFile = path.join(testCase.basePath, relPath) const headContent = fs.readFileSync(headFile, 'utf8') const baseContent = fs.existsSync(baseFile) ? fs.readFileSync(baseFile, 'utf8') : '' files.push({ filename: relPath, base: baseContent, head: headContent }) } return files } /** * Get all files recursively from a directory */ private getAllFiles(dir: string, basePath: string = ''): string[] { const files: string[] = [] const items = fs.readdirSync(dir) for (const item of items) { const fullPath = path.join(dir, item) const relPath = basePath ? path.join(basePath, item) : item if (fs.statSync(fullPath).isDirectory()) { files.push(...this.getAllFiles(fullPath, relPath)) } else { files.push(relPath) } } return files } /** * Generate unified diffs for files */ private generateDiffs( files: Array<{filename: string; base: string; head: string}> ): DiffFile[] { return files.map(f => { const diff = this.createUnifiedDiff(f.filename, f.base, f.head) return { filename: f.filename, baseContent: f.base, headContent: f.head, diff } }) } /** * Create unified diff format */ private createUnifiedDiff( filename: string, base: string, head: string ): string { const baseLines = base.split('\n') const headLines = head.split('\n') let diff = `--- a/${filename}\n+++ b/${filename}\n` // Simple line-by-line diff (for testing purposes) const maxLen = Math.max(baseLines.length, headLines.length) for (let i = 0; i < maxLen; i++) { const baseLine = baseLines[i] || '' const headLine = headLines[i] || '' if (baseLine !== headLine) { if (baseLine !== undefined) { diff += `-${baseLine}\n` } if (headLine !== undefined) { diff += `+${headLine}\n` } } else { diff += ` ${baseLine}\n` } } return diff } /** * Parse suggestions from review comments */ private parseSuggestionsFromComments(comments: any[]): any[] { const suggestions: any[] = [] for (const comment of comments) { // Extract suggestion blocks from comment body const suggestionMatch = comment.body.match( /```suggestion\n([\s\S]*?)\n```/ ) if (suggestionMatch) { suggestions.push({ filename: comment.path, startLine: comment.line || 1, endLine: comment.line || 1, suggestion: suggestionMatch[1].trim() }) } } return suggestions } /** * Infer issue type from comments */ private inferIssueType(comments: any[]): any { const types = ['security', 'bug', 'performance', 'style'] const text = comments .map(c => c.body) .join(' ') .toLowerCase() for (const type of types) { if (text.includes(type)) return type } return 'style' } /** * Infer confidence from comments */ private inferConfidence(comments: any[]): any { const text = comments .map(c => c.body) .join(' ') .toLowerCase() if (text.includes('high confidence') || text.includes('critical')) return 'high' if (text.includes('low confidence') || text.includes('minor')) return 'low' return 'medium' } } /** * Utility function to run a single test by path */ export async function runTestCase( testPath: string, options?: Partial ): Promise { const expectedPath = path.join(testPath, 'expected.json') if (!fs.existsSync(expectedPath)) { throw new Error(`No expected.json found at ${testPath}`) } const expected = JSON.parse(fs.readFileSync(expectedPath, 'utf8')) const testCase: TestCase = { name: path.basename(testPath), path: testPath, category: path.basename(path.dirname(testPath)), basePath: path.join(testPath, 'base'), headPath: path.join(testPath, 'head'), expected } const runner = new TestRunner({ testsDir: path.dirname(path.dirname(testPath)), outputDir: path.join(testPath, '..', '..', 'output'), ...options }) return runner.runTestCase(testCase) }