import {readFile, writeFile, unlink, stat} from 'fs/promises' import {existsSync} from 'fs' import {join, basename} from 'path' import {Project, SyntaxKind} from 'ts-morph' import {prixExecAsync} from './utils' import pino from 'pino' const logger = pino({ level: process.env.LOG_LEVEL || 'info' }) export interface VerificationResult { passed: boolean checks: VerificationCheck[] overallFeedback: string canRetry: boolean errorMessage?: string } export interface VerificationCheck { name: string passed: boolean feedback: string duration?: number } export interface VerificationConfig { enableLint: boolean enableSyntax: boolean enableAST: boolean enableTests: boolean maxRetries: number timeout: number } const DEFAULT_CONFIG: VerificationConfig = { enableLint: true, enableSyntax: true, enableAST: true, enableTests: false, maxRetries: 3, timeout: 30000 } export class FixVerifier { private config: VerificationConfig private project: Project constructor(config: Partial = {}, project?: Project) { this.config = {...DEFAULT_CONFIG, ...config} this.project = project || new Project() } async verify( filename: string, remedyCode: string, startLine: number, endLine: number, workingDir: string ): Promise { const checks: VerificationCheck[] = [] let allPassed = true let canRetry = true let errorMessage: string | undefined if (this.config.enableSyntax) { const syntaxCheck = await this.runSyntaxCheck( filename, remedyCode, workingDir ) checks.push(syntaxCheck) if (!syntaxCheck.passed) { allPassed = false errorMessage = syntaxCheck.feedback } } if (this.config.enableLint && allPassed) { const lintCheck = await this.runLintCheck( filename, remedyCode, workingDir ) checks.push(lintCheck) if (!lintCheck.passed) { allPassed = false canRetry = false errorMessage = lintCheck.feedback } } if (this.config.enableAST && allPassed) { const astCheck = this.runASTValidation(filename, remedyCode) checks.push(astCheck) if (!astCheck.passed) { allPassed = false errorMessage = astCheck.feedback } } if (this.config.enableTests && allPassed) { const testCheck = await this.runTestCheck( filename, remedyCode, startLine, endLine, workingDir ) checks.push(testCheck) if (!testCheck.passed) { allPassed = false canRetry = false errorMessage = testCheck.feedback } } const overallFeedback = checks .map(c => `${c.passed ? '✅' : '❌'} ${c.name}: ${c.feedback}`) .join('\n') return { passed: allPassed, checks, overallFeedback, canRetry, errorMessage } } private async runSyntaxCheck( filename: string, remedyCode: string, workingDir: string ): Promise { const startTime = Date.now() const ext = filename.split('.').pop()?.toLowerCase() || '' let feedback = '' let passed = true const tempFile = join(workingDir, `__prix_verify_${Date.now()}.${ext}`) try { if (ext === 'ts' || ext === 'js') { await writeFile(tempFile, remedyCode, 'utf8') feedback = await this.verifyJSTypeScript(tempFile, ext, workingDir) passed = !feedback.includes('❌') } else if (ext === 'py') { await writeFile(tempFile, remedyCode, 'utf8') feedback = await this.verifyPython(tempFile) passed = !feedback.includes('❌') } else if (ext === 'go') { await writeFile(tempFile, remedyCode, 'utf8') feedback = await this.verifyGo(tempFile) passed = !feedback.includes('❌') } else { feedback = '⚠️ Syntax check not supported for this file type' } } catch (e: any) { passed = false feedback = `❌ Syntax check failed: ${e.message}` } finally { await this.cleanup(tempFile) } return { name: 'Syntax Check', passed, feedback, duration: Date.now() - startTime } } private async verifyJSTypeScript( tempFile: string, ext: string, workingDir: string ): Promise { let feedback = '' try { if (ext === 'ts') { const hasTypeScript = existsSync(join(workingDir, 'node_modules/typescript')) || existsSync(join(workingDir, 'node_modules/.bin/tsc')) if (hasTypeScript) { try { await prixExecAsync('npx', ['tsc', tempFile, '--noEmit', '--esModuleInterop', '--skipLibCheck', '--jsx', 'react'], { cwd: workingDir, timeout: 15000 }) feedback = 'TS check passed' } catch (e: any) { const errorOutput = e.stdout?.toString() || e.message || '' const relevantErrors = this.filterErrorsForFile( errorOutput, basename(tempFile) ) if (relevantErrors) { feedback = `❌ TS error: ${relevantErrors}` } else { feedback = '❌ TS syntax error' } } } else { feedback = '⚠️ TypeScript not installed, skipping TS check' } } else { try { await prixExecAsync('node', ['--check', tempFile], { timeout: 5000 }) feedback = 'Syntax check passed' } catch (e: any) { feedback = `❌ Syntax error: ${e.message}` } } } catch (e: any) { feedback = `❌ Check failed: ${e.message}` } return feedback } private async verifyPython(tempFile: string): Promise { try { await prixExecAsync('python3', ['-m', 'py_compile', tempFile], { timeout: 5000 }) return 'Python syntax passed' } catch (e: any) { const errorOutput = e.stderr?.toString() || e.message return `❌ Python syntax error: ${errorOutput}`.substring(0, 200) } } private async verifyGo(tempFile: string): Promise { try { await prixExecAsync('go', ['vet', tempFile], {timeout: 5000}) return 'Go syntax passed' } catch (e: any) { const errorOutput = e.stderr?.toString() || e.message return `❌ Go syntax error: ${errorOutput}`.substring(0, 200) } } private async runLintCheck( filename: string, remedyCode: string, workingDir: string ): Promise { const startTime = Date.now() const ext = filename.split('.').pop()?.toLowerCase() || '' let feedback = '' let passed = true const tempFile = join(workingDir, `__prix_lint_${Date.now()}.${ext}`) try { if ( (ext === 'ts' || ext === 'js') && existsSync(join(workingDir, 'package.json')) ) { await writeFile(tempFile, remedyCode, 'utf8') const hasEslint = existsSync(join(workingDir, '.eslintrc.json')) || existsSync(join(workingDir, '.eslintrc.js')) || existsSync(join(workingDir, '.eslintrc')) if (hasEslint) { try { await prixExecAsync('npx', ['eslint', tempFile, '--max-warnings', '0'], { cwd: workingDir, timeout: 15000 }) feedback = 'Lint passed' } catch (e: any) { const errorOutput = e.stdout?.toString() || e.message || '' const relevantErrors = this.filterErrorsForFile( errorOutput, basename(tempFile) ) feedback = `❌ Lint failed: ${relevantErrors || ' ESLint errors'}` passed = false } } else { feedback = '⚠️ ESLint not configured' } } else if (ext === 'py') { const hasPylint = await this.commandExists('pylint') const hasFlake8 = await this.commandExists('flake8') await writeFile(tempFile, remedyCode, 'utf8') if (hasPylint) { try { await prixExecAsync('pylint', [tempFile, '--disable=all', '--enable=E'], { timeout: 10000 }) feedback = 'Pylint passed' } catch (e: any) { feedback = `❌ Pylint: ${e.message}`.substring(0, 200) passed = false } } else if (hasFlake8) { try { await prixExecAsync('flake8', [tempFile], {timeout: 10000}) feedback = 'Flake8 passed' } catch (e: any) { feedback = `❌ Flake8: ${e.message}`.substring(0, 200) passed = false } } else { feedback = '⚠️ Python linter not available' } } } catch (e: any) { passed = false feedback = `❌ Lint failed: ${e.message}` } finally { await this.cleanup(tempFile) } return { name: 'Lint Check', passed, feedback, duration: Date.now() - startTime } } private runASTValidation( filename: string, remedyCode: string ): VerificationCheck { const startTime = Date.now() if (!filename.endsWith('.ts') && !filename.endsWith('.js')) { return { name: 'AST Validation', passed: true, feedback: 'Not applicable for this file type', duration: 0 } } try { const tempFile = `__prix_ast_${Date.now()}_${filename}` const sourceFile = this.project.createSourceFile(tempFile, remedyCode, { overwrite: true }) const diagnostics = sourceFile.getPreEmitDiagnostics() if (diagnostics.length > 0) { const errorMessages = diagnostics .slice(0, 3) .map(d => d.getMessageText()) .join('; ') return { name: 'AST Validation', passed: false, feedback: `❌ AST errors: ${errorMessages}`, duration: Date.now() - startTime } } const identifiers = sourceFile.getDescendantsOfKind(SyntaxKind.Identifier) const undefinedSymbols: string[] = [] for (const id of identifiers) { if (id.getSymbol()) continue const text = id.getText() const skipList = [ 'console', 'process', 'require', 'exports', 'module', 'Buffer', 'Array', 'Object', 'String', 'Number', 'Boolean', 'Date', 'Promise', 'Math', 'JSON', 'Map', 'Set', 'RegExp', 'Error', 'undefined', 'NaN', 'Infinity', 'parseInt', 'parseFloat' ] if (skipList.includes(text)) continue if (id.getParentIfKind(SyntaxKind.ImportSpecifier)) continue if (id.getParentIfKind(SyntaxKind.PropertyAccessExpression)) continue const parentKind = id.getParent()?.getKind() if (parentKind === SyntaxKind.PropertyAssignment) continue undefinedSymbols.push(text) } const uniqueUndefined = [...new Set(undefinedSymbols)].slice(0, 5) if (uniqueUndefined.length > 0) { return { name: 'AST Validation', passed: false, feedback: `❌ Hallucination detected: undefined symbols: ${uniqueUndefined.join( ', ' )}`, duration: Date.now() - startTime } } return { name: 'AST Validation', passed: true, feedback: 'No hallucinations detected', duration: Date.now() - startTime } } catch (e: any) { return { name: 'AST Validation', passed: false, feedback: `❌ AST validation failed: ${e.message}`, duration: Date.now() - startTime } } } private async runTestCheck( filename: string, remedyCode: string, startLine: number, endLine: number, workingDir: string ): Promise { const startTime = Date.now() if (!existsSync(join(workingDir, 'package.json'))) { return { name: 'Test Check', passed: true, feedback: '⚠️ No package.json found', duration: 0 } } const testFile = this.findCorrespondingTestFile(filename) if (!testFile) { return { name: 'Test Check', passed: true, feedback: '⚠️ No corresponding test file found', duration: 0 } } try { const testContent = await readFile(testFile, 'utf8') const testsForFunction = this.extractRelevantTests( testContent, filename, startLine ) if (!testsForFunction) { return { name: 'Test Check', passed: true, feedback: '⚠️ No relevant tests found for modified code', duration: 0 } } const tempTestFile = join(workingDir, `__prix_test_${Date.now()}.test.ts`) await writeFile(tempTestFile, testsForFunction, 'utf8') try { const hasJest = existsSync(join(workingDir, 'node_modules/jest')) || existsSync(join(workingDir, 'node_modules/.bin/jest')) if (hasJest) { try { await prixExecAsync('npx', ['jest', tempTestFile, '--passWithNoTests'], { cwd: workingDir, timeout: 30000 }) return { name: 'Test Check', passed: true, feedback: 'Test check passed', duration: Date.now() - startTime } } catch (e: any) { return { name: 'Test Check', passed: false, feedback: `❌ Test failed: ${e.message}`.substring(0, 200), duration: Date.now() - startTime } } } } finally { await this.cleanup(tempTestFile) } return { name: 'Test Check', passed: true, feedback: '⚠️ Test runner not available', duration: 0 } } catch (e: any) { return { name: 'Test Check', passed: false, feedback: `❌ Test check failed: ${e.message}`, duration: Date.now() - startTime } } } private findCorrespondingTestFile(filename: string): string | null { const baseDir = filename.substring(0, filename.lastIndexOf('/')) const basename = filename.substring(filename.lastIndexOf('/') + 1) const nameWithoutExt = basename.replace(/\.[^.]+$/, '') const patterns = [ join(baseDir, `${nameWithoutExt}.test.ts`), join(baseDir, `${nameWithoutExt}.spec.ts`), join(baseDir, '__tests__', `${nameWithoutExt}.ts`), join(baseDir, 'test', `${nameWithoutExt}.ts`), join(baseDir, 'tests', `${nameWithoutExt}.ts`) ] for (const pattern of patterns) { if (existsSync(pattern)) { return pattern } } return null } private extractRelevantTests( testContent: string, sourceFile: string, lineNumber: number ): string | null { const functionNameMatch = testContent.match( new RegExp(`(test|it)\\(['"]{1,2}([^'"]{1,30})['"]{1,2}`) ) if (!functionNameMatch) return null return testContent } private filterErrorsForFile( errorOutput: string, tempFile: string ): string | null { const lines = errorOutput.split('\n') const relevantLines = lines.filter( l => l.includes(tempFile) || l.includes('__prix') ) if (relevantLines.length === 0) return null return relevantLines.slice(0, 3).join('; ') } private async commandExists(cmd: string): Promise { try { await prixExecAsync('which', [cmd], {timeout: 1000}) return true } catch { return false } } private async cleanup(tempFile: string): Promise { try { if (existsSync(tempFile)) { await unlink(tempFile) } } catch { // Cleanup failures are non-fatal } } } export function createVerifier(project?: Project): FixVerifier { return new FixVerifier({}, project) }