Spaces:
Paused
Paused
| export interface ConfidenceFactors { | |
| patternMatchScore: number | |
| contextRichness: number | |
| historicalAccuracy: number | |
| severityMatchesKnownBug: boolean | |
| hasExplicitEvidence: boolean | |
| teamAcceptanceRate: number | |
| patternTypeAccuracy: number | |
| } | |
| export interface CalibratedConfidence { | |
| score: number | |
| factors: ConfidenceFactors | |
| reasoning: string | |
| } | |
| const KNOWN_BUG_PATTERNS: Array<{ | |
| pattern: RegExp | |
| severity: 'critical' | 'major' | 'minor' | 'info' | |
| category: string | |
| }> = [ | |
| { | |
| pattern: /null.*pointer|NPE|null.*check/i, | |
| severity: 'critical', | |
| category: 'null-pointer' | |
| }, | |
| { | |
| pattern: /sql.*injection|SELECT.*WHERE.*\+|query.*concat/i, | |
| severity: 'critical', | |
| category: 'sql-injection' | |
| }, | |
| { | |
| pattern: /password|secret|api.*key.*=.*['"]|token.*=.*['"]/i, | |
| severity: 'critical', | |
| category: 'secret-exposure' | |
| }, | |
| { | |
| pattern: /eval\(|new.*Function\(|innerHTML\s*=|document\.write/i, | |
| severity: 'critical', | |
| category: 'xss' | |
| }, | |
| { | |
| pattern: /race.*condition|concurrent.*modif|deadlock|mutex/i, | |
| severity: 'major', | |
| category: 'concurrency' | |
| }, | |
| { | |
| pattern: /memory.*leak|dispose\(|close\(|gc\./i, | |
| severity: 'major', | |
| category: 'resource-leak' | |
| }, | |
| { | |
| pattern: /async.*without.*await|unhandled.*promise|catch.*without.*throw/i, | |
| severity: 'major', | |
| category: 'async-issues' | |
| }, | |
| { | |
| pattern: /try.*without.*catch|catch.*empty|swallow.*exception/i, | |
| severity: 'minor', | |
| category: 'error-handling' | |
| }, | |
| { | |
| pattern: /any.*type|cast.*any|as.*any/i, | |
| severity: 'info', | |
| category: 'type-safety' | |
| }, | |
| { | |
| pattern: /console\.(log|debug|warn)/i, | |
| severity: 'info', | |
| category: 'logging' | |
| }, | |
| { | |
| pattern: /TODO|FIXME|HACK|XXX|BUG:/i, | |
| severity: 'info', | |
| category: 'tech-debt' | |
| }, | |
| { | |
| pattern: /division.*by.*zero|divide.*0|x\s*\/\s*0/i, | |
| severity: 'major', | |
| category: 'arithmetic' | |
| }, | |
| { | |
| pattern: /off.*by.*one|index.*length|length.*-\s*1/i, | |
| severity: 'minor', | |
| category: 'logic-error' | |
| }, | |
| { | |
| pattern: /useEffect.*missing.*dep|dependency.*array/i, | |
| severity: 'minor', | |
| category: 'react-hooks' | |
| }, | |
| { | |
| pattern: /mutation.*props|props.*change|immutable/i, | |
| severity: 'minor', | |
| category: 'immutability' | |
| }, | |
| { | |
| pattern: /hardcoded|hard.*coded/i, | |
| severity: 'minor', | |
| category: 'hardcoding' | |
| }, | |
| { | |
| pattern: /circular.*depend|lazy.*load|circular/i, | |
| severity: 'major', | |
| category: 'architecture' | |
| }, | |
| { | |
| pattern: /sql.*string|query.*build|string.*concat.*sql/i, | |
| severity: 'critical', | |
| category: 'sql-injection' | |
| }, | |
| { | |
| pattern: /crypto.*md5|sha1.*encrypt|md5.*hash/i, | |
| severity: 'critical', | |
| category: 'crypto' | |
| }, | |
| { | |
| pattern: /http.*not.*https|mixed.*content|insecure.*request/i, | |
| severity: 'major', | |
| category: 'security' | |
| }, | |
| { | |
| pattern: /cookie.*without.*secure|secure.*flag.*false/i, | |
| severity: 'major', | |
| category: 'security' | |
| }, | |
| { | |
| pattern: /sudo|chmod\s+777|root.*login/i, | |
| severity: 'critical', | |
| category: 'security' | |
| }, | |
| { | |
| pattern: /catch.*(e|err).*{[\s]*}/i, | |
| severity: 'minor', | |
| category: 'error-handling' | |
| }, | |
| { | |
| pattern: /setTimeout.*0|setImmediate/i, | |
| severity: 'info', | |
| category: 'timing' | |
| }, | |
| { | |
| pattern: /double.*click|race.*click|multiple.*submit/i, | |
| severity: 'minor', | |
| category: 'ux' | |
| } | |
| ] | |
| const POSITIVE_INDICATORS = [ | |
| {indicator: 'suggestion', weight: 0.15}, | |
| {indicator: '```suggestion', weight: 0.2}, | |
| {indicator: 'fix:', weight: 0.1}, | |
| {indicator: 'fixed code', weight: 0.1}, | |
| {indicator: 'correct', weight: 0.05}, | |
| {indicator: 'instead of', weight: 0.05}, | |
| {indicator: 'should be', weight: 0.05}, | |
| {indicator: 'best practice', weight: 0.05} | |
| ] | |
| const NEGATIVE_INDICATORS = [ | |
| {indicator: 'maybe', weight: -0.15}, | |
| {indicator: 'might', weight: -0.1}, | |
| {indicator: 'possibly', weight: -0.1}, | |
| {indicator: 'i think', weight: -0.2}, | |
| {indicator: 'unclear', weight: -0.1}, | |
| {indicator: '不确定', weight: -0.15}, | |
| {indicator: 'perhaps', weight: -0.1}, | |
| {indicator: 'could be', weight: -0.1}, | |
| {indicator: 'not sure', weight: -0.15}, | |
| {indicator: 'likely', weight: -0.05} | |
| ] | |
| export class ConfidenceCalibrator { | |
| private historicalAccuracies: Map<string, number[]> = new Map() | |
| private feedbackHistory: Array<{ | |
| pattern: string | |
| helpful: boolean | |
| timestamp: number | |
| }> = [] | |
| private readonly persistPath = '/tmp/prix_feedback.json' | |
| private feedbackTracker: any = null | |
| constructor(feedbackTracker: any = null) { | |
| this.loadPersisted() | |
| if (feedbackTracker) { | |
| this.feedbackTracker = feedbackTracker | |
| } else { | |
| try { | |
| const {FeedbackTracker} = require('./feedback-tracker') | |
| this.feedbackTracker = new FeedbackTracker() | |
| } catch (e) { | |
| // FeedbackTracker not available, continue without it | |
| } | |
| } | |
| } | |
| private loadPersisted(): void { | |
| try { | |
| if (require('fs').existsSync(this.persistPath)) { | |
| const data = JSON.parse( | |
| require('fs').readFileSync(this.persistPath, 'utf8') | |
| ) | |
| if (Array.isArray(data)) { | |
| this.feedbackHistory = data.map((item: any) => ({ | |
| pattern: item.pattern || '', | |
| helpful: item.helpful || false, | |
| timestamp: item.timestamp || Date.now() | |
| })) | |
| this.rebuildHistoricalAccuracies() | |
| } | |
| } | |
| } catch (e) { | |
| // Start fresh if persistence fails | |
| } | |
| } | |
| private persist(): void { | |
| try { | |
| require('fs').writeFileSync( | |
| this.persistPath, | |
| JSON.stringify(this.feedbackHistory.slice(-1000)) | |
| ) | |
| } catch (e) { | |
| // Persistence failure is non-fatal | |
| } | |
| } | |
| private rebuildHistoricalAccuracies(): void { | |
| const patternMap = new Map<string, number[]>() | |
| for (const entry of this.feedbackHistory) { | |
| const accuracies = patternMap.get(entry.pattern) || [] | |
| accuracies.push(entry.helpful ? 1 : 0) | |
| patternMap.set(entry.pattern, accuracies) | |
| } | |
| for (const [pattern, accuracies] of patternMap) { | |
| this.historicalAccuracies.set(pattern, accuracies) | |
| } | |
| } | |
| recordFeedback(pattern: string, helpful: boolean): void { | |
| this.feedbackHistory.push({pattern, helpful, timestamp: Date.now()}) | |
| const accuracies = this.historicalAccuracies.get(pattern) || [] | |
| accuracies.push(helpful ? 1 : 0) | |
| this.historicalAccuracies.set(pattern, accuracies) | |
| this.persist() | |
| } | |
| calibrate( | |
| aiReportedConfidence: number | undefined, | |
| comment: string, | |
| contextTokens: number, | |
| maxContextTokens: number, | |
| severity: string | undefined, | |
| teamId?: string, | |
| repoSlug?: string, | |
| patternType?: string, | |
| filename?: string | |
| ): CalibratedConfidence { | |
| const factors = this.calculateFactors( | |
| comment, | |
| contextTokens, | |
| maxContextTokens, | |
| severity, | |
| teamId, | |
| repoSlug, | |
| patternType, | |
| filename | |
| ) | |
| let score: number | |
| if (aiReportedConfidence !== undefined) { | |
| const deBiased = this.removeLLMBias(aiReportedConfidence) | |
| score = this.combineWithFactors(deBiased, factors) | |
| } else { | |
| score = this.computeFromFactorsOnly(factors) | |
| } | |
| score = Math.max(0, Math.min(100, score)) | |
| return { | |
| score: Math.round(score), | |
| factors, | |
| reasoning: this.generateReasoning(factors, score) | |
| } | |
| } | |
| private calculateFactors( | |
| comment: string, | |
| contextTokens: number, | |
| maxContextTokens: number, | |
| severity: string | undefined, | |
| teamId?: string, | |
| repoSlug?: string, | |
| patternType?: string, | |
| filename?: string | |
| ): ConfidenceFactors { | |
| const patternMatchScore = this.calculatePatternMatch(comment) | |
| const contextRichness = Math.min( | |
| 1, | |
| contextTokens / Math.max(1, maxContextTokens * 0.5) | |
| ) | |
| const historicalAccuracy = this.calculateHistoricalAccuracy(comment) | |
| const severityMatchesKnownBug = severity | |
| ? this.matchesKnownBugPattern(comment, severity) | |
| : this.matchesAnyKnownBugPattern(comment) | |
| const hasExplicitEvidence = this.hasExplicitEvidence(comment) | |
| const {teamAcceptanceRate, patternTypeAccuracy} = this.calculateTeamMetrics( | |
| teamId, | |
| repoSlug, | |
| patternType, | |
| filename | |
| ) | |
| return { | |
| patternMatchScore, | |
| contextRichness, | |
| historicalAccuracy, | |
| severityMatchesKnownBug, | |
| hasExplicitEvidence, | |
| teamAcceptanceRate, | |
| patternTypeAccuracy | |
| } | |
| } | |
| private calculateTeamMetrics( | |
| teamId?: string, | |
| repoSlug?: string, | |
| patternType?: string, | |
| filename?: string | |
| ): {teamAcceptanceRate: number; patternTypeAccuracy: number} { | |
| if (!this.feedbackTracker || !teamId || !repoSlug) { | |
| return {teamAcceptanceRate: 0.5, patternTypeAccuracy: 0.5} | |
| } | |
| let teamAcceptanceRate = 0.5 | |
| let patternTypeAccuracy = 0.5 | |
| if (patternType && filename) { | |
| teamAcceptanceRate = this.feedbackTracker.getPatternAcceptanceRate( | |
| patternType, | |
| teamId, | |
| repoSlug | |
| ) | |
| patternTypeAccuracy = this.feedbackTracker.getFilenameAcceptanceRate( | |
| filename, | |
| teamId, | |
| repoSlug | |
| ) | |
| } | |
| return {teamAcceptanceRate, patternTypeAccuracy} | |
| } | |
| private calculatePatternMatch(comment: string): number { | |
| const lowerComment = comment.toLowerCase() | |
| let score = 0.5 | |
| for (const {indicator, weight} of POSITIVE_INDICATORS) { | |
| if (lowerComment.includes(indicator.toLowerCase())) { | |
| score += weight | |
| } | |
| } | |
| for (const {indicator, weight} of NEGATIVE_INDICATORS) { | |
| if (lowerComment.includes(indicator.toLowerCase())) { | |
| score += weight | |
| } | |
| } | |
| if (lowerComment.includes('why this matters')) { | |
| score += 0.1 | |
| } | |
| if ( | |
| lowerComment.includes('production') || | |
| lowerComment.includes('production environment') | |
| ) { | |
| score += 0.05 | |
| } | |
| return Math.max(0, Math.min(1, score)) | |
| } | |
| private calculateHistoricalAccuracy(comment: string): number { | |
| const lowerComment = comment.toLowerCase() | |
| for (const [pattern, accuracies] of this.historicalAccuracies) { | |
| if (lowerComment.includes(pattern.toLowerCase())) { | |
| const avg = accuracies.reduce((a, b) => a + b, 0) / accuracies.length | |
| return avg | |
| } | |
| } | |
| const relevantFeedback = this.feedbackHistory.filter(f => | |
| f.pattern | |
| .split(' ') | |
| .some( | |
| word => word.length > 4 && lowerComment.includes(word.toLowerCase()) | |
| ) | |
| ) | |
| if (relevantFeedback.length === 0) { | |
| return 0.5 | |
| } | |
| const accuracy = | |
| relevantFeedback.filter(f => f.helpful).length / relevantFeedback.length | |
| return accuracy | |
| } | |
| private matchesKnownBugPattern(comment: string, severity: string): boolean { | |
| const lowerComment = comment.toLowerCase() | |
| for (const {pattern, severity: patternSeverity} of KNOWN_BUG_PATTERNS) { | |
| if (pattern.test(lowerComment) && patternSeverity === severity) { | |
| return true | |
| } | |
| } | |
| return false | |
| } | |
| private matchesAnyKnownBugPattern(comment: string): boolean { | |
| const lowerComment = comment.toLowerCase() | |
| for (const {pattern} of KNOWN_BUG_PATTERNS) { | |
| if (pattern.test(lowerComment)) { | |
| return true | |
| } | |
| } | |
| return false | |
| } | |
| private hasExplicitEvidence(comment: string): boolean { | |
| const explicitPhrases = [ | |
| 'this will crash', | |
| 'this causes', | |
| 'this leads to', | |
| 'results in', | |
| 'fails when', | |
| 'breaks when', | |
| 'error when', | |
| 'crash if', | |
| 'exception if', | |
| 'undefined behavior', | |
| 'undefined is', | |
| 'will throw', | |
| 'will fail' | |
| ] | |
| const lowerComment = comment.toLowerCase() | |
| return explicitPhrases.some(phrase => lowerComment.includes(phrase)) | |
| } | |
| private removeLLMBias(reportedConfidence: number): number { | |
| if (reportedConfidence >= 95) { | |
| return 75 | |
| } | |
| if (reportedConfidence >= 85) { | |
| return 70 | |
| } | |
| if (reportedConfidence >= 70) { | |
| return 62 | |
| } | |
| if (reportedConfidence >= 60) { | |
| return 55 | |
| } | |
| return reportedConfidence * 0.9 | |
| } | |
| private combineWithFactors( | |
| baseScore: number, | |
| factors: ConfidenceFactors | |
| ): number { | |
| const weights = { | |
| base: 0.25, | |
| pattern: 0.2, | |
| context: 0.1, | |
| historical: 0.15, | |
| knownBug: 0.05, | |
| teamAcceptance: 0.15, | |
| patternAccuracy: 0.1 | |
| } | |
| let score = baseScore * weights.base | |
| score += factors.patternMatchScore * 100 * weights.pattern | |
| score += factors.contextRichness * 100 * weights.context | |
| score += factors.historicalAccuracy * 100 * weights.historical | |
| if (factors.severityMatchesKnownBug) { | |
| score += 10 * weights.knownBug | |
| } | |
| if (factors.hasExplicitEvidence) { | |
| score += 5 | |
| } | |
| score += factors.teamAcceptanceRate * 100 * weights.teamAcceptance | |
| score += factors.patternTypeAccuracy * 100 * weights.patternAccuracy | |
| return score | |
| } | |
| private computeFromFactorsOnly(factors: ConfidenceFactors): number { | |
| const weights = { | |
| pattern: 0.3, | |
| context: 0.15, | |
| historical: 0.2, | |
| knownBug: 0.15, | |
| teamAcceptance: 0.12, | |
| patternAccuracy: 0.08 | |
| } | |
| let score = 50 | |
| score += factors.patternMatchScore * 50 * weights.pattern | |
| score += factors.contextRichness * 50 * weights.context | |
| score += factors.historicalAccuracy * 50 * weights.historical | |
| if (factors.severityMatchesKnownBug) { | |
| score += 10 * weights.knownBug | |
| } | |
| if (factors.hasExplicitEvidence) { | |
| score += 5 | |
| } | |
| score += factors.teamAcceptanceRate * 50 * weights.teamAcceptance | |
| score += factors.patternTypeAccuracy * 50 * weights.patternAccuracy | |
| return score | |
| } | |
| private generateReasoning(factors: ConfidenceFactors, score: number): string { | |
| const reasons: string[] = [] | |
| if (factors.patternMatchScore > 0.7) { | |
| reasons.push('strong pattern match') | |
| } else if (factors.patternMatchScore < 0.4) { | |
| reasons.push('weak pattern match') | |
| } | |
| if (factors.contextRichness > 0.8) { | |
| reasons.push('rich context') | |
| } else if (factors.contextRichness < 0.3) { | |
| reasons.push('limited context') | |
| } | |
| if (factors.historicalAccuracy > 0.7) { | |
| reasons.push('historically accurate') | |
| } else if ( | |
| factors.historicalAccuracy < 0.4 && | |
| factors.historicalAccuracy !== 0.5 | |
| ) { | |
| reasons.push('historically unreliable') | |
| } | |
| if (factors.severityMatchesKnownBug) { | |
| reasons.push('matches known bug pattern') | |
| } | |
| if (factors.hasExplicitEvidence) { | |
| reasons.push('has explicit evidence') | |
| } | |
| if (factors.teamAcceptanceRate > 0.6) { | |
| reasons.push('high team acceptance') | |
| } else if ( | |
| factors.teamAcceptanceRate < 0.3 && | |
| factors.teamAcceptanceRate !== 0.5 | |
| ) { | |
| reasons.push('low team acceptance') | |
| } | |
| if (factors.patternTypeAccuracy > 0.6) { | |
| reasons.push('matches accepted patterns') | |
| } | |
| return reasons.length > 0 | |
| ? `Calibrated based on: ${reasons.join(', ')}` | |
| : 'Calibrated using default heuristics' | |
| } | |
| } | |
| export const confidenceCalibrator = new ConfidenceCalibrator() | |