PRIX / src /confidence.ts
Rachit-Tw's picture
Upload 169 files
9284ad7 verified
Raw
History Blame Contribute Delete
15.7 kB
export interface ConfidenceFactors {
patternMatchScore: number
contextRichness: number
historicalAccuracy: number
severityMatchesKnownBug: boolean
hasExplicitEvidence: boolean
teamAcceptanceRate: number
patternTypeAccuracy: number
}
export interface CalibratedConfidence {
score: number
factors: ConfidenceFactors
reasoning: string
}
const KNOWN_BUG_PATTERNS: Array<{
pattern: RegExp
severity: 'critical' | 'major' | 'minor' | 'info'
category: string
}> = [
{
pattern: /null.*pointer|NPE|null.*check/i,
severity: 'critical',
category: 'null-pointer'
},
{
pattern: /sql.*injection|SELECT.*WHERE.*\+|query.*concat/i,
severity: 'critical',
category: 'sql-injection'
},
{
pattern: /password|secret|api.*key.*=.*['"]|token.*=.*['"]/i,
severity: 'critical',
category: 'secret-exposure'
},
{
pattern: /eval\(|new.*Function\(|innerHTML\s*=|document\.write/i,
severity: 'critical',
category: 'xss'
},
{
pattern: /race.*condition|concurrent.*modif|deadlock|mutex/i,
severity: 'major',
category: 'concurrency'
},
{
pattern: /memory.*leak|dispose\(|close\(|gc\./i,
severity: 'major',
category: 'resource-leak'
},
{
pattern: /async.*without.*await|unhandled.*promise|catch.*without.*throw/i,
severity: 'major',
category: 'async-issues'
},
{
pattern: /try.*without.*catch|catch.*empty|swallow.*exception/i,
severity: 'minor',
category: 'error-handling'
},
{
pattern: /any.*type|cast.*any|as.*any/i,
severity: 'info',
category: 'type-safety'
},
{
pattern: /console\.(log|debug|warn)/i,
severity: 'info',
category: 'logging'
},
{
pattern: /TODO|FIXME|HACK|XXX|BUG:/i,
severity: 'info',
category: 'tech-debt'
},
{
pattern: /division.*by.*zero|divide.*0|x\s*\/\s*0/i,
severity: 'major',
category: 'arithmetic'
},
{
pattern: /off.*by.*one|index.*length|length.*-\s*1/i,
severity: 'minor',
category: 'logic-error'
},
{
pattern: /useEffect.*missing.*dep|dependency.*array/i,
severity: 'minor',
category: 'react-hooks'
},
{
pattern: /mutation.*props|props.*change|immutable/i,
severity: 'minor',
category: 'immutability'
},
{
pattern: /hardcoded|hard.*coded/i,
severity: 'minor',
category: 'hardcoding'
},
{
pattern: /circular.*depend|lazy.*load|circular/i,
severity: 'major',
category: 'architecture'
},
{
pattern: /sql.*string|query.*build|string.*concat.*sql/i,
severity: 'critical',
category: 'sql-injection'
},
{
pattern: /crypto.*md5|sha1.*encrypt|md5.*hash/i,
severity: 'critical',
category: 'crypto'
},
{
pattern: /http.*not.*https|mixed.*content|insecure.*request/i,
severity: 'major',
category: 'security'
},
{
pattern: /cookie.*without.*secure|secure.*flag.*false/i,
severity: 'major',
category: 'security'
},
{
pattern: /sudo|chmod\s+777|root.*login/i,
severity: 'critical',
category: 'security'
},
{
pattern: /catch.*(e|err).*{[\s]*}/i,
severity: 'minor',
category: 'error-handling'
},
{
pattern: /setTimeout.*0|setImmediate/i,
severity: 'info',
category: 'timing'
},
{
pattern: /double.*click|race.*click|multiple.*submit/i,
severity: 'minor',
category: 'ux'
}
]
const POSITIVE_INDICATORS = [
{indicator: 'suggestion', weight: 0.15},
{indicator: '```suggestion', weight: 0.2},
{indicator: 'fix:', weight: 0.1},
{indicator: 'fixed code', weight: 0.1},
{indicator: 'correct', weight: 0.05},
{indicator: 'instead of', weight: 0.05},
{indicator: 'should be', weight: 0.05},
{indicator: 'best practice', weight: 0.05}
]
const NEGATIVE_INDICATORS = [
{indicator: 'maybe', weight: -0.15},
{indicator: 'might', weight: -0.1},
{indicator: 'possibly', weight: -0.1},
{indicator: 'i think', weight: -0.2},
{indicator: 'unclear', weight: -0.1},
{indicator: '不确定', weight: -0.15},
{indicator: 'perhaps', weight: -0.1},
{indicator: 'could be', weight: -0.1},
{indicator: 'not sure', weight: -0.15},
{indicator: 'likely', weight: -0.05}
]
export class ConfidenceCalibrator {
private historicalAccuracies: Map<string, number[]> = new Map()
private feedbackHistory: Array<{
pattern: string
helpful: boolean
timestamp: number
}> = []
private readonly persistPath = '/tmp/prix_feedback.json'
private feedbackTracker: any = null
constructor(feedbackTracker: any = null) {
this.loadPersisted()
if (feedbackTracker) {
this.feedbackTracker = feedbackTracker
} else {
try {
const {FeedbackTracker} = require('./feedback-tracker')
this.feedbackTracker = new FeedbackTracker()
} catch (e) {
// FeedbackTracker not available, continue without it
}
}
}
private loadPersisted(): void {
try {
if (require('fs').existsSync(this.persistPath)) {
const data = JSON.parse(
require('fs').readFileSync(this.persistPath, 'utf8')
)
if (Array.isArray(data)) {
this.feedbackHistory = data.map((item: any) => ({
pattern: item.pattern || '',
helpful: item.helpful || false,
timestamp: item.timestamp || Date.now()
}))
this.rebuildHistoricalAccuracies()
}
}
} catch (e) {
// Start fresh if persistence fails
}
}
private persist(): void {
try {
require('fs').writeFileSync(
this.persistPath,
JSON.stringify(this.feedbackHistory.slice(-1000))
)
} catch (e) {
// Persistence failure is non-fatal
}
}
private rebuildHistoricalAccuracies(): void {
const patternMap = new Map<string, number[]>()
for (const entry of this.feedbackHistory) {
const accuracies = patternMap.get(entry.pattern) || []
accuracies.push(entry.helpful ? 1 : 0)
patternMap.set(entry.pattern, accuracies)
}
for (const [pattern, accuracies] of patternMap) {
this.historicalAccuracies.set(pattern, accuracies)
}
}
recordFeedback(pattern: string, helpful: boolean): void {
this.feedbackHistory.push({pattern, helpful, timestamp: Date.now()})
const accuracies = this.historicalAccuracies.get(pattern) || []
accuracies.push(helpful ? 1 : 0)
this.historicalAccuracies.set(pattern, accuracies)
this.persist()
}
calibrate(
aiReportedConfidence: number | undefined,
comment: string,
contextTokens: number,
maxContextTokens: number,
severity: string | undefined,
teamId?: string,
repoSlug?: string,
patternType?: string,
filename?: string
): CalibratedConfidence {
const factors = this.calculateFactors(
comment,
contextTokens,
maxContextTokens,
severity,
teamId,
repoSlug,
patternType,
filename
)
let score: number
if (aiReportedConfidence !== undefined) {
const deBiased = this.removeLLMBias(aiReportedConfidence)
score = this.combineWithFactors(deBiased, factors)
} else {
score = this.computeFromFactorsOnly(factors)
}
score = Math.max(0, Math.min(100, score))
return {
score: Math.round(score),
factors,
reasoning: this.generateReasoning(factors, score)
}
}
private calculateFactors(
comment: string,
contextTokens: number,
maxContextTokens: number,
severity: string | undefined,
teamId?: string,
repoSlug?: string,
patternType?: string,
filename?: string
): ConfidenceFactors {
const patternMatchScore = this.calculatePatternMatch(comment)
const contextRichness = Math.min(
1,
contextTokens / Math.max(1, maxContextTokens * 0.5)
)
const historicalAccuracy = this.calculateHistoricalAccuracy(comment)
const severityMatchesKnownBug = severity
? this.matchesKnownBugPattern(comment, severity)
: this.matchesAnyKnownBugPattern(comment)
const hasExplicitEvidence = this.hasExplicitEvidence(comment)
const {teamAcceptanceRate, patternTypeAccuracy} = this.calculateTeamMetrics(
teamId,
repoSlug,
patternType,
filename
)
return {
patternMatchScore,
contextRichness,
historicalAccuracy,
severityMatchesKnownBug,
hasExplicitEvidence,
teamAcceptanceRate,
patternTypeAccuracy
}
}
private calculateTeamMetrics(
teamId?: string,
repoSlug?: string,
patternType?: string,
filename?: string
): {teamAcceptanceRate: number; patternTypeAccuracy: number} {
if (!this.feedbackTracker || !teamId || !repoSlug) {
return {teamAcceptanceRate: 0.5, patternTypeAccuracy: 0.5}
}
let teamAcceptanceRate = 0.5
let patternTypeAccuracy = 0.5
if (patternType && filename) {
teamAcceptanceRate = this.feedbackTracker.getPatternAcceptanceRate(
patternType,
teamId,
repoSlug
)
patternTypeAccuracy = this.feedbackTracker.getFilenameAcceptanceRate(
filename,
teamId,
repoSlug
)
}
return {teamAcceptanceRate, patternTypeAccuracy}
}
private calculatePatternMatch(comment: string): number {
const lowerComment = comment.toLowerCase()
let score = 0.5
for (const {indicator, weight} of POSITIVE_INDICATORS) {
if (lowerComment.includes(indicator.toLowerCase())) {
score += weight
}
}
for (const {indicator, weight} of NEGATIVE_INDICATORS) {
if (lowerComment.includes(indicator.toLowerCase())) {
score += weight
}
}
if (lowerComment.includes('why this matters')) {
score += 0.1
}
if (
lowerComment.includes('production') ||
lowerComment.includes('production environment')
) {
score += 0.05
}
return Math.max(0, Math.min(1, score))
}
private calculateHistoricalAccuracy(comment: string): number {
const lowerComment = comment.toLowerCase()
for (const [pattern, accuracies] of this.historicalAccuracies) {
if (lowerComment.includes(pattern.toLowerCase())) {
const avg = accuracies.reduce((a, b) => a + b, 0) / accuracies.length
return avg
}
}
const relevantFeedback = this.feedbackHistory.filter(f =>
f.pattern
.split(' ')
.some(
word => word.length > 4 && lowerComment.includes(word.toLowerCase())
)
)
if (relevantFeedback.length === 0) {
return 0.5
}
const accuracy =
relevantFeedback.filter(f => f.helpful).length / relevantFeedback.length
return accuracy
}
private matchesKnownBugPattern(comment: string, severity: string): boolean {
const lowerComment = comment.toLowerCase()
for (const {pattern, severity: patternSeverity} of KNOWN_BUG_PATTERNS) {
if (pattern.test(lowerComment) && patternSeverity === severity) {
return true
}
}
return false
}
private matchesAnyKnownBugPattern(comment: string): boolean {
const lowerComment = comment.toLowerCase()
for (const {pattern} of KNOWN_BUG_PATTERNS) {
if (pattern.test(lowerComment)) {
return true
}
}
return false
}
private hasExplicitEvidence(comment: string): boolean {
const explicitPhrases = [
'this will crash',
'this causes',
'this leads to',
'results in',
'fails when',
'breaks when',
'error when',
'crash if',
'exception if',
'undefined behavior',
'undefined is',
'will throw',
'will fail'
]
const lowerComment = comment.toLowerCase()
return explicitPhrases.some(phrase => lowerComment.includes(phrase))
}
private removeLLMBias(reportedConfidence: number): number {
if (reportedConfidence >= 95) {
return 75
}
if (reportedConfidence >= 85) {
return 70
}
if (reportedConfidence >= 70) {
return 62
}
if (reportedConfidence >= 60) {
return 55
}
return reportedConfidence * 0.9
}
private combineWithFactors(
baseScore: number,
factors: ConfidenceFactors
): number {
const weights = {
base: 0.25,
pattern: 0.2,
context: 0.1,
historical: 0.15,
knownBug: 0.05,
teamAcceptance: 0.15,
patternAccuracy: 0.1
}
let score = baseScore * weights.base
score += factors.patternMatchScore * 100 * weights.pattern
score += factors.contextRichness * 100 * weights.context
score += factors.historicalAccuracy * 100 * weights.historical
if (factors.severityMatchesKnownBug) {
score += 10 * weights.knownBug
}
if (factors.hasExplicitEvidence) {
score += 5
}
score += factors.teamAcceptanceRate * 100 * weights.teamAcceptance
score += factors.patternTypeAccuracy * 100 * weights.patternAccuracy
return score
}
private computeFromFactorsOnly(factors: ConfidenceFactors): number {
const weights = {
pattern: 0.3,
context: 0.15,
historical: 0.2,
knownBug: 0.15,
teamAcceptance: 0.12,
patternAccuracy: 0.08
}
let score = 50
score += factors.patternMatchScore * 50 * weights.pattern
score += factors.contextRichness * 50 * weights.context
score += factors.historicalAccuracy * 50 * weights.historical
if (factors.severityMatchesKnownBug) {
score += 10 * weights.knownBug
}
if (factors.hasExplicitEvidence) {
score += 5
}
score += factors.teamAcceptanceRate * 50 * weights.teamAcceptance
score += factors.patternTypeAccuracy * 50 * weights.patternAccuracy
return score
}
private generateReasoning(factors: ConfidenceFactors, score: number): string {
const reasons: string[] = []
if (factors.patternMatchScore > 0.7) {
reasons.push('strong pattern match')
} else if (factors.patternMatchScore < 0.4) {
reasons.push('weak pattern match')
}
if (factors.contextRichness > 0.8) {
reasons.push('rich context')
} else if (factors.contextRichness < 0.3) {
reasons.push('limited context')
}
if (factors.historicalAccuracy > 0.7) {
reasons.push('historically accurate')
} else if (
factors.historicalAccuracy < 0.4 &&
factors.historicalAccuracy !== 0.5
) {
reasons.push('historically unreliable')
}
if (factors.severityMatchesKnownBug) {
reasons.push('matches known bug pattern')
}
if (factors.hasExplicitEvidence) {
reasons.push('has explicit evidence')
}
if (factors.teamAcceptanceRate > 0.6) {
reasons.push('high team acceptance')
} else if (
factors.teamAcceptanceRate < 0.3 &&
factors.teamAcceptanceRate !== 0.5
) {
reasons.push('low team acceptance')
}
if (factors.patternTypeAccuracy > 0.6) {
reasons.push('matches accepted patterns')
}
return reasons.length > 0
? `Calibrated based on: ${reasons.join(', ')}`
: 'Calibrated using default heuristics'
}
}
export const confidenceCalibrator = new ConfidenceCalibrator()