PRIX / src /feedback-tracker.ts
Rachit-Tw's picture
Upload 169 files
9284ad7 verified
Raw
History Blame Contribute Delete
13.2 kB
import {writeFileSync, readFileSync, existsSync, mkdirSync} from 'fs'
import {join} from 'path'
import {minimatch} from 'minimatch'
import {
PatternFeedback,
PatternStats,
TeamPatternInsights,
FeedbackQuery,
FeedbackSummary,
FeedbackType,
PatternCategory
} from './feedback-types'
const info = console.log
function getPersistBaseDir(): string {
if (process.env.PRIX_DATA_DIR) return process.env.PRIX_DATA_DIR
if (process.env.HF_HOME) return join(process.env.HF_HOME, 'prix-data')
if (process.env.HUGGING_FACE_HUB_TOKEN) return '/tmp/prix-hf'
return '/tmp/prix'
}
export class FeedbackTracker {
private feedbackStore: PatternFeedback[] = []
private teamInsights: Map<string, TeamPatternInsights> = new Map()
private readonly persistPath: string
private readonly maxStoreSize = 10000
private initialized = false
constructor(persistDir?: string) {
const baseDir = persistDir || getPersistBaseDir()
this.persistPath = join(baseDir, 'feedback')
this.ensureStorageDir()
this.loadPersisted()
}
private ensureStorageDir(): void {
try {
if (!existsSync(this.persistPath)) {
mkdirSync(this.persistPath, {recursive: true})
}
} catch (e) {
info(`Failed to create feedback storage dir: ${e}`)
}
}
private loadPersisted(): void {
try {
const feedbackFile = join(this.persistPath, 'feedback.json')
if (existsSync(feedbackFile)) {
const data = JSON.parse(readFileSync(feedbackFile, 'utf8'))
if (Array.isArray(data)) {
this.feedbackStore = data.slice(-this.maxStoreSize)
this.rebuildTeamInsights()
this.initialized = true
info(`Loaded ${this.feedbackStore.length} feedback entries`)
}
}
} catch (e) {
info(`Failed to load feedback data: ${e}`)
}
}
private persist(): void {
try {
const feedbackFile = join(this.persistPath, 'feedback.json')
writeFileSync(
feedbackFile,
JSON.stringify(this.feedbackStore.slice(-this.maxStoreSize))
)
} catch (e) {
info(`Failed to persist feedback: ${e}`)
}
}
private rebuildTeamInsights(): void {
const teamMap = new Map<string, PatternFeedback[]>()
for (const fb of this.feedbackStore) {
const key = `${fb.teamId}:${fb.repoSlug}`
if (!teamMap.has(key)) {
teamMap.set(key, [])
}
teamMap.get(key)!.push(fb)
}
for (const [key, feedbacks] of teamMap) {
const [teamId, repoSlug] = key.split(':')
const insights = this.computeInsights(teamId, repoSlug, feedbacks)
this.teamInsights.set(key, insights)
}
}
private computeInsights(
teamId: string,
repoSlug: string,
feedbacks: PatternFeedback[]
): TeamPatternInsights {
const patternMap = new Map<PatternCategory, PatternFeedback[]>()
for (const fb of feedbacks) {
if (!patternMap.has(fb.patternType)) {
patternMap.set(fb.patternType, [])
}
patternMap.get(fb.patternType)!.push(fb)
}
const patternStats = new Map<PatternCategory, PatternStats>()
for (const [patternType, pFeedbacks] of patternMap) {
const accepted = pFeedbacks.filter(
f => f.feedbackType === 'accepted'
).length
const rejected = pFeedbacks.filter(
f => f.feedbackType === 'rejected'
).length
const ignored = pFeedbacks.filter(
f => f.feedbackType === 'ignored'
).length
const total = pFeedbacks.length
const avgConfidence =
pFeedbacks.reduce((sum, f) => sum + f.confidence, 0) / total
patternStats.set(patternType, {
patternType,
total,
accepted,
rejected,
ignored,
acceptanceRate: total > 0 ? accepted / total : 0,
avgConfidence
})
}
const allAccepted = feedbacks.filter(
f => f.feedbackType === 'accepted'
).length
const overallAcceptanceRate =
feedbacks.length > 0 ? allAccepted / feedbacks.length : 0
return {
teamId,
repoSlug,
patternStats,
overallAcceptanceRate,
lastUpdated: Date.now()
}
}
recordFeedback(feedback: Omit<PatternFeedback, 'id' | 'timestamp'>): string {
const id = `fb_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`
const fullFeedback: PatternFeedback = {
...feedback,
id,
timestamp: Date.now()
}
this.feedbackStore.push(fullFeedback)
if (this.feedbackStore.length > this.maxStoreSize) {
this.feedbackStore = this.feedbackStore.slice(-this.maxStoreSize)
}
const key = `${feedback.teamId}:${feedback.repoSlug}`
const existing = this.teamInsights.get(key)
if (existing) {
const updatedFeedbacks = [
...this.feedbackStore.filter(
f => f.teamId === feedback.teamId && f.repoSlug === feedback.repoSlug
)
]
const insights = this.computeInsights(
feedback.teamId,
feedback.repoSlug,
updatedFeedbacks
)
this.teamInsights.set(key, insights)
}
this.persist()
return id
}
recordAcceptance(
patternType: PatternCategory,
severity: 'critical' | 'major' | 'minor' | 'info',
filename: string,
repoSlug: string,
teamId: string,
confidence: number,
commentId?: string,
reviewId?: string
): string {
return this.recordFeedback({
patternType,
severity,
filenamePattern: filename,
repoSlug,
teamId,
confidence,
feedbackType: 'accepted',
commentId,
reviewId
})
}
recordRejection(
patternType: PatternCategory,
severity: 'critical' | 'major' | 'minor' | 'info',
filename: string,
repoSlug: string,
teamId: string,
confidence: number,
commentId?: string,
reviewId?: string
): string {
return this.recordFeedback({
patternType,
severity,
filenamePattern: filename,
repoSlug,
teamId,
confidence,
feedbackType: 'rejected',
commentId,
reviewId
})
}
recordIgnored(
patternType: PatternCategory,
severity: 'critical' | 'major' | 'minor' | 'info',
filename: string,
repoSlug: string,
teamId: string,
confidence: number,
commentId?: string,
reviewId?: string
): string {
return this.recordFeedback({
patternType,
severity,
filenamePattern: filename,
repoSlug,
teamId,
confidence,
feedbackType: 'ignored',
commentId,
reviewId
})
}
getPatternAcceptanceRate(
patternType: PatternCategory,
teamId: string,
repoSlug: string
): number {
const key = `${teamId}:${repoSlug}`
const insights = this.teamInsights.get(key)
if (!insights) return 0.5
const stats = insights.patternStats.get(patternType)
if (!stats) return 0.5
if (stats.total < 5) return 0.5
return stats.acceptanceRate
}
getFilenameAcceptanceRate(
filenamePattern: string,
teamId: string,
repoSlug: string
): number {
const feedbacks = this.feedbackStore.filter(
f =>
f.teamId === teamId &&
f.repoSlug === repoSlug &&
f.feedbackType !== 'ignored'
)
if (feedbacks.length === 0) return 0.5
const matching = feedbacks.filter(
f =>
minimatch(f.filenamePattern, filenamePattern) ||
minimatch(filenamePattern, f.filenamePattern)
)
if (matching.length === 0) return 0.5
const accepted = matching.filter(f => f.feedbackType === 'accepted').length
return accepted / matching.length
}
getTeamInsights(
teamId: string,
repoSlug: string
): TeamPatternInsights | null {
const key = `${teamId}:${repoSlug}`
return this.teamInsights.get(key) || null
}
getAdjustedConfidence(
baseConfidence: number,
patternType: PatternCategory,
filename: string,
teamId: string,
repoSlug: string
): number {
const patternRate = this.getPatternAcceptanceRate(
patternType,
teamId,
repoSlug
)
const filenameRate = this.getFilenameAcceptanceRate(
filename,
teamId,
repoSlug
)
const feedbackWeight = 0.3
const teamRate = patternRate * 0.6 + filenameRate * 0.4
const adjustedConfidence =
baseConfidence * (1 - feedbackWeight) + teamRate * 100 * feedbackWeight
return Math.max(0, Math.min(100, adjustedConfidence))
}
shouldSuppressPattern(
patternType: PatternCategory,
teamId: string,
repoSlug: string,
threshold = 0.2,
minSamples = 10
): boolean {
const key = `${teamId}:${repoSlug}`
const insights = this.teamInsights.get(key)
if (!insights) return false
const stats = insights.patternStats.get(patternType)
if (!stats || stats.total < minSamples) return false
return stats.acceptanceRate < threshold
}
getQuery(query: FeedbackQuery): PatternFeedback[] {
let results = [...this.feedbackStore]
if (query.teamId) {
results = results.filter(f => f.teamId === query.teamId)
}
if (query.repoSlug) {
results = results.filter(f => f.repoSlug === query.repoSlug)
}
if (query.patternType) {
results = results.filter(f => f.patternType === query.patternType)
}
if (query.filenamePattern) {
results = results.filter(f =>
minimatch(f.filenamePattern, query.filenamePattern!)
)
}
if (query.since) {
results = results.filter(f => f.timestamp >= query.since!)
}
if (query.limit) {
results = results.slice(-query.limit)
}
return results
}
getSummary(teamId?: string, repoSlug?: string): FeedbackSummary {
let feedbacks = [...this.feedbackStore]
if (teamId) {
feedbacks = feedbacks.filter(f => f.teamId === teamId)
}
if (repoSlug) {
feedbacks = feedbacks.filter(f => f.repoSlug === repoSlug)
}
const patternMap = new Map<PatternCategory, PatternFeedback[]>()
const repoMap = new Map<string, number>()
const teamMap = new Map<string, number>()
for (const fb of feedbacks) {
if (!patternMap.has(fb.patternType)) {
patternMap.set(fb.patternType, [])
}
patternMap.get(fb.patternType)!.push(fb)
repoMap.set(fb.repoSlug, (repoMap.get(fb.repoSlug) || 0) + 1)
teamMap.set(fb.teamId, (teamMap.get(fb.teamId) || 0) + 1)
}
const byPattern = new Map<PatternCategory, PatternStats>()
for (const [patternType, pFeedbacks] of patternMap) {
const accepted = pFeedbacks.filter(
f => f.feedbackType === 'accepted'
).length
const rejected = pFeedbacks.filter(
f => f.feedbackType === 'rejected'
).length
const ignored = pFeedbacks.filter(
f => f.feedbackType === 'ignored'
).length
const total = pFeedbacks.length
const avgConfidence =
pFeedbacks.reduce((sum, f) => sum + f.confidence, 0) / total
byPattern.set(patternType, {
patternType,
total,
accepted,
rejected,
ignored,
acceptanceRate: total > 0 ? accepted / total : 0,
avgConfidence
})
}
const recentWindow = 30 * 24 * 60 * 60 * 1000
const recentFeedbacks = feedbacks.filter(
f => f.timestamp > Date.now() - recentWindow
)
const olderFeedbacks = feedbacks.filter(
f => f.timestamp <= Date.now() - recentWindow
)
const recentAcceptance =
recentFeedbacks.filter(f => f.feedbackType === 'accepted').length /
(recentFeedbacks.length || 1)
const olderAcceptance =
olderFeedbacks.filter(f => f.feedbackType === 'accepted').length /
(olderFeedbacks.length || 1)
let recentTrend: 'improving' | 'declining' | 'stable' = 'stable'
if (recentAcceptance > olderAcceptance + 0.05) {
recentTrend = 'improving'
} else if (recentAcceptance < olderAcceptance - 0.05) {
recentTrend = 'declining'
}
return {
totalFeedback: feedbacks.length,
byPattern,
byRepo: repoMap,
byTeam: teamMap,
recentTrend
}
}
exportForTraining(): Array<{
patternType: PatternCategory
severity: 'critical' | 'major' | 'minor' | 'info'
filenamePattern: string
confidence: number
accepted: boolean
repoSlug: string
}> {
return this.feedbackStore
.filter(f => f.feedbackType !== 'ignored')
.map(f => ({
patternType: f.patternType,
severity: f.severity,
filenamePattern: f.filenamePattern,
confidence: f.confidence,
accepted: f.feedbackType === 'accepted',
repoSlug: f.repoSlug
}))
}
}
export const feedbackTracker = new FeedbackTracker()