rai-bench / src /lib /parseGuardrails.ts
rohanjaggi
fix: remove guardrails accuracy
2b01b44
Raw
History Blame Contribute Delete
1.76 kB
import fs from 'fs'
import path from 'path'
import type { GuardrailData, GuardrailThresholds } from './types'
function pct(val: string): number | null {
const s = val?.trim()
if (!s) return null
const n = parseFloat(s.replace('%', ''))
return isNaN(n) ? null : n / 100
}
export function loadGuardrails(): GuardrailData[] {
const filePath = path.join(process.cwd(), 'public', 'guardrails_data.csv')
if (!fs.existsSync(filePath)) return []
const text = fs.readFileSync(filePath, 'utf-8')
const lines = text.split('\n').filter(l => l.trim()).slice(1)
return lines
.map(line => {
const cols = line.split(',')
return {
rank: parseInt(cols[0]) || 0,
creator: cols[1]?.trim() || '',
guardrail: cols[2]?.trim() || '',
recall: pct(cols[3]),
precision: pct(cols[4]),
f1: pct(cols[5]),
detections: cols[7]?.trim() || '',
lastRun: cols[8]?.trim() || '',
}
})
.filter(g => g.creator && g.guardrail)
}
function empiricalPercentile(vals: number[], p: number): number {
const sorted = [...vals].sort((a, b) => a - b)
const idx = (p / 100) * (sorted.length - 1)
const lo = Math.floor(idx)
const hi = Math.ceil(idx)
return sorted[lo] + (idx - lo) * (sorted[hi] - sorted[lo])
}
export function computeGuardrailThresholds(guardrails: GuardrailData[]): GuardrailThresholds {
const recalls = guardrails.map(g => g.recall).filter((v): v is number => v !== null)
const f1s = guardrails.map(g => g.f1).filter((v): v is number => v !== null)
return {
recall: { p33: empiricalPercentile(recalls, 33), p67: empiricalPercentile(recalls, 67) },
f1: { p33: empiricalPercentile(f1s, 33), p67: empiricalPercentile(f1s, 67) },
}
}