File size: 3,951 Bytes
38be44d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 | /**
* @license
* Copyright 2026 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import fs from 'node:fs';
import path from 'node:path';
import { execSync } from 'node:child_process';
import os from 'node:os';
/**
* Finds all report.json files recursively in a directory.
*/
export function findReports(dir) {
const reports = [];
if (!fs.existsSync(dir)) return reports;
const files = fs.readdirSync(dir);
for (const file of files) {
const fullPath = path.join(dir, file);
const stat = fs.statSync(fullPath);
if (stat.isDirectory()) {
reports.push(...findReports(fullPath));
} else if (file === 'report.json') {
reports.push(fullPath);
}
}
return reports;
}
/**
* Extracts the model name from the artifact path.
*/
export function getModelFromPath(reportPath) {
const parts = reportPath.split(path.sep);
// Look for the directory that follows the 'eval-logs-' pattern
const artifactDir = parts.find((p) => p.startsWith('eval-logs-'));
if (!artifactDir) return 'unknown';
const match = artifactDir.match(/^eval-logs-(.+)-(\d+)$/);
return match ? match[1] : 'unknown';
}
/**
* Escapes special characters in a string for use in a regular expression.
*/
export function escapeRegex(string) {
return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
}
/**
* Aggregates stats from a list of report.json files.
* @returns {Record<string, Record<string, {passed: number, total: number, file?: string}>>} statsByModel
*/
export function getStatsFromReports(reports) {
const statsByModel = {};
for (const reportPath of reports) {
try {
const model = getModelFromPath(reportPath);
if (!statsByModel[model]) {
statsByModel[model] = {};
}
const testStats = statsByModel[model];
const content = fs.readFileSync(reportPath, 'utf-8');
const json = JSON.parse(content);
for (const testResult of json.testResults) {
const filePath = testResult.name;
for (const assertion of testResult.assertionResults) {
const name = assertion.title;
if (!testStats[name]) {
testStats[name] = { passed: 0, total: 0, file: filePath };
}
testStats[name].total++;
if (assertion.status === 'passed') {
testStats[name].passed++;
}
}
}
} catch (error) {
console.error(`Error processing report at ${reportPath}:`, error.message);
}
}
return statsByModel;
}
/**
* Fetches historical nightly data using the GitHub CLI.
* @returns {Array<{runId: string, stats: Record<string, any>}>} history
*/
export function fetchNightlyHistory(lookbackCount) {
const history = [];
try {
const cmd = `gh run list --workflow evals-nightly.yml --branch main --limit ${
lookbackCount + 2
} --json databaseId,status`;
const runsJson = execSync(cmd, { encoding: 'utf-8' });
let runs = JSON.parse(runsJson);
// Filter for completed runs and take the top N
runs = runs.filter((r) => r.status === 'completed').slice(0, lookbackCount);
for (const run of runs) {
const tmpDir = fs.mkdtempSync(
path.join(os.tmpdir(), `gemini-evals-hist-${run.databaseId}-`),
);
try {
execSync(
`gh run download ${run.databaseId} -p "eval-logs-*" -D "${tmpDir}"`,
{ stdio: 'ignore' },
);
const runReports = findReports(tmpDir);
if (runReports.length > 0) {
history.push({
runId: run.databaseId,
stats: getStatsFromReports(runReports),
});
}
} catch (error) {
console.error(
`Failed to process artifacts for run ${run.databaseId}:`,
error.message,
);
} finally {
fs.rmSync(tmpDir, { recursive: true, force: true });
}
}
} catch (error) {
console.error('Failed to fetch history:', error.message);
}
return history;
}
|