| import fs from "node:fs";
|
| import path from "node:path";
|
| import { fileURLToPath } from "node:url";
|
|
|
| const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
|
| const manifestPath = fs.existsSync(path.join(__dirname, "benchlocal.pack.json"))
|
| ? path.join(__dirname, "benchlocal.pack.json")
|
| : path.join(__dirname, "benchlocal.pack.txt");
|
|
|
| const scenariosPath = fs.existsSync(path.join(__dirname, "scenarios.json"))
|
| ? path.join(__dirname, "scenarios.json")
|
| : path.join(__dirname, "scenarios.txt");
|
|
|
| function readJsonSafe(filePath, fallback) {
|
| try {
|
| if (fs.existsSync(filePath)) {
|
| const raw = fs.readFileSync(filePath, "utf8").replace(/^\uFEFF/, "").trim();
|
| return JSON.parse(raw);
|
| }
|
| } catch (err) {
|
| console.error(`[GPQA-Polyglot] Ошибка чтения ${filePath}:`, err.message);
|
| }
|
| return fallback;
|
| }
|
|
|
| const defaultManifest = {
|
| schemaVersion: 1,
|
| protocolVersion: 1,
|
| id: "GPQA-mini",
|
| name: "GPQA-Polyglot (45 задач STEM уровня PhD)",
|
| version: "1.0.0",
|
| description: "Сложный Google-Proof бенчмарк уровня аспирантуры (10 вариантов ответов A-J, случайный порог 10%): 15 RU, 15 EN и 15 MIX задач",
|
| entry: "./index.mjs",
|
| samplingDefaults: { temperature: 0.0 },
|
| capabilities: { tools: false, multiTurn: false, streamingProgress: false, verification: false }
|
| };
|
|
|
| export const manifest = readJsonSafe(manifestPath, defaultManifest);
|
|
|
| function loadScenarios() {
|
| return readJsonSafe(scenariosPath, []);
|
| }
|
|
|
| export function listScenarios() {
|
| return loadScenarios().map(s => ({
|
| id: s.id,
|
| name: s.name,
|
| category: s.category || "Общие",
|
| description: s.name
|
| }));
|
| }
|
|
|
| export function getScenarios() {
|
| return listScenarios();
|
| }
|
|
|
| |
| |
| |
| |
|
|
| export function extractAnswer(rawText) {
|
| const text = (rawText || "").trim();
|
| if (!text) return null;
|
|
|
|
|
| const pattern = /(?:final\s*(?:verdict|answer)|итоговый\s*(?:ответ|вердикт)|ответ)[\s:=]*[\[\(]?\s*([A-J])\s*[\]\)]?/gi;
|
| const explicitMatches = Array.from(text.matchAll(pattern));
|
| if (explicitMatches.length > 0) {
|
| return explicitMatches[explicitMatches.length - 1][1].toUpperCase();
|
| }
|
|
|
| const bracketMatches = Array.from(text.matchAll(/[\[\(]([A-J])[\]\)]/g));
|
| if (bracketMatches.length > 0) {
|
| return bracketMatches[bracketMatches.length - 1][1].toUpperCase();
|
| }
|
|
|
| return null;
|
| }
|
|
|
| function resolveHelpers(context) {
|
| if (context && typeof context.getRequiredProvider === "function") {
|
| return context;
|
| }
|
| if (context?.helpers && typeof context.helpers.getRequiredProvider === "function") {
|
| return context.helpers;
|
| }
|
| return {
|
| getRequiredProvider(providerId) {
|
| const providers = context?.providers || [];
|
| const found = Array.isArray(providers)
|
| ? providers.find((p) => p.id === providerId || p.provider === providerId)
|
| : providers[providerId];
|
| return found || { baseUrl: "http://127.0.0.1:11434/v1", apiKey: "none" };
|
| },
|
| getInferenceEndpoint(modelId) {
|
| return context?.endpoints?.[modelId] || null;
|
| }
|
| };
|
| }
|
|
|
| export async function prepare(context) {
|
| const helpers = resolveHelpers(context);
|
|
|
| return {
|
| async runScenario(input) {
|
| const allScenarios = loadScenarios();
|
| const scenario = allScenarios.find(s => s.id === input.scenario.id);
|
| if (!scenario) {
|
| return {
|
| scenarioId: input.scenario.id,
|
| status: "fail",
|
| score: 0,
|
| summary: "Сценарий не найден",
|
| rawLog: "error=Scenario not found"
|
| };
|
| }
|
|
|
| const provider = helpers.getRequiredProvider(input.model.provider, { enabledOnly: true });
|
| const inference = helpers.getInferenceEndpoint ? helpers.getInferenceEndpoint(input.model.id) : null;
|
|
|
| const rawBaseUrl = (inference && inference.status === "running" ? inference.baseUrl : provider?.baseUrl) || "http://127.0.0.1:11434/v1";
|
| const baseUrl = rawBaseUrl.replace(/\/+$/, "");
|
| const apiKey = provider?.apiKey || "none";
|
|
|
| const requestBody = {
|
| model: input.model.model,
|
| messages: [{ role: "user", content: scenario.prompt }],
|
| temperature: 0,
|
| max_tokens: 4096
|
| };
|
|
|
| let text = "";
|
| let rawResponseData = null;
|
|
|
| try {
|
| const response = await fetch(`${baseUrl}/chat/completions`, {
|
| method: "POST",
|
| headers: {
|
| "Content-Type": "application/json",
|
| Authorization: `Bearer ${apiKey}`
|
| },
|
| body: JSON.stringify(requestBody)
|
| });
|
|
|
| if (!response.ok) {
|
| throw new Error(`HTTP ${response.status} ${response.statusText}`);
|
| }
|
|
|
| rawResponseData = await response.json();
|
| text = (rawResponseData.choices?.[0]?.message?.content || "").trim();
|
| } catch (err) {
|
| return {
|
| scenarioId: scenario.id,
|
| status: "fail",
|
| score: 0,
|
| summary: `Ошибка API: ${err.message}`,
|
| rawLog: `error=${err.message}`
|
| };
|
| }
|
|
|
| const extracted = extractAnswer(text);
|
| const passed = extracted === scenario.correct_answer.toUpperCase();
|
|
|
| const rawLog = [
|
| `model=${input.model.model}`,
|
| `scenario=${scenario.id} ${scenario.name}`,
|
| `category=${scenario.category}`,
|
| `user_turn_1=${scenario.prompt}`,
|
| `assistant_turn_1=${text}`,
|
| `extracted_choice=${extracted || "NONE"}`,
|
| `expected_choice=${scenario.correct_answer}`,
|
| `verdict=${passed ? "pass" : "fail"}`,
|
| `score=${passed ? 100 : 0}`,
|
| `summary=${passed ? "Правильный ответ" : `Неверно (выбрано: ${extracted || 'ничего'}, ожидалось: ${scenario.correct_answer})`}`,
|
| `\n--- RAW API RESPONSE ---`,
|
| JSON.stringify(rawResponseData, null, 2)
|
| ].join("\n");
|
|
|
| return {
|
| scenarioId: scenario.id,
|
| status: passed ? "pass" : "fail",
|
| score: passed ? 100 : 0,
|
| summary: passed ? "Правильный ответ" : `Выбрано '${extracted || "нет"}', ожидалось '${scenario.correct_answer}'`,
|
| rawLog: rawLog
|
| };
|
| },
|
|
|
| async dispose() {}
|
| };
|
| }
|
|
|
| export function scoreModelResults(results) {
|
| const scored = results || [];
|
| const allScenarios = loadScenarios();
|
| const scenarioMap = new Map(allScenarios.map(s => [s.id, s]));
|
|
|
| const total = scored.reduce((sum, r) => sum + (r.score || 0), 0);
|
| const max = scored.length * 100;
|
|
|
| const categoryStats = {};
|
| for (const r of scored) {
|
| const s = scenarioMap.get(r.scenarioId);
|
| const cat = s?.category || "Общие";
|
| if (!categoryStats[cat]) {
|
| categoryStats[cat] = { total: 0, count: 0 };
|
| }
|
| categoryStats[cat].total += (r.score || 0);
|
| categoryStats[cat].count += 1;
|
| }
|
|
|
| const categories = Object.entries(categoryStats).map(([name, stat]) => ({
|
| name,
|
| score: stat.count > 0 ? Math.round((stat.total / (stat.count * 100)) * 100) : 0
|
| }));
|
|
|
| return {
|
| totalScore: max > 0 ? Math.round((total / max) * 100) : 0,
|
| categories
|
| };
|
| }
|
|
|
| export default {
|
| manifest,
|
| listScenarios,
|
| getScenarios,
|
| prepare,
|
| createRuntime: prepare,
|
| scoreModelResults
|
| }; |
