| import fs from "node:fs";
|
| import path from "node:path";
|
| import { fileURLToPath } from "node:url";
|
|
|
| const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
| const manifestPath = path.join(__dirname, "benchlocal.pack.json");
|
| const scenariosPath = path.join(__dirname, "scenarios.json");
|
|
|
| export const manifest = JSON.parse(fs.readFileSync(manifestPath, "utf8"));
|
|
|
| function loadScenarios() {
|
| return JSON.parse(fs.readFileSync(scenariosPath, "utf8"));
|
| }
|
|
|
| export function listScenarios() {
|
| return loadScenarios().map((s) => ({
|
| id: s.id,
|
| name: s.name,
|
| category: s.category || "Общие",
|
| description: `[${(s.language || "ru").toUpperCase()}] ${s.name}`
|
| }));
|
| }
|
|
|
| export function getScenarios() {
|
| return listScenarios();
|
| }
|
|
|
| function normalizeStr(s) {
|
| return String(s || "")
|
| .trim()
|
| .toLowerCase()
|
| .replace(/[«»"'`]/g, "")
|
| .replace(/\s+/g, " ");
|
| }
|
|
|
| |
| |
|
|
| function formatExpectedAnswer(scenario) {
|
| if (scenario.category === "murder_mysteries") {
|
| return [
|
| `Подозреваемый: (${scenario.expectedSuspect})`,
|
| `Орудие преступления: (${scenario.expectedWeapon})`,
|
| `Фактическое место: (${scenario.expectedLocation})`,
|
| `Финальная строка: Мой ответ: (${scenario.expectedSuspect}), (${scenario.expectedWeapon}), (${scenario.expectedLocation})`
|
| ].join("\n");
|
| }
|
|
|
| if (scenario.category === "object_placements") {
|
| const lines = ["Ожидания персонажей (Theory of Mind):"];
|
| const sortedBeliefs = Object.entries(scenario.expectedBeliefs || {}).sort(([a], [b]) =>
|
| a.localeCompare(b)
|
| );
|
| for (const [charLetter, locNum] of sortedBeliefs) {
|
| lines.push(` (${charLetter}) ожидает найти в: ${locNum}`);
|
| }
|
| lines.push(`Фактическое физическое местоположение: (${scenario.expectedFinalLocation})`);
|
| lines.push(`Эталонный блок ответа:`);
|
| lines.push(`Мой ответ:`);
|
| for (const [charLetter, locNum] of sortedBeliefs) {
|
| lines.push(`(${charLetter}) - ${locNum}`);
|
| }
|
| lines.push(`(${scenario.expectedFinalLocation})`);
|
| return lines.join("\n");
|
| }
|
|
|
| if (scenario.category === "team_allocation") {
|
| const sorted = [...(scenario.expectedTeam || [])].map((x) => x.toUpperCase()).sort();
|
| return [
|
| `Отобранная тройка специалистов: (${sorted.join("), (")})`,
|
| `Финальная строка: Мой ответ: (${sorted.join("), (")})`
|
| ].join("\n");
|
| }
|
|
|
| return "Не определено";
|
| }
|
|
|
| |
| |
|
|
| function validateResponse(scenario, rawText) {
|
| if (!rawText || !rawText.trim()) {
|
| return {
|
| pass: false,
|
| score: 0,
|
| reason: "Модель вернула пустой ответ.",
|
| expected: formatExpectedAnswer(scenario)
|
| };
|
| }
|
|
|
| const text = rawText.trim();
|
| const category = scenario.category;
|
|
|
|
|
| if (category === "murder_mysteries") {
|
| const match = text.match(
|
| /(?:мой ответ|my answer)\s*:\s*\(?([А-ЯA-Z])\)?\s*,\s*\(?(\d+)\)?\s*,\s*\(?([^)\n\r]+)\)?/i
|
| );
|
|
|
| const expSuspect = scenario.expectedSuspect.toUpperCase();
|
| const expWeapon = String(scenario.expectedWeapon).trim();
|
| const expLocation = normalizeStr(scenario.expectedLocation);
|
| const expectedStr = `Мой ответ: (${expSuspect}), (${expWeapon}), (${expLocation})`;
|
|
|
| if (!match) {
|
| return {
|
| pass: false,
|
| score: 0,
|
| reason: `Не найден маркерный формат строки: 'Мой ответ: (Буква), (Цифра), (место)'. Ожидалось: "${expectedStr}".`,
|
| expected: formatExpectedAnswer(scenario)
|
| };
|
| }
|
|
|
| const suspect = match[1].toUpperCase();
|
| const weapon = String(match[2]).trim();
|
| const location = normalizeStr(match[3]);
|
|
|
| const okS = suspect === expSuspect;
|
| const okW = weapon === expWeapon;
|
| const okL = location === expLocation;
|
|
|
| if (okS && okW && okL) {
|
| return {
|
| pass: true,
|
| score: 100,
|
| reason: `Дело раскрыто безошибочно: виновен (${suspect}), орудие (${weapon}), место (${location}).`,
|
| expected: formatExpectedAnswer(scenario)
|
| };
|
| }
|
|
|
| const diffs = [];
|
| if (!okS) diffs.push(`Убийца: указан (${suspect}), верный (${expSuspect})`);
|
| if (!okW) diffs.push(`Орудие: указано (${weapon}), верное (${expWeapon})`);
|
| if (!okL) diffs.push(`Место: указано (${location}), верное (${expLocation})`);
|
|
|
| return {
|
| pass: false,
|
| score: 0,
|
| reason: `Ошибки в дедукции: ${diffs.join("; ")}. Правильный ответ: "${expectedStr}".`,
|
| expected: formatExpectedAnswer(scenario)
|
| };
|
| }
|
|
|
|
|
| if (category === "object_placements") {
|
| const markerIdx = text.search(/(?:мой ответ|my answer)\s*:/i);
|
| if (markerIdx === -1) {
|
| return {
|
| pass: false,
|
| score: 0,
|
| reason: "Отсутствует обязательный заголовок блока 'Мой ответ:' или 'My answer:'.",
|
| expected: formatExpectedAnswer(scenario)
|
| };
|
| }
|
|
|
| const block = text.slice(markerIdx);
|
| const beliefsMatches = [...block.matchAll(/\(([А-ЯA-Z])\)\s*[-–—]\s*(\d+)/gi)];
|
| const finalMatches = [...block.matchAll(/^\s*\((\d+)\)\s*$/gm)];
|
|
|
| if (beliefsMatches.length === 0 || finalMatches.length === 0) {
|
| return {
|
| pass: false,
|
| score: 0,
|
| reason:
|
| "Нарушен построчный синтаксис персонажей '(Буква) - Номер' либо отсутствует финальная строка '([номер])'.",
|
| expected: formatExpectedAnswer(scenario)
|
| };
|
| }
|
|
|
| const extractedBeliefs = {};
|
| for (const m of beliefsMatches) {
|
| extractedBeliefs[m[1].toUpperCase()] = Number(m[2]);
|
| }
|
|
|
| const extractedFinal = Number(finalMatches[finalMatches.length - 1][1]);
|
| const expectedBeliefs = scenario.expectedBeliefs;
|
| const expectedFinal = Number(scenario.expectedFinalLocation);
|
|
|
| const diffs = [];
|
| for (const [charLetter, expLoc] of Object.entries(expectedBeliefs)) {
|
| const actLoc = extractedBeliefs[charLetter.toUpperCase()];
|
| if (actLoc !== expLoc) {
|
| diffs.push(`(${charLetter}): получено ${actLoc ?? "не указано"}, должно быть ${expLoc}`);
|
| }
|
| }
|
|
|
| if (extractedFinal !== expectedFinal) {
|
| diffs.push(`Реальное положение: получено (${extractedFinal}), должно быть (${expectedFinal})`);
|
| }
|
|
|
| if (diffs.length === 0) {
|
| return {
|
| pass: true,
|
| score: 100,
|
| reason: "Все когнитивные ожидания персонажей (ToM) и физическая реальность вычислены верно.",
|
| expected: formatExpectedAnswer(scenario)
|
| };
|
| }
|
|
|
| return {
|
| pass: false,
|
| score: 0,
|
| reason: `Ошибки в Theory of Mind: ${diffs.join("; ")}.`,
|
| expected: formatExpectedAnswer(scenario)
|
| };
|
| }
|
|
|
|
|
| if (category === "team_allocation") {
|
| const match = text.match(/(?:мой ответ|my answer)\s*:\s*([^\n\r]+)/i);
|
| const expectedTeam = [...scenario.expectedTeam].map((x) => x.toUpperCase()).sort();
|
| const expectedStr = `Мой ответ: (${expectedTeam.join("), (")})`;
|
|
|
| if (!match) {
|
| return {
|
| pass: false,
|
| score: 0,
|
| reason: `Не найдена завершающая строка 'Мой ответ: (Буква 1), (Буква 2), (Буква 3)'. Ожидалось: "${expectedStr}".`,
|
| expected: formatExpectedAnswer(scenario)
|
| };
|
| }
|
|
|
| const rawMembers = [...match[1].matchAll(/\(([А-ЯA-Z])\)/gi)].map((m) => m[1].toUpperCase());
|
|
|
| const isAlphabetical = rawMembers
|
| .slice()
|
| .every((val, i, arr) => !i || arr[i - 1].localeCompare(val) <= 0);
|
| if (!isAlphabetical) {
|
| return {
|
| pass: false,
|
| score: 0,
|
| reason: `Кандидаты не упорядочены по алфавиту: [${rawMembers.join(", ")}]. Ожидалось: "${expectedStr}".`,
|
| expected: formatExpectedAnswer(scenario)
|
| };
|
| }
|
|
|
| const sortedRaw = rawMembers.slice().sort();
|
| const isMatch =
|
| sortedRaw.length === expectedTeam.length &&
|
| sortedRaw.every((val, idx) => val === expectedTeam[idx]);
|
|
|
| if (isMatch) {
|
| return {
|
| pass: true,
|
| score: 100,
|
| reason: `Команда сформирована верно: (${expectedTeam.join("), (")}).`,
|
| expected: formatExpectedAnswer(scenario)
|
| };
|
| }
|
|
|
| return {
|
| pass: false,
|
| score: 0,
|
| reason: `Неверный состав: получено [${rawMembers.join(", ")}], ожидалось: [${expectedTeam.join(", ")}].`,
|
| expected: formatExpectedAnswer(scenario)
|
| };
|
| }
|
|
|
| return {
|
| pass: false,
|
| score: 0,
|
| reason: "Неизвестная категория сценария.",
|
| expected: "Неизвестно"
|
| };
|
| }
|
|
|
| function resolveHelpers(context) {
|
| if (context && typeof context.getRequiredProvider === "function") return context;
|
| if (context?.helpers && typeof context.helpers.getRequiredProvider === "function")
|
| return context.helpers;
|
| return {
|
| getRequiredProvider(providerId) {
|
| const providers = context?.providers || [];
|
| const found = Array.isArray(providers)
|
| ? providers.find((p) => p.id === providerId || p.provider === providerId)
|
| : providers[providerId];
|
| return found || { baseUrl: "http://127.0.0.1:11434/v1", apiKey: "none" };
|
| },
|
| getInferenceEndpoint(modelId) {
|
| return context?.endpoints?.[modelId] || null;
|
| }
|
| };
|
| }
|
|
|
| export async function prepare(context) {
|
| const helpers = resolveHelpers(context);
|
|
|
| return {
|
| async runScenario(input) {
|
| const allScenarios = loadScenarios();
|
| const scenario = allScenarios.find((s) => s.id === input.scenario.id);
|
| if (!scenario) {
|
| return {
|
| scenarioId: input.scenario.id,
|
| status: "fail",
|
| score: 0,
|
| summary: "Сценарий не найден",
|
| rawLog: "error=Scenario not found"
|
| };
|
| }
|
|
|
| const provider = helpers.getRequiredProvider(input.model.provider, { enabledOnly: true });
|
| const inference = helpers.getInferenceEndpoint
|
| ? helpers.getInferenceEndpoint(input.model.id)
|
| : null;
|
|
|
| const rawBaseUrl =
|
| (inference && inference.status === "running" ? inference.baseUrl : provider?.baseUrl) ||
|
| "http://127.0.0.1:11434/v1";
|
| const baseUrl = rawBaseUrl.replace(/\/+$/, "");
|
| const apiKey = provider?.apiKey || "none";
|
|
|
| const requestBody = {
|
| model: input.model.model,
|
| messages: [{ role: "user", content: scenario.prompt }],
|
| temperature: 0
|
| };
|
|
|
| const response = await fetch(`${baseUrl}/chat/completions`, {
|
| method: "POST",
|
| headers: {
|
| "Content-Type": "application/json",
|
| Authorization: `Bearer ${apiKey}`
|
| },
|
| body: JSON.stringify(requestBody)
|
| });
|
|
|
| const data = await response.json();
|
| const text = (data.choices?.[0]?.message?.content || "").trim();
|
| const result = validateResponse(scenario, text);
|
|
|
|
|
| const rawLog = [
|
| `============================================================`,
|
| `СЦЕНАРИЙ: ${scenario.id} — ${scenario.name}`,
|
| `КАТЕГОРИЯ: ${scenario.category} | ЯЗЫК: ${scenario.language}`,
|
| `МОДЕЛЬ: ${input.model.model}`,
|
| `============================================================`,
|
| ``,
|
| `=== ИСХОДНАЯ ЗАДАЧА (PROMPT) ===`,
|
| scenario.prompt,
|
| ``,
|
| `=== ЭТАЛОННЫЙ ПРАВИЛЬНЫЙ ОТВЕТ ===`,
|
| result.expected,
|
| ``,
|
| `=== ОТВЕТ АССИСТЕНТА ===`,
|
| text || "<МОДЕЛЬ ВЕРНУЛА ПУСТОЙ ОТВЕТ>",
|
| ``,
|
| `=== ВЕРДИКТ И ПРОВЕРКА ===`,
|
| `Статус: ${result.pass ? "ВЕРНО (PASS)" : "НЕВЕРНО (FAIL)"}`,
|
| `Балл: ${result.score}/100`,
|
| `Подробности проверки: ${result.reason}`,
|
| ``,
|
| `--- СТАНДАРТНЫЕ МЕТКИ ДЛЯ BENCHLOCAL ---`,
|
| `model=${input.model.model}`,
|
| `scenario=${scenario.id} ${scenario.name}`,
|
| `category=${scenario.category}`,
|
| `language=${scenario.language}`,
|
| `verdict=${result.pass ? "pass" : "fail"}`,
|
| `score=${result.score}`,
|
| `summary=${result.reason}`,
|
| `\n--- RAW API RESPONSE (JSON) ---`,
|
| JSON.stringify(data, null, 2)
|
| ].join("\n");
|
|
|
| return {
|
| scenarioId: scenario.id,
|
| status: result.pass ? "pass" : "fail",
|
| score: result.score,
|
| summary: result.reason,
|
| rawLog
|
| };
|
| },
|
|
|
| async dispose() {}
|
| };
|
| }
|
|
|
| export function scoreModelResults(results) {
|
| const scored = results || [];
|
| const total = scored.reduce((sum, r) => sum + (r.score || 0), 0);
|
| const max = scored.length * 100;
|
| const allScenarios = loadScenarios();
|
|
|
| const categoryNames = {
|
| murder_mysteries: "Детективы (Murder Mysteries - Triplet)",
|
| object_placements: "Перемещение объектов (Object Placements - ToM)",
|
| team_allocation: "Формирование команд (Team Allocation - Psych)"
|
| };
|
|
|
| const categories = Object.keys(categoryNames).map((catKey) => {
|
| const subsetIds = new Set(allScenarios.filter((s) => s.category === catKey).map((s) => s.id));
|
| const catResults = scored.filter((r) => subsetIds.has(r.scenarioId));
|
| const catScore = catResults.reduce((sum, r) => sum + (r.score || 0), 0);
|
| const catMax = catResults.length * 100;
|
| return {
|
| id: catKey,
|
| name: categoryNames[catKey],
|
| score: catMax > 0 ? Math.round((catScore / catMax) * 100) : 0
|
| };
|
| });
|
|
|
| return {
|
| totalScore: max > 0 ? Math.round((total / max) * 100) : 0,
|
| categories
|
| };
|
| }
|
|
|
| export default {
|
| manifest,
|
| listScenarios,
|
| getScenarios,
|
| prepare,
|
| createRuntime: prepare,
|
| scoreModelResults
|
| }; |
