Upload 3 files

GPQA-mini/benchlocal.pack.json

@@ -0,0 +1,18 @@
+{
+  "schemaVersion": 1,
+  "protocolVersion": 1,
+  "id": "GPQA-mini",
+  "name": "GPQA-Polyglot (45 задач STEM уровня PhD)",
+  "version": "1.0.0",
+  "description": "Сложный Google-Proof бенчмарк уровня аспирантуры (10 вариантов ответов A-J, случайный порог 10%): 15 RU, 15 EN и 15 MIX задач",
+  "entry": "./index.mjs",
+  "samplingDefaults": {
+    "temperature": 0.0
+  },
+  "capabilities": {
+    "tools": false,
+    "multiTurn": false,
+    "streamingProgress": false,
+    "verification": false
+  }
+}

GPQA-mini/index.mjs

@@ -0,0 +1,231 @@
+import fs from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+const manifestPath = fs.existsSync(path.join(__dirname, "benchlocal.pack.json"))
+  ? path.join(__dirname, "benchlocal.pack.json")
+  : path.join(__dirname, "benchlocal.pack.txt");
+
+const scenariosPath = fs.existsSync(path.join(__dirname, "scenarios.json"))
+  ? path.join(__dirname, "scenarios.json")
+  : path.join(__dirname, "scenarios.txt");
+
+function readJsonSafe(filePath, fallback) {
+  try {
+    if (fs.existsSync(filePath)) {
+      const raw = fs.readFileSync(filePath, "utf8").replace(/^\uFEFF/, "").trim();
+      return JSON.parse(raw);
+    }
+  } catch (err) {
+    console.error(`[GPQA-Polyglot] Ошибка чтения ${filePath}:`, err.message);
+  }
+  return fallback;
+}
+
+const defaultManifest = {
+  schemaVersion: 1,
+  protocolVersion: 1,
+  id: "GPQA-mini",
+  name: "GPQA-Polyglot (45 задач STEM уровня PhD)",
+  version: "1.0.0",
+  description: "Сложный Google-Proof бенчмарк уровня аспирантуры (10 вариантов ответов A-J, случайный порог 10%): 15 RU, 15 EN и 15 MIX задач",
+  entry: "./index.mjs", // ИСПРАВЛЕНО (было: entry": "./index.mjs",)
+  samplingDefaults: { temperature: 0.0 },
+  capabilities: { tools: false, multiTurn: false, streamingProgress: false, verification: false }
+};
+
+export const manifest = readJsonSafe(manifestPath, defaultManifest);
+
+function loadScenarios() {
+  return readJsonSafe(scenariosPath, []);
+}
+
+export function listScenarios() {
+  return loadScenarios().map(s => ({
+    id: s.id,
+    name: s.name,
+    category: s.category || "Общие",
+    description: s.name
+  }));
+}
+
+export function getScenarios() {
+  return listScenarios();
+}
+
+/**
+ * Валидатор извлечения ответа:
+ * 1. Ищет конструкцию "Final Answer: (X)" или "Final Answer: X".
+ * 2. Если префикс опущен, ищет последнюю пару скобок с буквой: (X) или [X].
+ */
+export function extractAnswer(rawText) {
+  const text = (rawText || "").trim();
+  if (!text) return null;
+
+  // Поддерживает как вариант со скобками: (A), так и без них: Final Answer: A
+  const pattern = /(?:final\s*(?:verdict|answer)|итоговый\s*(?:ответ|вердикт)|ответ)[\s:=]*[\[\(]?\s*([A-J])\s*[\]\)]?/gi;
+  const explicitMatches = Array.from(text.matchAll(pattern));
+  if (explicitMatches.length > 0) {
+    return explicitMatches[explicitMatches.length - 1][1].toUpperCase();
+  }
+
+  const bracketMatches = Array.from(text.matchAll(/[\[\(]([A-J])[\]\)]/g));
+  if (bracketMatches.length > 0) {
+    return bracketMatches[bracketMatches.length - 1][1].toUpperCase();
+  }
+
+  return null;
+}
+
+function resolveHelpers(context) {
+  if (context && typeof context.getRequiredProvider === "function") {
+    return context;
+  }
+  if (context?.helpers && typeof context.helpers.getRequiredProvider === "function") {
+    return context.helpers;
+  }
+  return {
+    getRequiredProvider(providerId) {
+      const providers = context?.providers || [];
+      const found = Array.isArray(providers)
+        ? providers.find((p) => p.id === providerId || p.provider === providerId)
+        : providers[providerId];
+      return found || { baseUrl: "http://127.0.0.1:11434/v1", apiKey: "none" };
+    },
+    getInferenceEndpoint(modelId) {
+      return context?.endpoints?.[modelId] || null;
+    }
+  };
+}
+
+export async function prepare(context) {
+  const helpers = resolveHelpers(context);
+
+  return {
+    async runScenario(input) {
+      const allScenarios = loadScenarios();
+      const scenario = allScenarios.find(s => s.id === input.scenario.id);
+      if (!scenario) {
+        return {
+          scenarioId: input.scenario.id,
+          status: "fail",
+          score: 0,
+          summary: "Сценарий не найден",
+          rawLog: "error=Scenario not found"
+        };
+      }
+
+      const provider = helpers.getRequiredProvider(input.model.provider, { enabledOnly: true });
+      const inference = helpers.getInferenceEndpoint ? helpers.getInferenceEndpoint(input.model.id) : null;
+
+      const rawBaseUrl = (inference && inference.status === "running" ? inference.baseUrl : provider?.baseUrl) || "http://127.0.0.1:11434/v1";
+      const baseUrl = rawBaseUrl.replace(/\/+$/, "");
+      const apiKey = provider?.apiKey || "none";
+
+      const requestBody = {
+        model: input.model.model,
+        messages: [{ role: "user", content: scenario.prompt }],
+        temperature: 0,
+        max_tokens: 4096
+      };
+
+      let text = "";
+      let rawResponseData = null;
+
+      try {
+        const response = await fetch(`${baseUrl}/chat/completions`, {
+          method: "POST",
+          headers: {
+            "Content-Type": "application/json",
+            Authorization: `Bearer ${apiKey}`
+          },
+          body: JSON.stringify(requestBody)
+        });
+
+        if (!response.ok) {
+          throw new Error(`HTTP ${response.status} ${response.statusText}`);
+        }
+
+        rawResponseData = await response.json();
+        text = (rawResponseData.choices?.[0]?.message?.content || "").trim();
+      } catch (err) {
+        return {
+          scenarioId: scenario.id,
+          status: "fail",
+          score: 0,
+          summary: `Ошибка API: ${err.message}`,
+          rawLog: `error=${err.message}`
+        };
+      }
+
+      const extracted = extractAnswer(text);
+      const passed = extracted === scenario.correct_answer.toUpperCase();
+
+      const rawLog = [
+        `model=${input.model.model}`,
+        `scenario=${scenario.id} ${scenario.name}`,
+        `category=${scenario.category}`,
+        `user_turn_1=${scenario.prompt}`,
+        `assistant_turn_1=${text}`,
+        `extracted_choice=${extracted || "NONE"}`,
+        `expected_choice=${scenario.correct_answer}`,
+        `verdict=${passed ? "pass" : "fail"}`,
+        `score=${passed ? 100 : 0}`,
+        `summary=${passed ? "Правильный ответ" : `Неверно (выбрано: ${extracted || 'ничего'}, ожидалось: ${scenario.correct_answer})`}`,
+        `\n--- RAW API RESPONSE ---`,
+        JSON.stringify(rawResponseData, null, 2)
+      ].join("\n");
+
+      return {
+        scenarioId: scenario.id,
+        status: passed ? "pass" : "fail",
+        score: passed ? 100 : 0,
+        summary: passed ? "Правильный ответ" : `Выбрано '${extracted || "нет"}', ожидалось '${scenario.correct_answer}'`,
+        rawLog: rawLog
+      };
+    },
+
+    async dispose() {}
+  };
+}
+
+export function scoreModelResults(results) {
+  const scored = results || [];
+  const allScenarios = loadScenarios();
+  const scenarioMap = new Map(allScenarios.map(s => [s.id, s]));
+
+  const total = scored.reduce((sum, r) => sum + (r.score || 0), 0);
+  const max = scored.length * 100;
+
+  const categoryStats = {};
+  for (const r of scored) {
+    const s = scenarioMap.get(r.scenarioId);
+    const cat = s?.category || "Общие";
+    if (!categoryStats[cat]) {
+      categoryStats[cat] = { total: 0, count: 0 };
+    }
+    categoryStats[cat].total += (r.score || 0);
+    categoryStats[cat].count += 1;
+  }
+
+  const categories = Object.entries(categoryStats).map(([name, stat]) => ({
+    name,
+    score: stat.count > 0 ? Math.round((stat.total / (stat.count * 100)) * 100) : 0
+  }));
+
+  return {
+    totalScore: max > 0 ? Math.round((total / max) * 100) : 0,
+    categories
+  };
+}
+
+export default {
+  manifest,
+  listScenarios,
+  getScenarios,
+  prepare,
+  createRuntime: prepare,
+  scoreModelResults
+};