Upload 3 files

ifeval-mini/benchlocal.pack.json

@@ -0,0 +1,18 @@
+{
+  "schemaVersion": 1,
+  "protocolVersion": 1,
+  "id": "ifeval-hard",
+  "name": "IFEval-Hard (90 комплексных задач)",
+  "version": "1.0.0",
+  "description": "Аутентичный сложный IFEval с мульти-композициями: 30 RU, 30 EN и 30 Code-Switching задач (по 2-3 строгих ограничения на промпт)",
+  "entry": "./index.mjs",
+  "samplingDefaults": {
+    "temperature": 0.0
+  },
+  "capabilities": {
+    "tools": false,
+    "multiTurn": false,
+    "streamingProgress": false,
+    "verification": false
+  }
+}

ifeval-mini/index.mjs

@@ -0,0 +1,406 @@
+import fs from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const manifestPath = path.join(__dirname, "benchlocal.pack.json");
+const scenariosPath = path.join(__dirname, "scenarios.json");
+
+function readJsonSafe(filePath, fallback) {
+  try {
+    if (fs.existsSync(filePath)) {
+      const raw = fs.readFileSync(filePath, "utf8").replace(/^\uFEFF/, "").trim();
+      return JSON.parse(raw);
+    }
+  } catch (err) {
+    console.error(`[IFEval-Hard] Ошибка чтения ${filePath}:`, err.message);
+  }
+  return fallback;
+}
+
+const defaultManifest = {
+  schemaVersion: 1,
+  protocolVersion: 1,
+  id: "ifeval-hard",
+  name: "IFEval-Hard (90 комплексных задач)",
+  version: "1.0.0",
+  description: "Аутентичный сложный IFEval с мульти-композициями: 30 RU, 30 EN и 30 Code-Switching задач (по 2-3 строгих ограничения на промпт)",
+  entry: "./index.mjs",
+  samplingDefaults: { temperature: 0.0 },
+  capabilities: { tools: false, multiTurn: false, streamingProgress: false, verification: false }
+};
+
+export const manifest = readJsonSafe(manifestPath, defaultManifest);
+
+function loadScenarios() {
+  return readJsonSafe(scenariosPath, []);
+}
+
+export function listScenarios() {
+  return loadScenarios().map(s => ({
+    id: s.id,
+    name: s.name,
+    category: s.category || "Общие",
+    description: s.name
+  }));
+}
+
+export function getScenarios() {
+  return listScenarios();
+}
+
+function validateSingleRule(type, args, text) {
+  const clean = (text || "").trim();
+  const lower = clean.toLowerCase();
+
+  switch (type) {
+    case "code_switching": {
+      const words = clean.split(/\s+/).filter(w => /[\p{L}\p{N}]/u.test(w));
+      const ruWords = words.filter(w => /[а-яА-ЯёЁ]/.test(w)).length;
+      const enWords = words.filter(w => /[a-zA-Z]/.test(w)).length;
+      const minRu = args.minRu || 5;
+      const minEn = args.minEn || 5;
+
+      if (ruWords < minRu) {
+        return { pass: false, reason: `Мало русских слов: ${ruWords} (надо минимум ${minRu})` };
+      }
+      if (enWords < minEn) {
+        return { pass: false, reason: `Мало английских слов: ${enWords} (надо минимум ${minEn})` };
+      }
+      return { pass: true, reason: `Code-switching OK (${ruWords} RU / ${enWords} EN слов)` };
+    }
+
+    case "include_keywords": {
+      const missing = (args.keywords || []).filter(kw => !lower.includes(kw.toLowerCase()));
+      return {
+        pass: missing.length === 0,
+        reason: missing.length === 0 ? "Все ключевые слова найдены" : `Нет слов: ${missing.join(", ")}`
+      };
+    }
+
+    case "forbidden_words": {
+      const found = (args.words || []).filter(w => lower.includes(w.toLowerCase()));
+      return {
+        pass: found.length === 0,
+        reason: found.length === 0 ? "Запрещенных слов нет" : `Найдены запрещенные слова: ${found.join(", ")}`
+      };
+    }
+
+    case "keyword_frequency": {
+      const escaped = args.word.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+      const re = new RegExp(`(?<!\\p{L})${escaped}(?!\\p{L})`, "gui");
+      const count = (lower.match(re) || []).length;
+      return {
+        pass: count === args.count,
+        reason: count === args.count ? `Слово '${args.word}' x${count}` : `Слово '${args.word}' x${count} (требовалось ${args.count})`
+      };
+    }
+
+    case "letter_frequency": {
+      const char = args.letter.toLowerCase();
+      const count = lower.split(char).length - 1;
+      return {
+        pass: count === args.count,
+        reason: count === args.count ? `Буква '${char}' x${count}` : `Буква '${char}' x${count} (требовалось ${args.count})`
+      };
+    }
+
+    case "response_language": {
+      if (args.lang === "ru") {
+        const pass = !/[a-zA-Z]/.test(clean) && /[а-яА-ЯёЁ]/.test(clean);
+        return { pass, reason: pass ? "Строго русский язык без латиницы" : "Обнаружена латиница" };
+      }
+      const samples = {
+        es: ["de", "la", "el", "en"],
+        fr: ["le", "la", "du", "est"],
+        de: ["und", "der", "die", "das"],
+        it: ["di", "per", "con", "il"]
+      }[args.lang] || [];
+      const pass = samples.some(s => lower.includes(s));
+      return { pass, reason: pass ? `Язык (${args.lang}) подтвержден` : `Маркеры (${args.lang}) не найдены` };
+    }
+
+    case "number_words": {
+      const words = clean.split(/\s+/).filter(w => /[\p{L}\p{N}]/u.test(w));
+      const count = words.length;
+      if (args.exact) return { pass: count === args.exact, reason: `Слов: ${count} (надо ровно ${args.exact})` };
+      if (args.min && count < args.min) return { pass: false, reason: `Слов: ${count} (мин. ${args.min})` };
+      if (args.max && count > args.max) return { pass: false, reason: `Слов: ${count} (макс. ${args.max})` };
+      return { pass: true, reason: `Слов в норме (${count})` };
+    }
+
+    case "number_sentences": {
+      const normalized = clean.replace(/P\.S\./gi, "PS");
+      const count = normalized.split(/[.!?]+/).filter(s => s.trim().length > 0).length;
+      return { pass: count === args.count, reason: `Предложений: ${count} (надо ровно ${args.count})` };
+    }
+
+    case "number_paragraphs": {
+      const count = clean.split(/\n\s*\n/).filter(p => p.trim().length > 0).length;
+      return { pass: count === args.count, reason: `Абзацев: ${count} (надо ровно ${args.count})` };
+    }
+
+    case "paragraph_first_word": {
+      const paras = clean.split(/\n\s*\n/).filter(p => p.trim().length > 0);
+      const target = paras[args.paragraphIndex];
+      if (!target) return { pass: false, reason: `Абзац ${args.paragraphIndex + 1} отсутствует` };
+      const firstWord = target.split(/\s+/)[0].replace(/^[^\p{L}\p{N}]+/u, "").replace(/[^\p{L}\p{N}]+$/u, "");
+      const pass = firstWord.toLowerCase() === args.word.toLowerCase();
+      return { pass, reason: pass ? `Первое слово: '${firstWord}'` : `Первое слово: '${firstWord}' (ожидалось '${args.word}')` };
+    }
+
+    case "number_bullet_lists": {
+      const bullets = clean.split("\n").filter(line => /^\s*([-*•]|\d+[.)])\s+/.test(line));
+      return { pass: bullets.length === args.count, reason: `Пунктов списка: ${bullets.length} (надо ${args.count})` };
+    }
+
+    case "title": {
+      const pass = /<<[^>]+>>/.test(clean);
+      return { pass, reason: pass ? "Заголовок <<...>> найден" : "Заголовок <<...>> не найден" };
+    }
+
+    case "constrained_response": {
+      const pass = clean.toLowerCase() === (args.expected || "").toLowerCase();
+      return { pass, reason: pass ? "Точный ответ" : `Получено '${clean}' (ожидалось '${args.expected}')` };
+    }
+
+    case "highlighted_section": {
+      const matches = clean.match(/(?<!\*)\*(?!\*)([^*\n]+?)(?<!\*)\*(?!\*)/g) || [];
+      const count = matches.length;
+      if (args.exact) return { pass: count === args.exact, reason: `Выделений звездочками: ${count} (надо ${args.exact})` };
+      const min = args.min || 1;
+      return { pass: count >= min, reason: `Выделений звездочками: ${count} (мин. ${min})` };
+    }
+
+    case "multiple_sections": {
+      const matches = clean.match(/(?:^|\n)\s*(?:[#*_\->•]|\d+[.)]|\s)*\bSection\s+\d+\b/gi) || [];
+      return { pass: matches.length === args.count, reason: `Секций Section X: ${matches.length} (надо ${args.count})` };
+    }
+
+    case "json_format": {
+      try {
+        const c = clean.replace(/^```json\s*/i, "").replace(/^```\s*/i, "").replace(/```$/, "").trim();
+        const parsed = JSON.parse(c);
+        if (args.isArray && !Array.isArray(parsed)) return { pass: false, reason: "Не массив JSON" };
+        if (Array.isArray(args.requiredKeys)) {
+          const obj = Array.isArray(parsed) ? parsed[0] : parsed;
+          if (!obj || typeof obj !== "object") return { pass: false, reason: "Некорректный объект JSON" };
+          for (const k of args.requiredKeys) {
+            if (!(k in obj)) return { pass: false, reason: `Нет ключа '${k}' в JSON` };
+          }
+        }
+        return { pass: true, reason: "Валидный JSON с нужными ключами" };
+      } catch {
+        return { pass: false, reason: "Не является валидным JSON" };
+      }
+    }
+
+    case "number_of_placeholders": {
+      const matches = clean.match(/\[[^\]\n]+\]/g) || [];
+      const count = matches.length;
+      if (args.exact) return { pass: count === args.exact, reason: `Плейсхолдеров [...]: ${count} (надо ${args.exact})` };
+      const min = args.min || 1;
+      return { pass: count >= min, reason: `Плейсхолдеров [...]: ${count} (мин. ${min})` };
+    }
+
+    case "postscript": {
+      const pass = /(?:\n|^)\s*P\.S\..*$/is.test(clean);
+      return { pass, reason: pass ? "P.S. найден в конце" : "P.S. в конце отсутствует" };
+    }
+
+    case "repeat_prompt": {
+      const promptClean = (args.prompt || "").trim().toLowerCase();
+      const pass = lower.startsWith(promptClean);
+      return { pass, reason: pass ? "Запрос повторен дословно" : "Запрос не повторен дословно в начале" };
+    }
+
+    case "two_responses": {
+      const parts = clean.split("******");
+      const pass = parts.length === 2 && parts[0].trim().length > 0 && parts[1].trim().length > 0;
+      return { pass, reason: pass ? "Два ответа через ******" : "Разделитель ****** отсутствует" };
+    }
+
+    case "all_uppercase": {
+      const hasL = /\p{L}/u.test(clean);
+      const pass = hasL && clean === clean.toUpperCase();
+      return { pass, reason: pass ? "Все ЗАГЛАВНЫМИ БУКВАМИ" : "Найдены строчные буквы" };
+    }
+
+    case "all_lowercase": {
+      const hasL = /\p{L}/u.test(clean);
+      const pass = hasL && clean === clean.toLowerCase();
+      return { pass, reason: pass ? "Все строчными буквами" : "Найдены заглавные буквы" };
+    }
+
+    case "capital_word_frequency": {
+      const rawWords = clean.split(/\s+/).filter(Boolean);
+      const capWords = [];
+      for (let i = 0; i < rawWords.length; i++) {
+        const w = rawWords[i];
+        const letters = w.replace(/[^\p{L}]/gu, "");
+        if (!letters || letters !== letters.toUpperCase()) continue;
+        if (letters.length >= 2) capWords.push(letters);
+        else if (letters.length === 1) {
+          const prev = i > 0 ? rawWords[i - 1] : "";
+          const isStart = i === 0 || /[.!?][)"'»*]*$/.test(prev);
+          const next = i < rawWords.length - 1 ? rawWords[i + 1] : "";
+          if (isStart && /\p{Ll}/u.test(next)) continue;
+          capWords.push(letters);
+        }
+      }
+      return { pass: capWords.length === args.count, reason: `Слов капсом: ${capWords.length} (надо ${args.count})` };
+    }
+
+    case "end_checker": {
+      const pass = clean.endsWith(args.phrase);
+      return { pass, reason: pass ? `Фраза в конце: '${args.phrase}'` : `Нет фразы '${args.phrase}' в конце` };
+    }
+
+    case "quotation": {
+      const pass = (clean.startsWith('"') && clean.endsWith('"')) || (clean.startsWith("«") && clean.endsWith("»"));
+      return { pass, reason: pass ? "Весь текст в кавычках" : "Текст не заключен в кавычки целиком" };
+    }
+
+    case "no_comma": {
+      const pass = !clean.includes(",") && !clean.includes("，");
+      return { pass, reason: pass ? "Без запятых" : "Найдена запрещенная запятая" };
+    }
+
+    default:
+      return { pass: true, reason: "Неизвестный тип валидатора" };
+  }
+}
+
+function validateResponse(scenario, rawText) {
+  const validators = scenario.validators || (scenario.validatorType ? [{ type: scenario.validatorType, args: scenario.validatorArgs }] : []);
+  if (validators.length === 0) return { pass: true, reason: "Нет валидаторов" };
+
+  const results = validators.map(v => {
+    const vArgs = { ...(v.args || {}) };
+    if (v.type === "repeat_prompt") {
+      vArgs.prompt = scenario.prompt;
+    }
+    return {
+      type: v.type,
+      res: validateSingleRule(v.type, vArgs, rawText)
+    };
+  });
+
+  const allPass = results.every(r => r.res.pass);
+  const summary = results.map(r => `[${r.res.pass ? "OK" : "FAIL"}: ${r.type} -> ${r.res.reason}]`).join(" ");
+
+  return {
+    pass: allPass,
+    reason: summary
+  };
+}
+
+function resolveHelpers(context) {
+  if (context && typeof context.getRequiredProvider === "function") {
+    return context;
+  }
+  if (context?.helpers && typeof context.helpers.getRequiredProvider === "function") {
+    return context.helpers;
+  }
+  return {
+    getRequiredProvider(providerId) {
+      const providers = context?.providers || [];
+      const found = Array.isArray(providers)
+        ? providers.find((p) => p.id === providerId || p.provider === providerId)
+        : providers[providerId];
+      return found || { baseUrl: "http://127.0.0.1:11434/v1", apiKey: "none" };
+    },
+    getInferenceEndpoint(modelId) {
+      return context?.endpoints?.[modelId] || null;
+    }
+  };
+}
+
+export async function prepare(context) {
+  const helpers = resolveHelpers(context);
+
+  return {
+    async runScenario(input) {
+      const allScenarios = loadScenarios();
+      const scenario = allScenarios.find(s => s.id === input.scenario.id);
+      if (!scenario) {
+        return {
+          scenarioId: input.scenario.id,
+          status: "fail",
+          score: 0,
+          summary: "Сценарий не найден",
+          rawLog: "error=Scenario not found"
+        };
+      }
+
+      const provider = helpers.getRequiredProvider(input.model.provider, { enabledOnly: true });
+      const inference = helpers.getInferenceEndpoint ? helpers.getInferenceEndpoint(input.model.id) : null;
+
+      const rawBaseUrl = (inference && inference.status === "running" ? inference.baseUrl : provider?.baseUrl) || "http://127.0.0.1:11434/v1";
+      const baseUrl = rawBaseUrl.replace(/\/+$/, "");
+      const apiKey = provider?.apiKey || "none";
+
+      const requestBody = {
+        model: input.model.model,
+        messages: [{ role: "user", content: scenario.prompt }],
+        temperature: 0
+      };
+
+      const response = await fetch(`${baseUrl}/chat/completions`, {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
+          Authorization: `Bearer ${apiKey}`
+        },
+        body: JSON.stringify(requestBody)
+      });
+
+      const data = await response.json();
+      const text = (data.choices?.[0]?.message?.content || "").trim();
+      const result = validateResponse(scenario, text);
+
+      const rawLog = [
+        `model=${input.model.model}`,
+        `scenario=${scenario.id} ${scenario.name}`,
+        `user_turn_1=${scenario.prompt}`,
+        `assistant_turn_1=${text}`,
+        `final_answer=${text}`,
+        `verdict=${result.pass ? "pass" : "fail"}`,
+        `score=${result.pass ? 100 : 0}`,
+        `summary=${result.reason}`,
+        `\n--- RAW API RESPONSE ---`,
+        JSON.stringify(data, null, 2)
+      ].join("\n");
+
+      return {
+        scenarioId: scenario.id,
+        status: result.pass ? "pass" : "fail",
+        score: result.pass ? 100 : 0,
+        summary: result.reason,
+        rawLog: rawLog
+      };
+    },
+
+    async dispose() {}
+  };
+}
+
+export function scoreModelResults(results) {
+  const scored = results || [];
+  const total = scored.reduce((sum, r) => sum + (r.score || 0), 0);
+  const max = scored.length * 100;
+
+  return {
+    totalScore: max > 0 ? Math.round((total / max) * 100) : 0,
+    categories: []
+  };
+}
+
+export default {
+  manifest,
+  listScenarios,
+  getScenarios,
+  prepare,
+  createRuntime: prepare,
+  scoreModelResults
+};