Spaces:
Running
Running
SilverElixir commited on
Commit ·
eeb83cf
1
Parent(s): d0d695d
Persona eval: probe-tools dispatch mode + grounding metadata report
Browse files- .github/workflows/persona-eval.yml +13 -1
- eval/persona_models.py +5 -1
.github/workflows/persona-eval.yml
CHANGED
|
@@ -8,10 +8,18 @@ name: Persona eval
|
|
| 8 |
on:
|
| 9 |
workflow_dispatch:
|
| 10 |
inputs:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 11 |
models:
|
| 12 |
description: "Модели через запятую: provider:model (provider: gemini|gemma|groq|or)"
|
| 13 |
required: false
|
| 14 |
-
default: "gemma:gemma-4-31b-it,gemma:gemma-4-26b-a4b-it,gemini:gemini-3.5-flash-lite,or:
|
| 15 |
judge:
|
| 16 |
description: "Модель-судья (provider:model)"
|
| 17 |
required: false
|
|
@@ -49,6 +57,10 @@ jobs:
|
|
| 49 |
OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
|
| 50 |
run: |
|
| 51 |
set -euo pipefail
|
|
|
|
|
|
|
|
|
|
|
|
|
| 52 |
if [ -n "${{ inputs.only }}" ]; then
|
| 53 |
python eval/persona_models.py run \
|
| 54 |
--models "${{ inputs.models }}" \
|
|
|
|
| 8 |
on:
|
| 9 |
workflow_dispatch:
|
| 10 |
inputs:
|
| 11 |
+
mode:
|
| 12 |
+
description: "run (замер) или probe-tools (проверка поиска у Gemma)"
|
| 13 |
+
required: false
|
| 14 |
+
default: "run"
|
| 15 |
+
model:
|
| 16 |
+
description: "модель для probe-tools (без префикса провайдера)"
|
| 17 |
+
required: false
|
| 18 |
+
default: "gemma-4-31b-it"
|
| 19 |
models:
|
| 20 |
description: "Модели через запятую: provider:model (provider: gemini|gemma|groq|or)"
|
| 21 |
required: false
|
| 22 |
+
default: "gemma:gemma-4-31b-it,gemma:gemma-4-26b-a4b-it,gemini:gemini-3.5-flash-lite,or:nvidia/nemotron-3.5-lightning:free"
|
| 23 |
judge:
|
| 24 |
description: "Модель-судья (provider:model)"
|
| 25 |
required: false
|
|
|
|
| 57 |
OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
|
| 58 |
run: |
|
| 59 |
set -euo pipefail
|
| 60 |
+
if [ "${{ inputs.mode }}" = "probe-tools" ]; then
|
| 61 |
+
python eval/persona_models.py probe-tools --model "${{ inputs.model }}"
|
| 62 |
+
exit 0
|
| 63 |
+
fi
|
| 64 |
if [ -n "${{ inputs.only }}" ]; then
|
| 65 |
python eval/persona_models.py run \
|
| 66 |
--models "${{ inputs.models }}" \
|
eval/persona_models.py
CHANGED
|
@@ -631,7 +631,11 @@ def cmd_probe_tools(args: argparse.Namespace) -> int:
|
|
| 631 |
config = types.GenerateContentConfig(system_instruction=get_system_prompt(model), tools=tools) if tools else types.GenerateContentConfig(system_instruction=get_system_prompt(model))
|
| 632 |
resp = client.models.generate_content(model=model, contents=contents, config=config)
|
| 633 |
answer = (resp.text or "").strip()
|
| 634 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 635 |
except Exception as exc:
|
| 636 |
log(f"{model} {label}: ОШИБКА {exc}")
|
| 637 |
return 0
|
|
|
|
| 631 |
config = types.GenerateContentConfig(system_instruction=get_system_prompt(model), tools=tools) if tools else types.GenerateContentConfig(system_instruction=get_system_prompt(model))
|
| 632 |
resp = client.models.generate_content(model=model, contents=contents, config=config)
|
| 633 |
answer = (resp.text or "").strip()
|
| 634 |
+
gm = None
|
| 635 |
+
cands = resp.candidates or []
|
| 636 |
+
if cands:
|
| 637 |
+
gm = getattr(cands[0], "grounding_metadata", None)
|
| 638 |
+
log(f"{model} {label}: ответ {len(answer)} симв., grounding {'ЕСТЬ' if gm else 'нет'} — {answer[:160]!r}")
|
| 639 |
except Exception as exc:
|
| 640 |
log(f"{model} {label}: ОШИБКА {exc}")
|
| 641 |
return 0
|