SilverElixir commited on
Commit
eeb83cf
·
1 Parent(s): d0d695d

Persona eval: probe-tools dispatch mode + grounding metadata report

Browse files
.github/workflows/persona-eval.yml CHANGED
@@ -8,10 +8,18 @@ name: Persona eval
8
  on:
9
  workflow_dispatch:
10
  inputs:
 
 
 
 
 
 
 
 
11
  models:
12
  description: "Модели через запятую: provider:model (provider: gemini|gemma|groq|or)"
13
  required: false
14
- default: "gemma:gemma-4-31b-it,gemma:gemma-4-26b-a4b-it,gemini:gemini-3.5-flash-lite,or:inclusionai/ling-3.0-flash-sante:free"
15
  judge:
16
  description: "Модель-судья (provider:model)"
17
  required: false
@@ -49,6 +57,10 @@ jobs:
49
  OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
50
  run: |
51
  set -euo pipefail
 
 
 
 
52
  if [ -n "${{ inputs.only }}" ]; then
53
  python eval/persona_models.py run \
54
  --models "${{ inputs.models }}" \
 
8
  on:
9
  workflow_dispatch:
10
  inputs:
11
+ mode:
12
+ description: "run (замер) или probe-tools (проверка поиска у Gemma)"
13
+ required: false
14
+ default: "run"
15
+ model:
16
+ description: "модель для probe-tools (без префикса провайдера)"
17
+ required: false
18
+ default: "gemma-4-31b-it"
19
  models:
20
  description: "Модели через запятую: provider:model (provider: gemini|gemma|groq|or)"
21
  required: false
22
+ default: "gemma:gemma-4-31b-it,gemma:gemma-4-26b-a4b-it,gemini:gemini-3.5-flash-lite,or:nvidia/nemotron-3.5-lightning:free"
23
  judge:
24
  description: "Модель-судья (provider:model)"
25
  required: false
 
57
  OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
58
  run: |
59
  set -euo pipefail
60
+ if [ "${{ inputs.mode }}" = "probe-tools" ]; then
61
+ python eval/persona_models.py probe-tools --model "${{ inputs.model }}"
62
+ exit 0
63
+ fi
64
  if [ -n "${{ inputs.only }}" ]; then
65
  python eval/persona_models.py run \
66
  --models "${{ inputs.models }}" \
eval/persona_models.py CHANGED
@@ -631,7 +631,11 @@ def cmd_probe_tools(args: argparse.Namespace) -> int:
631
  config = types.GenerateContentConfig(system_instruction=get_system_prompt(model), tools=tools) if tools else types.GenerateContentConfig(system_instruction=get_system_prompt(model))
632
  resp = client.models.generate_content(model=model, contents=contents, config=config)
633
  answer = (resp.text or "").strip()
634
- log(f"{model} {label}: ответ {len(answer)} симв. — {answer[:160]!r}")
 
 
 
 
635
  except Exception as exc:
636
  log(f"{model} {label}: ОШИБКА {exc}")
637
  return 0
 
631
  config = types.GenerateContentConfig(system_instruction=get_system_prompt(model), tools=tools) if tools else types.GenerateContentConfig(system_instruction=get_system_prompt(model))
632
  resp = client.models.generate_content(model=model, contents=contents, config=config)
633
  answer = (resp.text or "").strip()
634
+ gm = None
635
+ cands = resp.candidates or []
636
+ if cands:
637
+ gm = getattr(cands[0], "grounding_metadata", None)
638
+ log(f"{model} {label}: ответ {len(answer)} симв., grounding {'ЕСТЬ' if gm else 'нет'} — {answer[:160]!r}")
639
  except Exception as exc:
640
  log(f"{model} {label}: ОШИБКА {exc}")
641
  return 0