Spaces:
Paused
Paused
File size: 3,828 Bytes
2f86d9b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 | # Copyright 2026 DeepMind Technologies Limited.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""PubMedQA evaluation: answer extraction, accuracy, and BLEU metrics."""
from __future__ import annotations
import dataclasses
import re
from gemma.diffusion.hackable_diffusion_adapter.eval import base_metric
import sacrebleu
def extract_pubmedqa_answer(text: str) -> str | None:
"""Extract yes/no/maybe answer from generated text.
Uses a single, reliable strategy: look for the explicit marker
``"The answer is: {answer}"`` which the structured system prompt
instructs the model to produce.
No fallback heuristics are used (e.g. checking the first word or
scanning for any occurrence of yes/no/maybe) because they are too
fragile and lead to false positives — for example "yesterday" would
match "yes", or "no significant effect" would match "no".
Args:
text: Generated text that should contain a yes/no/maybe answer.
Returns:
Extracted answer string ('yes', 'no', or 'maybe'), or None if not found.
"""
text_lower = text.strip().lower()
# Explicit "The answer is: {answer}" marker.
marker_match = re.search(
r'the answer is[:\s]+\b(yes|no|maybe)\b', text_lower
)
if marker_match:
return marker_match.group(1)
return None
def score_pubmedqa(generated: str, ground_truth: str) -> bool:
"""Check whether the generated answer matches the ground truth.
Args:
generated: Full generated text.
ground_truth: The ground-truth answer string ('yes', 'no', or 'maybe').
Returns:
True if the extracted answer matches.
"""
gen_ans = extract_pubmedqa_answer(generated)
gt_ans = ground_truth.strip().lower()
if gen_ans is None:
return False
return gen_ans == gt_ans
@dataclasses.dataclass(kw_only=True, frozen=True)
class PubMedQAAccuracy(base_metric.BaseSimpleTextMetric):
"""Compute accuracy on PubMedQA by extracting yes/no/maybe from samples.
This metric detokenizes generated token sequences, extracts the
yes/no/maybe answer, and compares against the ground-truth short_answer.
"""
def score_example(self, generated_text: str, ground_truth_text: str) -> bool:
"""Scores a single example."""
return score_pubmedqa(generated_text, ground_truth_text)
def score_bleu(generated: str, ground_truth: str) -> float:
"""Compute smoothed sentence-level BLEU between generated and ground truth.
Args:
generated: The text produced by the model.
ground_truth: The reference text.
Returns:
Sentence-level BLEU score in [0, 100].
"""
generated = generated.strip()
ground_truth = ground_truth.strip()
if not generated or not ground_truth:
return 0.0
bleu = sacrebleu.corpus_bleu(
[generated],
[[ground_truth]],
smooth_method='floor',
use_effective_order=True,
)
return bleu.score
@dataclasses.dataclass(kw_only=True, frozen=True)
class BLEUScore(base_metric.BaseSimpleTextMetric):
"""Compute sentence-level BLEU between generated and ground-truth text.
This metric detokenizes generated token sequences and computes
smoothed sentence-level BLEU against the ground-truth tokens.
"""
def score_example(self, generated_text: str, ground_truth_text: str) -> float:
"""Scores a single example."""
return score_bleu(generated_text, ground_truth_text)
|