Spaces:
Running
Running
Download inspect_eval/solver/cb_solvers.py from daKeshra/ConspiracyBench: direct link, hf CLI and curl.
- Browser
- Download file 6.94 kB
-
https://huggingface.co/spaces/daKeshra/ConspiracyBench/resolve/main/inspect_eval/solver/cb_solvers.py
- Command line
-
hf download hf://spaces/daKeshra/ConspiracyBench/inspect_eval/solver/cb_solvers.py
-
curl -L -o cb_solvers.py https://huggingface.co/spaces/daKeshra/ConspiracyBench/resolve/main/inspect_eval/solver/cb_solvers.py
6.94 kB
| """The ConspiracyBench solver: the opening question, then the item's pressure turns.""" | |
| from __future__ import annotations | |
| from inspect_ai.model import ChatMessageUser, ContentReasoning, ContentText, get_model | |
| from inspect_ai.solver import Generate, TaskState, solver | |
| from cb_budget import ( | |
| CONTEXT_CLIPPED, | |
| CONTEXT_OVERFLOW_MODES, | |
| CUT_OFF_STOP_REASONS, | |
| DEFAULT_CONTEXT_OVERFLOW, | |
| DEFAULT_REPLY_TOKENS, | |
| OVERFLOW_TURN, | |
| STOP_REASONS, | |
| TRUNCATED_TURNS, | |
| TURNS_PLANNED, | |
| fits, | |
| slide_window, | |
| ) | |
| def without_reasoning(messages: list) -> list: | |
| """``messages`` with the reasoning taken out of every assistant message. | |
| Inspect sends an earlier turn's reasoning back to the provider, and whether the model then | |
| sees it depends on its chat template: in the smoke test K2 Horizon's template kept it, while | |
| Granite's, Gemma 4's and Nemotron's dropped it (and Gemma 4's card says it must be dropped). | |
| Emptying it here gives every model the same history: its earlier answers only. The reasoning | |
| block is kept but empty, because some templates (K2 Horizon's) reject an assistant message | |
| with no thinking field at all; ``internal`` is kept so it still goes out as that field rather | |
| than as inline ``<think>`` text. | |
| """ | |
| out = [] | |
| for message in messages: | |
| if message.role == "assistant": | |
| parts = [ContentText(text=message.content)] if isinstance(message.content, str) else message.content | |
| content = [c.model_copy(update={"reasoning": "", "summary": None, "signature": None}) | |
| if c.type == "reasoning" else c for c in parts] | |
| if not any(c.type == "reasoning" for c in content): | |
| # A reply can come back with no reasoning at all; K2's template still needs the field | |
| # (the full run's cb-0175 failed on exactly that). | |
| content.insert(0, ContentReasoning(reasoning="", internal="reasoning")) | |
| message = message.model_copy(update={"content": content}) | |
| out.append(message) | |
| return out | |
| def pressure_turns( | |
| pressure_ids: list[str] | None = None, | |
| context_tokens: int | None = None, | |
| max_tokens: int | None = None, | |
| context_overflow: str = DEFAULT_CONTEXT_OVERFLOW, | |
| ): | |
| """Answer the question, then each of the item's ``pressure_turns`` in order. | |
| The model's own replies stay in context between turns, without their reasoning | |
| (``without_reasoning``); the transcript keeps the reasoning. Items without pressure turns | |
| get a single generation, so single-turn items run exactly as a plain question would. | |
| ``pressure_ids`` applies the repeated-sampling protocol in SCHEMA.md: every epoch | |
| samples the opening turn, but the pressure sequence plays only in epoch 1, except for | |
| the listed sample ids, which play it in every epoch. ``None`` plays it in every epoch | |
| for every item. | |
| ``context_tokens`` is the token budget for the conversation (see ``cb_budget``). A turn | |
| whose prompt plus ``max_tokens`` of reply would not fit is never sent whole, since a | |
| provider may silently drop the start of it. With ``context_overflow="slide"`` (default) | |
| the oldest exchanges are left out until it fits, so the window keeps the most recent | |
| turns, and the turn is recorded in ``context_clipped``. With ``"stop"``, or when even the | |
| latest message alone doesn't fit, the sequence stops there and the turn is recorded in | |
| the sample's metadata for the scorer. ``None`` turns the check off. | |
| Why each reply ended is recorded in the sample's metadata, one Inspect stop reason per turn, | |
| as ``stop_reasons``. The turns that were cut off (they hit ``max_tokens`` or the model's own | |
| limit) are recorded as ``truncated_turns``, worked out from that list. | |
| """ | |
| if context_overflow not in CONTEXT_OVERFLOW_MODES: | |
| raise ValueError(f"context_overflow must be one of {CONTEXT_OVERFLOW_MODES}, got {context_overflow!r}") | |
| repeat = None if pressure_ids is None else set(pressure_ids) | |
| reply_tokens = max_tokens or DEFAULT_REPLY_TOKENS | |
| async def generate_if_it_fits(state: TaskState, generate: Generate) -> tuple[TaskState, bool]: | |
| turn = sum(1 for m in state.messages if m.role == "assistant") | |
| sent = without_reasoning(state.messages) | |
| window = sent | |
| if not await fits(get_model(), window, reply_tokens, context_tokens): | |
| window = None | |
| if context_overflow == "slide": | |
| window = await slide_window(get_model(), sent, reply_tokens, context_tokens) | |
| if window is None: | |
| state.metadata[OVERFLOW_TURN] = turn | |
| return state, False | |
| # Send only the most recent part of the conversation; the transcript keeps all of it. | |
| state.metadata.setdefault(CONTEXT_CLIPPED, []).append({"turn": turn, "dropped": len(sent) - len(window)}) | |
| # The model is sent its earlier answers without their reasoning; the transcript keeps both. | |
| full = list(state.messages) | |
| state.messages = window | |
| state = await generate(state) | |
| state.messages = full + state.messages[len(window):] | |
| reasons = state.metadata.setdefault(STOP_REASONS, []) | |
| reasons.append(state.output.stop_reason) | |
| # A reply that ran out of room before the model finished it. Derived from the list, so the | |
| # scorer can tell a complete answer from one that stops mid-sentence, and the two agree. | |
| cut_off = [turn for turn, reason in enumerate(reasons) if reason in CUT_OFF_STOP_REASONS] | |
| if cut_off: | |
| state.metadata[TRUNCATED_TURNS] = cut_off | |
| return state, True | |
| async def solve(state: TaskState, generate: Generate) -> TaskState: | |
| # Checked before any model call, so a malformed item doesn't spend one first. | |
| turns = state.metadata["item"].get("pressure_turns") | |
| if turns is not None: | |
| if not isinstance(turns, list): | |
| raise TypeError("pressure_turns must be a list") | |
| for turn in turns: | |
| if not isinstance(turn, str): | |
| raise TypeError("each turn in pressure_turns must be a string") | |
| # Recorded before the first call, so a sample that errors partway still says how far it | |
| # should have gone. | |
| plays_pressure = repeat is None or state.epoch == 1 or str(state.sample_id) in repeat | |
| state.metadata[TURNS_PLANNED] = 1 + len(turns or []) if plays_pressure else 1 | |
| state, sent = await generate_if_it_fits(state, generate) | |
| if not sent: | |
| return state | |
| if not plays_pressure: | |
| return state | |
| for turn in turns or []: | |
| state.messages.append(ChatMessageUser(content=turn)) | |
| state, sent = await generate_if_it_fits(state, generate) | |
| if not sent: | |
| break | |
| return state | |
| return solve | |