Hackathon-IA-VisualNovel / tests /test_orchestrator_guards.py
WillHbx's picture
test: engine split, llm helpers, and orchestrator guard coverage
1ea8154
Raw
History Blame Contribute Delete
9.62 kB
"""Code-side guards around the real-LLM director path: the anti-repetition retry,
new_character repair defaults, and the /no_think compaction switch."""
from __future__ import annotations
from visualnovel import config, orchestrator
from visualnovel.schemas import (
Character,
Directives,
DirectorOutput,
GameState,
NewCharacter,
Scene,
Turn,
)
def _state() -> GameState:
s = GameState(
seed=1,
vibe="v",
style_guide="s",
scene=Scene(id="a", place="A", description="d", present=["moth"]),
characters={"moth": Character(id="moth", name="Moth", one_line="a moth")},
)
s.recent_turns.append(Turn(player="hi", speaker="moth", dialogue="The wood remembers you."))
return s
class _RecordingLLM:
"""Returns scripted complete_json payloads and records every call's kwargs."""
def __init__(self, replies: list[dict]) -> None:
self.replies = list(replies)
self.calls: list[dict] = []
def complete(self, messages, **kw): # pragma: no cover - unused here
return ""
def complete_json(self, messages, schema, **kw):
self.calls.append({"messages": messages, **kw})
return self.replies.pop(0)
def _payload(dialogue: str) -> dict:
return {"speaker": "moth", "dialogue": dialogue, "emotion": "neutral"}
def test_repetition_triggers_one_hotter_retry(monkeypatch):
monkeypatch.setattr(config, "USE_MOCK", False)
state = _state()
llm = _RecordingLLM([_payload("The wood remembers you."), _payload("A fresh new line.")])
out = orchestrator.direct_turn(llm, state, "hello")
assert len(llm.calls) == 2
assert llm.calls[1]["temperature"] == 0.9
assert llm.calls[1]["presence_penalty"] == 0.8
assert "You already said this line" in llm.calls[1]["messages"][1]["content"]
assert out.dialogue == "A fresh new line."
def test_no_repetition_means_single_call(monkeypatch):
monkeypatch.setattr(config, "USE_MOCK", False)
state = _state()
llm = _RecordingLLM([_payload("Something entirely different.")])
out = orchestrator.direct_turn(llm, state, "hello")
assert len(llm.calls) == 1
assert out.dialogue == "Something entirely different."
def test_failed_retry_keeps_first_output(monkeypatch):
monkeypatch.setattr(config, "USE_MOCK", False)
state = _state()
class _FailsSecond(_RecordingLLM):
def complete_json(self, messages, schema, **kw):
if self.calls:
self.calls.append({})
raise RuntimeError("boom")
return super().complete_json(messages, schema, **kw)
llm = _FailsSecond([_payload("The wood remembers you.")])
out = orchestrator.direct_turn(llm, state, "hello")
assert out.dialogue == "The wood remembers you." # repeat kept rather than crash
def test_llm_failure_falls_back_to_safe_line(monkeypatch):
monkeypatch.setattr(config, "USE_MOCK", False)
class _Broken:
def complete(self, *a, **kw): # pragma: no cover
return ""
def complete_json(self, *a, **kw):
raise RuntimeError("model exploded")
out = orchestrator.direct_turn(_Broken(), _state(), "hello")
assert out.speaker == "moth" # first present character
assert out.dialogue # a bland in-fiction line, never a crash
def test_mock_mode_never_calls_llm():
class _Bomb:
def complete(self, *a, **kw):
raise AssertionError("must not be called in mock mode")
def complete_json(self, *a, **kw):
raise AssertionError("must not be called in mock mode")
out = orchestrator.direct_turn(_Bomb(), _state(), "hello")
assert out.dialogue
def test_repair_fills_new_character_defaults():
state = _state()
out = DirectorOutput(
speaker="moth",
dialogue="x",
directives=Directives(
new_character=NewCharacter(id="aoi", name="Aoi", one_line="quiet girl", appearance="")
),
)
repaired = orchestrator._repair(state, out)
nc = repaired.directives.new_character
assert nc.traits == ["curious", "reserved"]
assert nc.appearance == "quiet girl" # falls back to one_line
def test_compact_memory_sends_no_think(monkeypatch):
monkeypatch.setattr(config, "USE_MOCK", False)
state = _state()
for i in range(2 * config.RECENT_TURNS_K + 1):
state.recent_turns.append(Turn(player=f"p{i}", speaker="moth", dialogue=f"d{i}"))
captured: dict = {}
class _Stub:
def complete(self, messages, **kw):
captured["messages"] = messages
return "A neat prose summary."
def complete_json(self, messages, schema, **kw): # pragma: no cover
return {}
orchestrator.compact_memory(_Stub(), state)
assert captured["messages"][1]["content"].rstrip().endswith("/no_think")
assert state.summary == "A neat prose summary."
def test_directive_schema_forces_emotion_and_delta():
from visualnovel import prompts
schema = prompts.directive_schema()
# The grammar must not let the model omit these (it does, when optional):
# emotion otherwise stays "neutral", relationship_delta stays 0 forever
assert "emotion" in schema["required"]
assert "relationship_delta" in schema["$defs"]["Directives"]["required"]
def test_scout_schema_requires_new_character():
from visualnovel import prompts
scout = prompts.scout_schema()["$defs"]["Directives"]
assert "new_character" in scout["required"]
assert scout["properties"]["new_character"] == {"$ref": "#/$defs/NewCharacter"}
# the cached base schema must stay untouched (new_character optional elsewhere)
base = prompts.directive_schema()["$defs"]["Directives"]
assert "new_character" not in base.get("required", [])
def test_scout_uses_guaranteed_schema(monkeypatch):
monkeypatch.setattr(config, "USE_MOCK", False)
state = _state()
llm = _RecordingLLM(
[
{
"speaker": "yuki",
"dialogue": "Oh - hello! I am Yuki.",
"emotion": "shy",
"directives": {
"new_character": {
"id": "yuki",
"name": "Yuki",
"one_line": "a quiet bookworm",
"appearance": "glasses",
}
},
}
]
)
captured: dict = {}
original = llm.complete_json
def _spy(messages, schema, **kw):
captured["schema"] = schema
return original(messages, schema, **kw)
llm.complete_json = _spy
out = orchestrator.direct_turn(llm, state, "", action="scout")
assert "new_character" in captured["schema"]["$defs"]["Directives"]["required"]
assert out.directives.new_character is not None
assert out.speaker == "yuki" # the newcomer may speak on arrival
def test_scout_stage_full_keeps_optional_schema(monkeypatch):
monkeypatch.setattr(config, "USE_MOCK", False)
state = _state()
state.scene.present = ["moth", "a", "b"] # MAX_PRESENT reached
llm = _RecordingLLM([_payload("The courtyard is quiet today.")])
captured: dict = {}
original = llm.complete_json
def _spy(messages, schema, **kw):
captured["schema"] = schema
captured["user"] = messages[1]["content"]
return original(messages, schema, **kw)
llm.complete_json = _spy
orchestrator.direct_turn(llm, state, "", action="scout")
assert "new_character" not in captured["schema"]["$defs"]["Directives"].get("required", [])
assert "The stage is full" in captured["user"]
def test_repair_clips_runaway_new_character_fields():
state = _state()
out = DirectorOutput(
speaker="moth",
dialogue="x",
directives=Directives(
new_character=NewCharacter(
id="kaaris",
name="Kaaris",
one_line="a towering president",
appearance="muscular, bald",
goals="to maintain control " * 300, # runaway generation
)
),
)
repaired = orchestrator._repair(state, out)
assert len(repaired.directives.new_character.goals.split()) <= 50
def test_action_note_outside_player_quotes():
from visualnovel import memory, prompts
state = _state()
note = prompts.action_note("scout")
ctx = memory.assemble_context(state, "", action_note=note)
assert "DIRECTOR NOTE (mandatory):" in ctx
# the note must not sit inside the quoted player line
assert 'NOW SAYS: "…"' in ctx
def test_newcomer_may_speak_on_arrival_turn():
# "Look around" introduces someone — the newcomer must survive _repair as speaker
# even though they are not yet in scene.present (apply_directives adds them after).
state = _state()
out = DirectorOutput(
speaker="Yuki!", # raw LLM id, needs slug normalization
dialogue="Oh - were you looking for me?",
directives=Directives(
new_character=NewCharacter(
id="Yuki!", name="Yuki", one_line="a quiet bookworm", appearance="glasses"
)
),
)
repaired = orchestrator._repair(state, out)
assert repaired.speaker == "yuki" # normalized to the id apply_directives will create
def test_director_output_backward_compatible():
# An old /tmp pending_out (pre remember_fact) must still validate
old = {"speaker": "moth", "dialogue": "hi", "emotion": "joy", "directives": {}}
out = DirectorOutput.model_validate(old)
assert out.directives.remember_fact is None