from hackathon_advisor.dashboard_chat import DashboardChatEngine
from tests.test_dashboard_repository import analyzed_repository, not_analyzed_repository
class ScriptedRunner:
"""ChatRunner double that replays prepared model outputs and records calls."""
backend = "scripted"
model_id = "scripted-test-model"
supports_thinking = False
def __init__(self, outputs: list[object]) -> None:
self.outputs = list(outputs)
self.calls: list[dict] = []
def stream(self, messages, *, tools=None, max_new_tokens, enable_thinking=False):
self.calls.append(
{
"messages": messages,
"tools": tools,
"max_new_tokens": max_new_tokens,
"enable_thinking": enable_thinking,
}
)
output = self.outputs.pop(0)
pieces = output if isinstance(output, list) else [output]
for count, piece in enumerate(pieces, start=1):
yield count, piece
class ThinkingScriptedRunner(ScriptedRunner):
"""ScriptedRunner whose outputs start inside a block (native thinking)."""
supports_thinking = True
def run_turn(runner, repository, message, history=None):
engine = DashboardChatEngine(runner, lambda: repository)
return list(engine.turn_stream(message, history))
def events_of(events, event_type):
return [event for event in events if event["type"] == event_type]
def test_tool_turn_streams_verified_data_before_prose() -> None:
runner = ScriptedRunner(
[
'',
"Project 9 leads with two quests.",
]
)
events = run_turn(runner, analyzed_repository(), "who completed the most quests?")
tool_call = events_of(events, "tool_call")[0]
assert tool_call["status"] == "valid"
assert tool_call["name"] == "top_projects_by_quests"
tool_result = events_of(events, "tool_result")[0]
assert tool_result["data"]["rows"][0]["id"] == "build-small-hackathon/project-9"
assert tool_result["map_action"]["type"] == "highlight_projects"
types = [event["type"] for event in events]
assert types.index("tool_result") < types.index("token")
done = events_of(events, "done")[0]
assert done["response"] == "Project 9 leads with two quests."
assert done["tool"] == "top_projects_by_quests"
assert done["history"][-2:] == [
{"role": "user", "content": "who completed the most quests?"},
{"role": "assistant", "content": "Project 9 leads with two quests."},
]
def test_model_digest_strips_urls_and_ids() -> None:
runner = ScriptedRunner(
[
'',
"Project 9 leads.",
]
)
run_turn(runner, analyzed_repository(), "quest leaderboard")
answer_call = runner.calls[1]
assert answer_call["tools"] is None
digest = answer_call["messages"][-1]["content"]
assert "url" not in digest
assert "huggingface.co" not in digest
assert 'title: "Project 9"' in digest
assert "quest_count: 2" in digest
def test_model_digest_trims_long_listings_and_bm25_total() -> None:
repository = analyzed_repository()
runner = ScriptedRunner(
['project', "Found a few."]
)
run_turn(runner, repository, "find project planners")
search_digest = runner.calls[1]["messages"][-1]["content"]
assert "total:" not in search_digest
runner = ScriptedRunner(['', "A few clusters."])
run_turn(runner, repository, "what clusters exist")
cluster_digest = runner.calls[1]["messages"][-1]["content"]
# Only the largest cluster is restatable; the full list lives on the UI cards.
assert cluster_digest.count("label:") == 1
assert f"cluster_count: {repository.list_clusters()['cluster_count']}" in cluster_digest
def test_not_analyzed_quests_skip_the_answer_pass() -> None:
runner = ScriptedRunner([''])
events = run_turn(runner, not_analyzed_repository(), "who completed the most quests?")
skipped = events_of(events, "answer_skipped")[0]
assert skipped["reason"] == "quests_not_analyzed"
assert events_of(events, "token") == []
assert len(runner.calls) == 1 # pass 2 never ran
assert events_of(events, "done")[0]["response"] == skipped["text"]
def test_empty_search_skips_the_answer_pass() -> None:
runner = ScriptedRunner(
['zzzz qqqq']
)
events = run_turn(runner, analyzed_repository(), "find zzzz qqqq")
skipped = events_of(events, "answer_skipped")[0]
assert skipped["reason"] == "no_search_results"
assert "zzzz qqqq" in skipped["text"]
assert len(runner.calls) == 1
def test_unknown_cluster_uses_templated_sentence() -> None:
runner = ScriptedRunner(
['flying castles']
)
events = run_turn(runner, analyzed_repository(), "show me the flying castles cluster")
skipped = events_of(events, "answer_skipped")[0]
assert skipped["reason"] == "unknown_cluster"
assert "flying castles" in skipped["text"]
assert events_of(events, "tool_result")[0]["map_action"] is None
def test_show_cluster_emits_filter_map_action() -> None:
repository = analyzed_repository()
label = repository.list_clusters()["clusters"][0]["label"]
runner = ScriptedRunner(
[
f'{label}',
"That cluster groups the local-first planners.",
]
)
events = run_turn(runner, repository, f"what is in {label}?")
map_action = events_of(events, "tool_result")[0]["map_action"]
assert map_action == {"type": "filter_cluster", "label": label}
def test_show_cluster_map_action_carries_canonical_label_for_fuzzy_input() -> None:
"""The UI resolves filter_cluster by exact label match, so the engine must
forward the repository's canonical label, never the user's fuzzy input."""
repository = analyzed_repository()
canonical = repository.list_clusters()["clusters"][0]["label"]
fuzzy = canonical.split()[0].lower()
assert fuzzy != canonical
runner = ScriptedRunner(
[
f'{fuzzy}',
"Here is that cluster.",
]
)
events = run_turn(runner, repository, f"what is in the {fuzzy} cluster?")
map_action = events_of(events, "tool_result")[0]["map_action"]
assert map_action == {"type": "filter_cluster", "label": canonical}
def test_plain_prose_routes_to_dedicated_smalltalk_pass() -> None:
runner = ScriptedRunner(
[
"Hello! Happy to help.",
"Hi! Ask me what everyone is building.",
]
)
events = run_turn(runner, analyzed_repository(), "hello there")
assert events_of(events, "tool_call")[0]["status"] == "none"
assert events_of(events, "tool_result") == []
done = events_of(events, "done")[0]
assert done["tool"] == ""
assert done["response"] == "Hi! Ask me what everyone is building."
assert runner.calls[1]["tools"] is None
def test_unmatched_data_question_defaults_to_search_not_smalltalk() -> None:
"""Regression: 'how many voice apps' once slipped into ungrounded small talk."""
runner = ScriptedRunner(
[
"I'm not sure, but I can provide more information if you'd like.",
"The closest matches are shown on the cards.",
]
)
events = run_turn(runner, analyzed_repository(), "how many voice apps")
tool_call = events_of(events, "tool_call")[0]
assert tool_call["status"] == "defaulted"
assert tool_call["name"] == "search_projects"
assert tool_call["arguments"]["query"] == "how many voice apps"
def test_short_followup_stays_on_smalltalk_path() -> None:
runner = ScriptedRunner(
[
"Because it scored well.",
"I should look that up rather than guess - ask me about the quest leaderboard!",
]
)
events = run_turn(runner, analyzed_repository(), "are you sure?")
assert events_of(events, "tool_call")[0]["status"] == "none"
assert events_of(events, "tool_result") == []
def test_show_project_streams_readme_and_highlights_the_dot() -> None:
runner = ScriptedRunner(
[
'Project 4',
"Project 4 is an offline planner; its app file loads gradio.",
]
)
events = run_turn(runner, analyzed_repository(), "how does Project 4 work?")
tool_result = events_of(events, "tool_result")[0]
assert tool_result["tool"] == "show_project"
assert "README evidence for project 4" in tool_result["data"]["readme_excerpt"]
assert tool_result["map_action"] == {
"type": "highlight_projects",
"ids": ["build-small-hackathon/project-4"],
}
digest = runner.calls[1]["messages"][-1]["content"]
assert "README evidence for project 4" in digest
assert "import gradio" in digest
assert events_of(events, "done")[0]["tool"] == "show_project"
def test_show_project_falls_back_to_search_for_unknown_names() -> None:
runner = ScriptedRunner(
[
'offline planner thing',
"The closest matches are on the cards.",
]
)
events = run_turn(runner, analyzed_repository(), "tell me about the offline planner thing")
tool_result = events_of(events, "tool_result")[0]
assert tool_result["tool"] == "search_projects"
assert tool_result["data"]["results"]
assert events_of(events, "done")[0]["tool"] == "search_projects"
def test_prose_answer_to_data_question_is_routed_by_intent() -> None:
runner = ScriptedRunner(
[
"I do not have access to quest completion data.",
"Project 9 leads with two quests.",
]
)
events = run_turn(runner, analyzed_repository(), "who completed the most quests?")
tool_call = events_of(events, "tool_call")[0]
assert tool_call["status"] == "defaulted"
assert tool_call["name"] == "top_projects_by_quests"
assert events_of(events, "tool_result")[0]["data"]["rows"]
assert events_of(events, "done")[0]["response"] == "Project 9 leads with two quests."
def test_malformed_call_degrades_through_intent_router() -> None:
runner = ScriptedRunner(
[
'',
"Project 9 leads the quest board.",
]
)
events = run_turn(runner, analyzed_repository(), "who completed the most quests")
tool_call = events_of(events, "tool_call")[0]
assert tool_call["status"] == "defaulted"
assert tool_call["name"] == "top_projects_by_quests"
assert tool_call["errors"]
def test_stray_function_block_is_cut_from_the_answer() -> None:
runner = ScriptedRunner(
[
'',
["The field has ten projects. ", '', " extra"],
]
)
events = run_turn(runner, analyzed_repository(), "overview please")
done = events_of(events, "done")[0]
assert done["response"] == "The field has ten projects."
for token in events_of(events, "token"):
assert " None:
runner = ThinkingScriptedRunner(
[
[
"The user wants the leaderboard. I could emit ",
' here but the right tool is top_projects_by_quests.',
"\n\n',
],
["Let me restate the digest.\n\nProject 9 leads with two quests."],
]
)
events = run_turn(runner, analyzed_repository(), "who completed the most quests?")
thinking_pass1 = [e for e in events if e["type"] == "thinking" and e["pass"] == 1]
assert "".join(e["text"] for e in thinking_pass1).startswith("The user wants the leaderboard")
# " not in token["text"]
# Thinking never leaks into the durable history.
assert all("Let me restate" not in entry["content"] for entry in done["history"])
def test_thinking_marker_split_across_pieces_is_handled() -> None:
runner = ThinkingScriptedRunner(
[
["step one ", "step two", '\n\n'],
["ok\n\nThe quests are on the cards."],
]
)
events = run_turn(runner, analyzed_repository(), "list the quests")
thinking = "".join(e["text"] for e in events if e["type"] == "thinking" and e["pass"] == 1)
assert thinking == "step one step two"
assert events_of(events, "tool_call")[0]["name"] == "list_quests"
def test_truncated_thinking_degrades_gracefully() -> None:
"""A 4096-token cut inside leaves no answer text; the turn must still
resolve (intent backstop on pass 1, templated sentence on pass 2)."""
runner = ThinkingScriptedRunner(
[
["I am still reasoning about which tool to"], # no , no call
["and the digest says\n\nProject 9 leads."],
]
)
events = run_turn(runner, analyzed_repository(), "who completed the most quests?")
tool_call = events_of(events, "tool_call")[0]
assert tool_call["status"] == "defaulted"
assert tool_call["name"] == "top_projects_by_quests"
assert events_of(events, "done")[0]["response"] == "Project 9 leads."
def test_chat_generations_use_thinking_and_4096_budget() -> None:
runner = ScriptedRunner(
[
'',
"Ten projects in view.",
]
)
events = run_turn(runner, analyzed_repository(), "overview")
for call in runner.calls:
assert call["enable_thinking"] is True
assert call["max_new_tokens"] >= 4096
for progress in events_of(events, "model_progress"):
assert progress["max_tokens"] >= 4096
def test_history_drops_repeated_assistant_answers() -> None:
"""Regression: a greedy 1B echoes any sentence that appears twice in history."""
runner = ScriptedRunner(
[
'',
"Ten projects in view.",
]
)
looping_history = [
{"role": "user", "content": "why"},
{"role": "assistant", "content": "I should look that up."},
{"role": "user", "content": "are you sure?"},
{"role": "assistant", "content": "I should look that up."},
{"role": "user", "content": "really?"},
{"role": "assistant", "content": "I should look that up."},
]
run_turn(runner, analyzed_repository(), "overview", looping_history)
pass1_messages = runner.calls[0]["messages"]
repeated = [m for m in pass1_messages if m.get("content") == "I should look that up."]
assert len(repeated) == 1
def test_grounded_answer_pass_sees_no_history() -> None:
"""Echoes regression: with history in the prompt, a greedy 1B repeats prior
answers instead of reading the digest. Facts come from the digest alone."""
runner = ScriptedRunner(
[
'',
"Ten projects in view.",
]
)
long_history = []
for index in range(6):
long_history.append({"role": "user", "content": f"question {index}"})
long_history.append({"role": "assistant", "content": f"answer {index}"})
run_turn(runner, analyzed_repository(), "overview", long_history)
answer_messages = runner.calls[1]["messages"]
roles = [m["role"] for m in answer_messages]
assert roles == ["system", "user", "assistant", "tool"]
def test_overview_digest_leads_with_most_liked_projects() -> None:
runner = ScriptedRunner(
[
'',
"Project 9 is the most liked.",
]
)
run_turn(runner, analyzed_repository(), "what is the coolest project?")
digest = runner.calls[1]["messages"][-1]["content"]
assert digest.startswith("most_liked_projects:")
assert "most_completed_quests:" in digest
def test_history_is_normalized_and_capped() -> None:
runner = ScriptedRunner(
[
'',
"Ten projects in view.",
]
)
junk_history = [
{"role": "user", "content": "old question"},
{"role": "assistant", "content": "old answer"},
{"role": "tool", "content": "should be dropped"},
"not even a dict",
{"role": "user", "content": ""},
]
events = run_turn(runner, analyzed_repository(), "overview", junk_history)
pass1_messages = runner.calls[0]["messages"]
roles = [message["role"] for message in pass1_messages]
assert roles == ["system", "user", "assistant", "user"]
done = events_of(events, "done")[0]
assert all(entry["role"] in ("user", "assistant") for entry in done["history"])
assert len(done["history"]) <= 12