Spaces:
Running
Running
| """ | |
| test_cognitive_gaps.py — Regression tests per COG-1/2/3/4 | |
| Firme reali verificate il 2026-06-16 su HEAD c82d71b: | |
| should_replan(exec_warn, exec_done) → bool | |
| replan(planner, original_goal, exec_warn, exec_done, error_context="") → dict|None | |
| _should_test_ts(content, path="") → bool (>= 5 linee non-commento + TS regex + path .ts/.tsx) | |
| run_tdd_check_ts(content, path, executor, on_warn=None) → dict | |
| needs_dynamic_tool(tool_name, description) → bool (True solo se COMPUTE_RE match) | |
| generate_and_register(description, tool_name, llm, executor) → dict | |
| """ | |
| from __future__ import annotations | |
| import asyncio | |
| import inspect | |
| import os | |
| import sys | |
| import unittest | |
| from unittest.mock import AsyncMock, MagicMock | |
| _BACKEND = os.path.join(os.path.dirname(__file__), "..") | |
| if _BACKEND not in sys.path: | |
| sys.path.insert(0, _BACKEND) | |
| def _run(coro): | |
| return asyncio.get_event_loop().run_until_complete(coro) | |
| # ═══════════════════════════════════════════════════════════════════════════════ | |
| # COG-1: Dynamic Replanner | |
| # ═══════════════════════════════════════════════════════════════════════════════ | |
| class TestCOG1DynamicReplanner(unittest.TestCase): | |
| def setUpClass(cls): | |
| try: | |
| import agents.dynamic_replanner as dr | |
| cls.dr = dr | |
| except ImportError as e: | |
| raise unittest.SkipTest(f"agents.dynamic_replanner non importabile: {e}") | |
| def test_should_replan_exists(self): | |
| """COG-1: should_replan() è esportata da dynamic_replanner.""" | |
| self.assertTrue(callable(self.dr.should_replan)) | |
| def test_should_replan_signature(self): | |
| """COG-1: should_replan(exec_warn, exec_done) — 2 parametri.""" | |
| sig = inspect.signature(self.dr.should_replan) | |
| params = list(sig.parameters) | |
| self.assertEqual(len(params), 2, | |
| f"should_replan deve avere 2 params, got {params}") | |
| def test_should_replan_false_no_failures(self): | |
| """COG-1: should_replan → False quando exec_warn è vuoto.""" | |
| result = self.dr.should_replan([], ["subtask ok"]) | |
| self.assertFalse(result) | |
| def test_should_replan_false_no_real_errors(self): | |
| """COG-1: should_replan → False quando exec_warn non contiene errori reali.""" | |
| # Risk skips non sono failure reali | |
| exec_warn = ["⚠ subtask #1 skip: risk:high", "⚠ subtask #2 skip: risk:high"] | |
| result = self.dr.should_replan(exec_warn, ["ok1", "ok2", "ok3"]) | |
| self.assertFalse(result, | |
| "should_replan deve restituire False per warning non-failure") | |
| def test_should_replan_true_with_real_failure(self): | |
| """COG-1: should_replan → True con failures reali >= successi.""" | |
| exec_warn = ["⚠ subtask #1 fallito: timeout", "⚠ subtask #2 failed: ConnectionError"] | |
| exec_done = [] | |
| result = self.dr.should_replan(exec_warn, exec_done) | |
| self.assertTrue(result, | |
| "should_replan deve restituire True con failures reali >= successi") | |
| def test_should_replan_false_too_many_successes(self): | |
| """COG-1: should_replan → False se fallimenti < successi.""" | |
| exec_warn = ["⚠ subtask #1 fallito: timeout"] | |
| exec_done = ["ok1", "ok2", "ok3", "ok4"] # 4 successi > 1 fallimento | |
| result = self.dr.should_replan(exec_warn, exec_done) | |
| self.assertFalse(result, | |
| "should_replan deve restituire False se fallimenti < successi") | |
| def test_replan_exists(self): | |
| """COG-1: replan() è esportata da dynamic_replanner.""" | |
| self.assertTrue(callable(self.dr.replan)) | |
| def test_replan_signature(self): | |
| """COG-1: replan ha almeno 4 param obbligatori (planner, original_goal, exec_warn, exec_done).""" | |
| sig = inspect.signature(self.dr.replan) | |
| params = list(sig.parameters) | |
| self.assertGreaterEqual(len(params), 4, | |
| f"replan deve avere >= 4 params, got {params}") | |
| def test_replan_is_coroutine(self): | |
| """COG-1: replan() deve essere async def.""" | |
| self.assertTrue(asyncio.iscoroutinefunction(self.dr.replan)) | |
| def test_replan_returns_none_without_planner(self): | |
| """COG-1: replan(None, ...) → None — fallback sicuro senza planner.""" | |
| result = _run(self.dr.replan(None, "goal", [], [])) | |
| self.assertIsNone(result, | |
| "replan con planner=None deve restituire None") | |
| # ═══════════════════════════════════════════════════════════════════════════════ | |
| # COG-2: Reflection Recording in unified_loop.py | |
| # ═══════════════════════════════════════════════════════════════════════════════ | |
| class TestCOG2ReflectionRecording(unittest.TestCase): | |
| def setUpClass(cls): | |
| ul_path = os.path.join(_BACKEND, "agents", "unified_loop.py") | |
| if not os.path.exists(ul_path): | |
| raise unittest.SkipTest("agents/unified_loop.py non trovato") | |
| with open(ul_path, encoding="utf-8") as fh: | |
| cls.src = fh.read() | |
| # P20-TD1-F3b: _reflective_debug estratto in unified_loop_helpers.py | |
| # src concatena entrambi i file → tutti gli assertIn rimangono validi | |
| helpers_path = os.path.join(_BACKEND, "agents", "unified_loop_helpers.py") | |
| if os.path.exists(helpers_path): | |
| with open(helpers_path, encoding="utf-8") as fh: | |
| cls.src += "\n" + fh.read() | |
| def test_record_failure_hook_present(self): | |
| """COG-2: record_failure() è chiamata in unified_loop (hook post-errore).""" | |
| self.assertIn("record_failure", self.src) | |
| def test_record_success_hook_present(self): | |
| """COG-2: record_success() è chiamata in unified_loop (hook post-successo).""" | |
| self.assertIn("record_success", self.src) | |
| def test_cog2_marker_in_source(self): | |
| """COG-2: marker COG-2 presente nel codice.""" | |
| self.assertIn("COG-2", self.src) | |
| def test_reflection_record_failure_in_reflective_debug(self): | |
| """COG-2: record_failure chiamata dentro _reflective_debug.""" | |
| idx_fn = self.src.find("_reflective_debug") | |
| self.assertGreater(idx_fn, 0, "_reflective_debug non trovato") | |
| idx_rf = self.src.find("record_failure", idx_fn) | |
| self.assertGreater(idx_rf, 0, | |
| "record_failure non trovata dopo _reflective_debug") | |
| # ═══════════════════════════════════════════════════════════════════════════════ | |
| # COG-3: TypeScript TDD | |
| # ═══════════════════════════════════════════════════════════════════════════════ | |
| _TS_SAMPLE = """\ | |
| import { useState } from 'react'; | |
| interface Props { name: string; } | |
| const Greet = ({ name }: Props) => { | |
| const [count, setCount] = useState(0); | |
| return <div>{name} {count}</div>; | |
| }; | |
| export default Greet; | |
| """ | |
| class TestCOG3TypeScriptTDD(unittest.TestCase): | |
| def setUpClass(cls): | |
| try: | |
| import agents.tdd_runner as tdd | |
| cls.tdd = tdd | |
| except ImportError as e: | |
| raise unittest.SkipTest(f"agents.tdd_runner non importabile: {e}") | |
| def test_should_test_ts_exists(self): | |
| """COG-3: _should_test_ts() è definita in tdd_runner.""" | |
| self.assertTrue(callable(self.tdd._should_test_ts)) | |
| def test_should_test_ts_returns_false_for_python_path(self): | |
| """COG-3: _should_test_ts → False se path termina con .py.""" | |
| result = self.tdd._should_test_ts("print('hello')\n" * 6, "main.py") | |
| self.assertFalse(result, | |
| "_should_test_ts deve restituire False per file .py") | |
| def test_should_test_ts_returns_true_for_ts_path_with_code(self): | |
| """COG-3: _should_test_ts → True per file .ts con >= 5 righe TS.""" | |
| result = self.tdd._should_test_ts(_TS_SAMPLE, "App.ts") | |
| self.assertTrue(result, | |
| "_should_test_ts deve restituire True per file .ts con codice TS") | |
| def test_should_test_ts_returns_true_for_tsx(self): | |
| """COG-3: _should_test_ts → True per file .tsx con >= 5 righe TS.""" | |
| result = self.tdd._should_test_ts(_TS_SAMPLE, "App.tsx") | |
| self.assertTrue(result, | |
| "_should_test_ts deve restituire True per file .tsx") | |
| def test_should_test_ts_returns_false_too_few_lines(self): | |
| """COG-3: _should_test_ts → False per file .ts con < 5 righe non-commento.""" | |
| result = self.tdd._should_test_ts("const x: number = 1;", "app.ts") | |
| self.assertFalse(result, | |
| "_should_test_ts deve restituire False per file con poche righe") | |
| def test_should_test_ts_returns_false_no_ts_code_pattern(self): | |
| """COG-3: _should_test_ts → False se contenuto non ha pattern TS.""" | |
| non_ts = "\n".join(["# nessun ts"] * 6) | |
| result = self.tdd._should_test_ts(non_ts, "app.ts") | |
| self.assertFalse(result, | |
| "_should_test_ts deve restituire False senza pattern TS nel contenuto") | |
| def test_run_tdd_check_ts_exists(self): | |
| """COG-3: run_tdd_check_ts() è definita in tdd_runner.""" | |
| self.assertTrue(callable(self.tdd.run_tdd_check_ts)) | |
| def test_run_tdd_check_ts_is_coroutine(self): | |
| """COG-3: run_tdd_check_ts() deve essere async def.""" | |
| self.assertTrue(asyncio.iscoroutinefunction(self.tdd.run_tdd_check_ts)) | |
| def test_run_tdd_check_ts_skips_non_ts(self): | |
| """COG-3: run_tdd_check_ts con file .py non chiama on_warn (early return).""" | |
| called = [] | |
| async def _fake_on_warn(msg): | |
| called.append(msg) | |
| result = _run(self.tdd.run_tdd_check_ts( | |
| content="print('hello')\n" * 6, | |
| path="main.py", | |
| executor=None, | |
| on_warn=_fake_on_warn, | |
| )) | |
| self.assertEqual(called, [], | |
| "run_tdd_check_ts non deve chiamare on_warn su file .py") | |
| # ═══════════════════════════════════════════════════════════════════════════════ | |
| # COG-4: Tool Generator | |
| # ═══════════════════════════════════════════════════════════════════════════════ | |
| class TestCOG4ToolGenerator(unittest.TestCase): | |
| def setUpClass(cls): | |
| try: | |
| import agents.tool_generator as tg | |
| cls.tg = tg | |
| except ImportError as e: | |
| raise unittest.SkipTest(f"agents.tool_generator non importabile: {e}") | |
| def tearDown(self): | |
| # Reset _GENERATED_COUNT after tests that might increment it | |
| try: | |
| self.tg._GENERATED_COUNT = 0 | |
| except Exception: | |
| pass | |
| def test_needs_dynamic_tool_exists(self): | |
| """COG-4: needs_dynamic_tool() è esportata da tool_generator.""" | |
| self.assertTrue(callable(self.tg.needs_dynamic_tool)) | |
| def test_needs_dynamic_tool_false_for_network_description(self): | |
| """COG-4: needs_dynamic_tool → False per descrizioni con operazioni di rete.""" | |
| for desc in ("fetch url", "download file via http", "scrape page", "api request"): | |
| with self.subTest(desc=desc): | |
| result = self.tg.needs_dynamic_tool("mio_tool", desc) | |
| self.assertFalse(result, | |
| f"needs_dynamic_tool deve restituire False per '{desc}'") | |
| def test_needs_dynamic_tool_false_for_empty(self): | |
| """COG-4: needs_dynamic_tool → False per tool name vuoto.""" | |
| result = self.tg.needs_dynamic_tool("", "calcola la media") | |
| # Note: empty name + compute desc → True per logica attuale (solo description conta) | |
| # Testiamo solo che non sollevi eccezioni | |
| self.assertIsInstance(result, bool) | |
| def test_needs_dynamic_tool_true_for_compute_description(self): | |
| """COG-4: needs_dynamic_tool → True per descrizione computazionale locale.""" | |
| for desc in ("calcola la media dei valori", "converti JSON in CSV", | |
| "analizza il testo", "estrai le parole chiave"): | |
| with self.subTest(desc=desc): | |
| self.tg._GENERATED_COUNT = 0 # reset guard | |
| result = self.tg.needs_dynamic_tool("mio_tool", desc) | |
| self.assertTrue(result, | |
| f"needs_dynamic_tool deve restituire True per '{desc}'") | |
| def test_needs_dynamic_tool_false_when_limit_reached(self): | |
| """COG-4: needs_dynamic_tool → False quando _MAX_GENERATED raggiunto.""" | |
| self.tg._GENERATED_COUNT = self.tg._MAX_GENERATED | |
| result = self.tg.needs_dynamic_tool("qualsiasi", "calcola la media") | |
| self.assertFalse(result, | |
| "needs_dynamic_tool deve restituire False al limite sessione") | |
| def test_generate_and_register_exists(self): | |
| """COG-4: generate_and_register() è esportata da tool_generator.""" | |
| self.assertTrue(callable(self.tg.generate_and_register)) | |
| def test_generate_and_register_is_coroutine(self): | |
| """COG-4: generate_and_register() deve essere async def.""" | |
| self.assertTrue(asyncio.iscoroutinefunction(self.tg.generate_and_register)) | |
| def test_generate_and_register_signature(self): | |
| """COG-4: generate_and_register ha >= 3 param (description, tool_name, llm, ...).""" | |
| sig = inspect.signature(self.tg.generate_and_register) | |
| params = list(sig.parameters) | |
| self.assertGreaterEqual(len(params), 3, | |
| f"generate_and_register deve avere >= 3 params, got {params}") | |
| def test_security_blocked_imports_list_exists(self): | |
| """COG-4: lista di import bloccati esiste in tool_generator.""" | |
| src = inspect.getsource(self.tg) | |
| blocked_keywords = ["requests", "httpx", "socket", "subprocess"] | |
| found = [kw for kw in blocked_keywords if kw in src] | |
| self.assertGreater(len(found), 0, | |
| "Nessun import di rete bloccato trovato — security gate assente") | |
| def test_session_limit_constant_exists(self): | |
| """COG-4: costante _MAX_GENERATED esiste con valore <= 10.""" | |
| self.assertTrue(hasattr(self.tg, "_MAX_GENERATED"), | |
| "_MAX_GENERATED non trovato in tool_generator") | |
| self.assertLessEqual(self.tg._MAX_GENERATED, 10, | |
| "_MAX_GENERATED dovrebbe essere <= 10 per sicurezza") | |
| # ═══════════════════════════════════════════════════════════════════════════════ | |
| # COG-4 Wiring: unified_loop.py ha l'elif _s_tool: block | |
| # ═══════════════════════════════════════════════════════════════════════════════ | |
| class TestCOG4WiringInUnifiedLoop(unittest.TestCase): | |
| def setUpClass(cls): | |
| ul_path = os.path.join(_BACKEND, "agents", "unified_loop.py") | |
| if not os.path.exists(ul_path): | |
| raise unittest.SkipTest("agents/unified_loop.py non trovato") | |
| with open(ul_path, encoding="utf-8") as fh: | |
| cls.src = fh.read() | |
| def test_tool_map_miss_handled(self): | |
| """COG-4 wiring: dopo TOOL_MAP miss, c'è elif _s_tool: (no silent skip).""" | |
| idx_get = self.src.find("_TOOL_MAP.get(_s_tool") | |
| self.assertGreater(idx_get, 0, "_TOOL_MAP.get(_s_tool non trovato") | |
| block = self.src[idx_get: idx_get + 2000] | |
| self.assertIn("elif _s_tool:", block) | |
| def test_tool_generator_import_in_elif_block(self): | |
| """COG-4 wiring: import tool_generator presente nel elif _s_tool: block.""" | |
| idx_elif = self.src.find("elif _s_tool:") | |
| self.assertGreater(idx_elif, 0) | |
| block = self.src[idx_elif: idx_elif + 800] | |
| self.assertIn("tool_generator", block) | |
| def test_generate_and_register_called_in_elif(self): | |
| """COG-4 wiring: generate_and_register chiamata nel block elif _s_tool:.""" | |
| idx_elif = self.src.find("elif _s_tool:") | |
| self.assertGreater(idx_elif, 0) | |
| block = self.src[idx_elif: idx_elif + 800] | |
| self.assertIn("generate_and_register", block) | |
| # ═══════════════════════════════════════════════════════════════════════════════ | |
| # COG-1 Wiring: unified_loop.py ha il dynamic replan post-gather | |
| # ═══════════════════════════════════════════════════════════════════════════════ | |
| class TestCOG1WiringInUnifiedLoop(unittest.TestCase): | |
| def setUpClass(cls): | |
| ul_path = os.path.join(_BACKEND, "agents", "unified_loop.py") | |
| if not os.path.exists(ul_path): | |
| raise unittest.SkipTest("agents/unified_loop.py non trovato") | |
| with open(ul_path, encoding="utf-8") as fh: | |
| cls.src = fh.read() | |
| def test_cog1_real_failures_filter_present(self): | |
| """COG-1 wiring: _cog1_real_failures filtra exec_warn per keyword.""" | |
| self.assertIn("_cog1_real_failures", self.src) | |
| def test_should_replan_called_in_loop(self): | |
| """COG-1 wiring: should_replan() chiamata in unified_loop post-gather.""" | |
| self.assertIn("should_replan", self.src) | |
| def test_dynamic_replanner_imported_in_loop(self): | |
| """COG-1 wiring: dynamic_replanner importato nel loop.""" | |
| self.assertIn("dynamic_replanner", self.src) | |
| def test_replanned_flag_blocks_recursion(self): | |
| """COG-1 wiring: plan._replanned flag presente per evitare loop infiniti.""" | |
| self.assertIn("_replanned", self.src) | |
| def test_replan_result_clears_exec_state(self): | |
| """COG-1 wiring: exec_done.clear() e exec_warn.clear() presenti nel replan block.""" | |
| idx_replan = self.src.find("_cog1_real_failures") | |
| self.assertGreater(idx_replan, 0) | |
| block = self.src[idx_replan: idx_replan + 4000] | |
| self.assertIn("exec_done.clear()", block) | |
| self.assertIn("exec_warn.clear()", block) | |
| # ═══════════════════════════════════════════════════════════════════════════════ | |
| # COG-3 Wiring: unified_loop.py chiama run_tdd_check_ts dopo write_file/.ts | |
| # ═══════════════════════════════════════════════════════════════════════════════ | |
| class TestCOG3WiringInUnifiedLoop(unittest.TestCase): | |
| def setUpClass(cls): | |
| ul_path = os.path.join(_BACKEND, "agents", "unified_loop.py") | |
| if not os.path.exists(ul_path): | |
| raise unittest.SkipTest("agents/unified_loop.py non trovato") | |
| with open(ul_path, encoding="utf-8") as fh: | |
| cls.src = fh.read() | |
| def test_tdd_runner_imported_near_write_file(self): | |
| """COG-3 wiring: tdd_runner importato vicino al blocco write_file.""" | |
| self.assertIn("tdd_runner", self.src) | |
| def test_run_tdd_check_ts_called_in_write_block(self): | |
| """COG-3 wiring: run_tdd_check_ts chiamata dopo write_file/apply_patch.""" | |
| self.assertIn("run_tdd_check_ts", self.src) | |
| def test_cog3_guard_checks_ts_extension(self): | |
| """COG-3 wiring: guard che controlla .ts/.tsx prima di chiamare type_check.""" | |
| self.assertIn('.endswith((".ts", ".tsx"))', self.src) | |
| def test_cog3_failure_adds_to_exec_warn(self): | |
| """COG-3 wiring: type_check failure aggiunge warning a exec_warn.""" | |
| idx_cog3 = self.src.find("run_tdd_check_ts") | |
| self.assertGreater(idx_cog3, 0) | |
| block = self.src[idx_cog3: idx_cog3 + 1200] | |
| self.assertIn("exec_warn.append", block) | |
| def test_cog3_timeout_protected(self): | |
| """COG-3 wiring: run_tdd_check_ts protetta da asyncio.wait_for timeout.""" | |
| idx_import = self.src.find("from agents.tdd_runner import run_tdd_check_ts") | |
| self.assertGreater(idx_import, 0) | |
| # La chiamata con wait_for viene dopo l'import — finestra più ampia | |
| block = self.src[idx_import: idx_import + 1000] | |
| self.assertIn("wait_for", block) | |
| def test_cog3_scaffold_project_hooked(self): | |
| """COG-3 wiring: hook post-scaffold_project presente in unified_loop.""" | |
| self.assertIn("scaffold_project", self.src) | |
| # Verifica che il hook COG-3 per scaffold sia presente | |
| idx_scaf = self.src.find('rn == "scaffold_project" and _r.get("success")') | |
| self.assertGreater(idx_scaf, 0, "COG-3 scaffold hook non trovato") | |
| def test_cog3_scaffold_uses_ts_stub(self): | |
| """COG-3 wiring: scaffold hook usa contenuto TS stub per trigger _should_test_ts.""" | |
| self.assertIn("_SCAF_TS_STUB", self.src) | |
| # ═══════════════════════════════════════════════════════════════════════════════ | |
| # Entrypoint | |
| # ═══════════════════════════════════════════════════════════════════════════════ | |
| # ============================================================================== | |
| # COG-5: Goal Drift Detector | |
| # ============================================================================== | |
| class TestCOG5GoalDriftDetector(unittest.TestCase): | |
| def setUpClass(cls): | |
| try: | |
| import agents.goal_drift_detector as gdd | |
| cls.gdd = gdd | |
| except ImportError as e: | |
| raise unittest.SkipTest(f"agents.goal_drift_detector non importabile: {e}") | |
| # ── _extract_keywords ──────────────────────────────────────────────── | |
| def test_extract_keywords_filters_short_words(self): | |
| """COG-5: _extract_keywords esclude parole < 4 char.""" | |
| kws = self.gdd._extract_keywords("il lo la di a e un una") | |
| self.assertEqual(len(kws), 0, "Stop-word brevi non devono essere estratte") | |
| def test_extract_keywords_returns_meaningful_words(self): | |
| """COG-5: _extract_keywords restituisce parole significative.""" | |
| kws = self.gdd._extract_keywords("analizza il meteo di Roma oggi") | |
| self.assertIn("analizza", kws) | |
| self.assertIn("meteo", kws) | |
| self.assertIn("roma", kws) | |
| def test_extract_keywords_case_insensitive(self): | |
| """COG-5: _extract_keywords normalizza a lowercase.""" | |
| kws = self.gdd._extract_keywords("Roma METEO Temperatura") | |
| self.assertIn("roma", kws) | |
| self.assertIn("meteo", kws) | |
| self.assertIn("temperatura", kws) | |
| # ── compute_drift_score ────────────────────────────────────────────── | |
| def test_drift_score_zero_for_empty_exec(self): | |
| """COG-5: drift score = 0.0 se exec_done vuoto.""" | |
| score = self.gdd.compute_drift_score("trova la temperatura a Roma", []) | |
| self.assertEqual(score, 0.0) | |
| def test_drift_score_low_when_on_target(self): | |
| """COG-5: drift score basso quando exec_done copre le keyword del goal.""" | |
| exec_done = [ | |
| "[subtask 1 — cerca temperatura]: temperatura Roma oggi 22 gradi", | |
| "[subtask 2 — meteo]: previsione meteo Roma domani", | |
| ] | |
| score = self.gdd.compute_drift_score("trova la temperatura a Roma", exec_done) | |
| self.assertLess(score, 0.5, f"Score troppo alto per task on-target: {score}") | |
| def test_drift_score_high_when_off_target(self): | |
| """COG-5: drift score alto quando exec_done non copre il goal.""" | |
| exec_done = [ | |
| "[subtask 1 — ricetta]: pasta carbonara uova pancetta", | |
| "[subtask 2 — cucina]: aggiungere sale pepe parmigiano", | |
| ] | |
| score = self.gdd.compute_drift_score("calcola l'equazione differenziale integrale", exec_done) | |
| self.assertGreater(score, 0.5, f"Score troppo basso per task off-target: {score}") | |
| def test_drift_score_in_range(self): | |
| """COG-5: drift score sempre in [0.0, 1.0].""" | |
| score = self.gdd.compute_drift_score("test", ["risultato casuale xyz abc"]) | |
| self.assertGreaterEqual(score, 0.0) | |
| self.assertLessEqual(score, 1.0) | |
| # ── should_check_drift ─────────────────────────────────────────────── | |
| def test_should_check_drift_false_below_min(self): | |
| """COG-5: should_check_drift → False se step_count < _MIN_EXEC_DONE.""" | |
| result = self.gdd.should_check_drift(1, 0) | |
| self.assertFalse(result, "Non deve controllare prima di _MIN_EXEC_DONE step") | |
| def test_should_check_drift_false_too_soon(self): | |
| """COG-5: should_check_drift → False se non sono passati DRIFT_CHECK_EVERY_N step.""" | |
| result = self.gdd.should_check_drift(3, 2) | |
| self.assertFalse(result, "Non deve controllare se non sono passati N step dal check") | |
| def test_should_check_drift_true_at_interval(self): | |
| """COG-5: should_check_drift → True esattamente ogni DRIFT_CHECK_EVERY_N step.""" | |
| n = self.gdd.DRIFT_CHECK_EVERY_N | |
| result = self.gdd.should_check_drift(n, 0) | |
| self.assertTrue(result, f"Deve controllare al passo {n}") | |
| def test_should_check_drift_true_after_prev_check(self): | |
| """COG-5: should_check_drift → True dopo DRIFT_CHECK_EVERY_N dal precedente check.""" | |
| n = self.gdd.DRIFT_CHECK_EVERY_N | |
| result = self.gdd.should_check_drift(6, 3) # 6-3 == 3 == N | |
| self.assertTrue(result) | |
| # ── detect_drift ───────────────────────────────────────────────────── | |
| def test_detect_drift_not_checked_below_interval(self): | |
| """COG-5: detect_drift.checked = False se intervallo non raggiunto.""" | |
| res = self.gdd.detect_drift("goal test qualsiasi", [], 1, 0) | |
| self.assertFalse(res["checked"]) | |
| self.assertFalse(res["drifted"]) | |
| def test_detect_drift_checked_at_interval(self): | |
| """COG-5: detect_drift.checked = True quando should_check_drift è True.""" | |
| n = self.gdd.DRIFT_CHECK_EVERY_N | |
| exec_done = [f"[subtask {i}]: risultato generico" for i in range(n)] | |
| res = self.gdd.detect_drift("goal test qualsiasi", exec_done, n, 0) | |
| self.assertTrue(res["checked"]) | |
| def test_detect_drift_updates_last_check(self): | |
| """COG-5: detect_drift aggiorna new_last_check al valore step_count.""" | |
| n = self.gdd.DRIFT_CHECK_EVERY_N | |
| exec_done = [f"[subtask {i}]: risultato" for i in range(n)] | |
| res = self.gdd.detect_drift("test", exec_done, n, 0) | |
| self.assertEqual(res["new_last_check"], n) | |
| def test_detect_drift_returns_score(self): | |
| """COG-5: detect_drift restituisce score float in [0,1] quando checked.""" | |
| n = self.gdd.DRIFT_CHECK_EVERY_N | |
| exec_done = ["[subtask 1]: cibo ricetta pasta", "[subtask 2]: ingredienti cucina", "[subtask 3]: cottura"] | |
| res = self.gdd.detect_drift("analizza dati finanziari borsa mercato", exec_done, n, 0) | |
| self.assertTrue(res["checked"]) | |
| self.assertIsInstance(res["score"], float) | |
| self.assertGreaterEqual(res["score"], 0.0) | |
| self.assertLessEqual(res["score"], 1.0) | |
| def test_detect_drift_no_mutation_on_exec_done(self): | |
| """COG-5: detect_drift non modifica exec_done (side-effect free).""" | |
| exec_done = ["[subtask 1]: risultato A", "[subtask 2]: risultato B", "[subtask 3]: C"] | |
| original = list(exec_done) | |
| n = self.gdd.DRIFT_CHECK_EVERY_N | |
| self.gdd.detect_drift("goal test", exec_done, n, 0) | |
| self.assertEqual(exec_done, original, "detect_drift non deve mutare exec_done") | |
| def test_detect_drift_reason_filled_on_drift(self): | |
| """COG-5: detect_drift.reason non vuoto se drifted=True.""" | |
| n = self.gdd.DRIFT_CHECK_EVERY_N | |
| # exec_done completamente off-topic rispetto al goal | |
| exec_done = [ | |
| "[subtask 1 — poesia]: verso strofa rima metafora", | |
| "[subtask 2 — letteratura]: romanzo narrativa personaggio", | |
| "[subtask 3 — arte]: pittura scultura museo opera", | |
| ] | |
| res = self.gdd.detect_drift( | |
| "calcola derivata integrale equazione differenziale", exec_done, n, 0 | |
| ) | |
| if res["drifted"]: | |
| self.assertGreater(len(res["reason"]), 0, "reason deve essere non vuoto se drifted") | |
| def test_constants_sane(self): | |
| """COG-5: costanti hanno valori ragionevoli.""" | |
| self.assertGreaterEqual(self.gdd.DRIFT_CHECK_EVERY_N, 2) | |
| self.assertLessEqual(self.gdd.DRIFT_CHECK_EVERY_N, 10) | |
| self.assertGreater(self.gdd.DRIFT_OVERLAP_THRESHOLD, 0.0) | |
| self.assertLess(self.gdd.DRIFT_OVERLAP_THRESHOLD, 1.0) | |
| class TestCOG5WiringInUnifiedLoop(unittest.TestCase): | |
| def setUpClass(cls): | |
| ul_path = os.path.join(_BACKEND, "agents", "unified_loop.py") | |
| if not os.path.exists(ul_path): | |
| raise unittest.SkipTest("agents/unified_loop.py non trovato") | |
| with open(ul_path, encoding="utf-8") as fh: | |
| cls.src = fh.read() | |
| def test_cog5_last_check_initialized(self): | |
| """COG-5 wiring: _cog5_last_check inizializzato nel loop.""" | |
| self.assertIn("_cog5_last_check", self.src) | |
| def test_goal_drift_detector_imported(self): | |
| """COG-5 wiring: goal_drift_detector importato in unified_loop.""" | |
| self.assertIn("goal_drift_detector", self.src) | |
| def test_detect_drift_called(self): | |
| """COG-5 wiring: detect_drift (o alias) chiamato in unified_loop.""" | |
| self.assertIn("detect_drift", self.src) | |
| def test_cog5_uses_state_goal(self): | |
| """COG-5 wiring: detect_drift riceve state.goal come goal originale.""" | |
| idx = self.src.find("goal_drift_detector") | |
| self.assertGreater(idx, 0) | |
| block = self.src[idx: idx + 800] | |
| self.assertIn("state.goal", block) | |
| def test_cog5_uses_exec_done(self): | |
| """COG-5 wiring: detect_drift riceve exec_done come lista risultati.""" | |
| idx = self.src.find("goal_drift_detector") | |
| self.assertGreater(idx, 0) | |
| block = self.src[idx: idx + 800] | |
| self.assertIn("exec_done", block) | |
| def test_cog5_drift_appends_to_exec_warn(self): | |
| """COG-5 wiring: in caso di drift, il messaggio viene aggiunto a exec_warn.""" | |
| idx = self.src.find("goal_drift_detector") | |
| self.assertGreater(idx, 0) | |
| block = self.src[idx: idx + 800] | |
| self.assertIn("exec_warn.append", block) | |
| def test_cog5_is_non_blocking(self): | |
| """COG-5 wiring: detect_drift è dentro try/except (non-blocking).""" | |
| idx = self.src.find("goal_drift_detector") | |
| self.assertGreater(idx, 0) | |
| # La try/except deve precedere l'import | |
| pre_block = self.src[max(0, idx - 200): idx + 800] | |
| self.assertIn("except Exception as _cog5_err", pre_block) | |
| def test_cog5_marker_in_source(self): | |
| """COG-5 wiring: marker COG-5 presente in unified_loop.""" | |
| self.assertIn("COG-5", self.src) | |
| # ============================================================================== | |
| # BUG-6: tool_generator _BLOCKED_RE open() write-mode bypass regression | |
| # ============================================================================== | |
| class TestBUG6BlockedReOpenBypass(unittest.TestCase): | |
| """ | |
| Regressione per BUG-6: _BLOCKED_RE non bloccava open() con modalità | |
| write binaria o con modificatori ('wb', 'w+', 'ab', 'wb+'). | |
| Fix: [wa][+bt]* invece di [wa] nel pattern open(). | |
| """ | |
| def setUpClass(cls): | |
| try: | |
| import agents.tool_generator as tg | |
| cls.tg = tg | |
| except ImportError as e: | |
| raise unittest.SkipTest(f"agents.tool_generator non importabile: {e}") | |
| def _is_blocked(self, code: str) -> bool: | |
| """True se il codice è bloccato da _BLOCKED_RE.""" | |
| return bool(self.tg._BLOCKED_RE.search(code)) | |
| # ── Write-mode bypassati prima del fix ─────────────────────────────── | |
| def test_blocks_open_write_binary_wb(self): | |
| """BUG-6 regression: open('file', 'wb') deve essere bloccato.""" | |
| self.assertTrue( | |
| self._is_blocked("open('output.bin', 'wb')"), | |
| "_BLOCKED_RE deve bloccare open() in modalità 'wb'", | |
| ) | |
| def test_blocks_open_write_plus(self): | |
| """BUG-6 regression: open('file', 'w+') deve essere bloccato.""" | |
| self.assertTrue( | |
| self._is_blocked("open('data.txt', 'w+')"), | |
| "_BLOCKED_RE deve bloccare open() in modalità 'w+'", | |
| ) | |
| def test_blocks_open_append_binary_ab(self): | |
| """BUG-6 regression: open('file', 'ab') deve essere bloccato.""" | |
| self.assertTrue( | |
| self._is_blocked("open('log.bin', 'ab')"), | |
| "_BLOCKED_RE deve bloccare open() in modalità 'ab'", | |
| ) | |
| def test_blocks_open_write_binary_plus_wbplus(self): | |
| """BUG-6 regression: open('file', 'wb+') deve essere bloccato.""" | |
| self.assertTrue( | |
| self._is_blocked("open('f', 'wb+')"), | |
| "_BLOCKED_RE deve bloccare open() in modalità 'wb+'", | |
| ) | |
| # ── Modalità lecite non devono essere bloccate ─────────────────────── | |
| def test_allows_open_read_mode(self): | |
| """BUG-6: open('file', 'r') NON deve essere bloccato (read-only).""" | |
| self.assertFalse( | |
| self._is_blocked("open('data.csv', 'r')"), | |
| "_BLOCKED_RE non deve bloccare open() in modalità 'r'", | |
| ) | |
| def test_allows_open_read_binary(self): | |
| """BUG-6: open('file', 'rb') NON deve essere bloccato (read binary).""" | |
| self.assertFalse( | |
| self._is_blocked("open('data.bin', 'rb')"), | |
| "_BLOCKED_RE non deve bloccare open() in modalità 'rb'", | |
| ) | |
| # ── Pattern storici (non-regressione) ──────────────────────────────── | |
| def test_still_blocks_open_write_single(self): | |
| """BUG-6 non-regressione: open('file', 'w') originale ancora bloccato.""" | |
| self.assertTrue( | |
| self._is_blocked("open('out.txt', 'w')"), | |
| "_BLOCKED_RE deve ancora bloccare open() in modalità 'w'", | |
| ) | |
| def test_still_blocks_import_requests(self): | |
| """BUG-6 non-regressione: import requests ancora bloccato.""" | |
| self.assertTrue( | |
| self._is_blocked("import requests"), | |
| "_BLOCKED_RE deve ancora bloccare import requests", | |
| ) | |
| def test_still_blocks_eval(self): | |
| """BUG-6 non-regressione: eval() ancora bloccato.""" | |
| self.assertTrue( | |
| self._is_blocked("eval('1+1')"), | |
| "_BLOCKED_RE deve ancora bloccare eval()", | |
| ) | |
| # ============================================================================ | |
| # BUG-10: unified_loop_tools.py _REAL_DATA_PREFIXES had "[PAGINA WEB" but | |
| # _t_read_page() returns "[PAGINA REALE: {url}]" -- prefix mismatch | |
| # means every successful read_page was counted as an error. | |
| # PFX-1..PFX-5 regression tests. | |
| # ============================================================================ | |
| class TestBUG10ReadPagePrefixMismatch(unittest.TestCase): | |
| """ | |
| BUG-10: _REAL_DATA_PREFIXES contained "[PAGINA WEB" but _t_read_page() | |
| returns strings starting with "[PAGINA REALE: {url}]". | |
| str.startswith("[PAGINA WEB") always False for those strings -> every | |
| successful read_page was counted as an error, not a success. | |
| Fix: replace "[PAGINA WEB" with "[PAGINA REALE" in _REAL_DATA_PREFIXES. | |
| """ | |
| _OLD_PREFIXES = ( | |
| "[RICERCA WEB REALE", "[METEO", "[CALCOLO REALE", | |
| "[IMMAGINE AI GENERATA", "[CODICE PYTHON ESEGUITO", | |
| "[PAGINA WEB", | |
| "[DATI REALI", "[RICERCA APPROFONDITA", "[STRUTTURA PROGETTO", | |
| "[FILE TROVATI", "[NOTIZIE", "[STATO GIT", | |
| ) | |
| _FIXED_PREFIXES = ( | |
| "[RICERCA WEB REALE", "[METEO", "[CALCOLO REALE", | |
| "[IMMAGINE AI GENERATA", "[CODICE PYTHON ESEGUITO", | |
| "[PAGINA REALE", | |
| "[DATI REALI", "[RICERCA APPROFONDITA", "[STRUTTURA PROGETTO", | |
| "[FILE TROVATI", "[NOTIZIE", "[STATO GIT", | |
| ) | |
| _SAMPLE = "[PAGINA REALE: https://example.com]\n(status 200)\nContent here" | |
| # PFX-1: old prefix does not match read_page output | |
| def test_pfx1_old_prefix_does_not_match(self): | |
| """PFX-1: [PAGINA WEB] does NOT match [PAGINA REALE:...] -- the old bug.""" | |
| self.assertFalse( | |
| self._SAMPLE.startswith("[PAGINA WEB"), | |
| "[PAGINA WEB] must NOT match read_page output (demonstrates bug)", | |
| ) | |
| # PFX-2: fixed prefix matches read_page output | |
| def test_pfx2_fixed_prefix_matches(self): | |
| """PFX-2: [PAGINA REALE] DOES match [PAGINA REALE:...] -- fix correct.""" | |
| self.assertTrue( | |
| self._SAMPLE.startswith("[PAGINA REALE"), | |
| "[PAGINA REALE] must match read_page output", | |
| ) | |
| # PFX-3: n_success = 0 with old broken prefix set | |
| def test_pfx3_old_prefix_gives_zero_success(self): | |
| """PFX-3: old prefix set => n_success=0 for a read_page result (the bug).""" | |
| results = [self._SAMPLE] | |
| n = sum(1 for r in results if any(r.startswith(p) for p in self._OLD_PREFIXES)) | |
| self.assertEqual(n, 0, "Old [PAGINA WEB] prefix => n_success=0 (bug confirmed)") | |
| # PFX-4: n_success = 1 with fixed prefix set | |
| def test_pfx4_fixed_prefix_gives_correct_success(self): | |
| """PFX-4: fixed prefix set => n_success=1 for a read_page result.""" | |
| results = [self._SAMPLE] | |
| n = sum(1 for r in results if any(r.startswith(p) for p in self._FIXED_PREFIXES)) | |
| self.assertEqual(n, 1, "Fixed [PAGINA REALE] => n_success=1") | |
| # PFX-5: all other tool prefixes still match correctly (non-regression) | |
| def test_pfx5_other_prefixes_non_regression(self): | |
| """PFX-5: non-regression -- all other tool prefixes still work.""" | |
| cases = [ | |
| ("[RICERCA WEB REALE: 'AI']\nresult", True), | |
| ("[METEO REALE -- Roma]\n22C", True), | |
| ("[CALCOLO REALE]\n2+2=4", True), | |
| ("[IMMAGINE AI GENERATA]\nURL: https://img", True), | |
| ("[CODICE PYTHON ESEGUITO]\noutput", True), | |
| ("[PAGINA REALE: https://x.com]\ncontent", True), | |
| ("[STRUTTURA PROGETTO: '.']\nfiles", True), | |
| ("[FILE TROVATI: pattern='*.py', 3]\nmatches", True), | |
| ("[NOTIZIE: 'AI']\narticle", True), | |
| ("[STATO GIT (branch: main)\nmodified: x", True), | |
| ("[web_search: NESSUN_RISULTATO]", False), | |
| ("[read_page: errore -- conn refused]", False), | |
| ("[git_status: timeout 8s]", False), | |
| ] | |
| for output, expected in cases: | |
| result = any(output.startswith(p) for p in self._FIXED_PREFIXES) | |
| self.assertEqual( | |
| result, expected, | |
| "Prefix check failed for: %r" % output[:60], | |
| ) | |
| if __name__ == "__main__": | |
| unittest.main(verbosity=2) | |