Terminal / tests /test_cognitive_gaps.py
Pulka
sync: 3 changed, 0 deleted — e46e604d (2026-06-20 23:51)
5437a2b verified
Raw
History Blame Contribute Delete
41.1 kB
"""
test_cognitive_gaps.py — Regression tests per COG-1/2/3/4
Firme reali verificate il 2026-06-16 su HEAD c82d71b:
should_replan(exec_warn, exec_done) → bool
replan(planner, original_goal, exec_warn, exec_done, error_context="") → dict|None
_should_test_ts(content, path="") → bool (>= 5 linee non-commento + TS regex + path .ts/.tsx)
run_tdd_check_ts(content, path, executor, on_warn=None) → dict
needs_dynamic_tool(tool_name, description) → bool (True solo se COMPUTE_RE match)
generate_and_register(description, tool_name, llm, executor) → dict
"""
from __future__ import annotations
import asyncio
import inspect
import os
import sys
import unittest
from unittest.mock import AsyncMock, MagicMock
_BACKEND = os.path.join(os.path.dirname(__file__), "..")
if _BACKEND not in sys.path:
sys.path.insert(0, _BACKEND)
def _run(coro):
return asyncio.get_event_loop().run_until_complete(coro)
# ═══════════════════════════════════════════════════════════════════════════════
# COG-1: Dynamic Replanner
# ═══════════════════════════════════════════════════════════════════════════════
class TestCOG1DynamicReplanner(unittest.TestCase):
@classmethod
def setUpClass(cls):
try:
import agents.dynamic_replanner as dr
cls.dr = dr
except ImportError as e:
raise unittest.SkipTest(f"agents.dynamic_replanner non importabile: {e}")
def test_should_replan_exists(self):
"""COG-1: should_replan() è esportata da dynamic_replanner."""
self.assertTrue(callable(self.dr.should_replan))
def test_should_replan_signature(self):
"""COG-1: should_replan(exec_warn, exec_done) — 2 parametri."""
sig = inspect.signature(self.dr.should_replan)
params = list(sig.parameters)
self.assertEqual(len(params), 2,
f"should_replan deve avere 2 params, got {params}")
def test_should_replan_false_no_failures(self):
"""COG-1: should_replan → False quando exec_warn è vuoto."""
result = self.dr.should_replan([], ["subtask ok"])
self.assertFalse(result)
def test_should_replan_false_no_real_errors(self):
"""COG-1: should_replan → False quando exec_warn non contiene errori reali."""
# Risk skips non sono failure reali
exec_warn = ["⚠ subtask #1 skip: risk:high", "⚠ subtask #2 skip: risk:high"]
result = self.dr.should_replan(exec_warn, ["ok1", "ok2", "ok3"])
self.assertFalse(result,
"should_replan deve restituire False per warning non-failure")
def test_should_replan_true_with_real_failure(self):
"""COG-1: should_replan → True con failures reali >= successi."""
exec_warn = ["⚠ subtask #1 fallito: timeout", "⚠ subtask #2 failed: ConnectionError"]
exec_done = []
result = self.dr.should_replan(exec_warn, exec_done)
self.assertTrue(result,
"should_replan deve restituire True con failures reali >= successi")
def test_should_replan_false_too_many_successes(self):
"""COG-1: should_replan → False se fallimenti < successi."""
exec_warn = ["⚠ subtask #1 fallito: timeout"]
exec_done = ["ok1", "ok2", "ok3", "ok4"] # 4 successi > 1 fallimento
result = self.dr.should_replan(exec_warn, exec_done)
self.assertFalse(result,
"should_replan deve restituire False se fallimenti < successi")
def test_replan_exists(self):
"""COG-1: replan() è esportata da dynamic_replanner."""
self.assertTrue(callable(self.dr.replan))
def test_replan_signature(self):
"""COG-1: replan ha almeno 4 param obbligatori (planner, original_goal, exec_warn, exec_done)."""
sig = inspect.signature(self.dr.replan)
params = list(sig.parameters)
self.assertGreaterEqual(len(params), 4,
f"replan deve avere >= 4 params, got {params}")
def test_replan_is_coroutine(self):
"""COG-1: replan() deve essere async def."""
self.assertTrue(asyncio.iscoroutinefunction(self.dr.replan))
def test_replan_returns_none_without_planner(self):
"""COG-1: replan(None, ...) → None — fallback sicuro senza planner."""
result = _run(self.dr.replan(None, "goal", [], []))
self.assertIsNone(result,
"replan con planner=None deve restituire None")
# ═══════════════════════════════════════════════════════════════════════════════
# COG-2: Reflection Recording in unified_loop.py
# ═══════════════════════════════════════════════════════════════════════════════
class TestCOG2ReflectionRecording(unittest.TestCase):
@classmethod
def setUpClass(cls):
ul_path = os.path.join(_BACKEND, "agents", "unified_loop.py")
if not os.path.exists(ul_path):
raise unittest.SkipTest("agents/unified_loop.py non trovato")
with open(ul_path, encoding="utf-8") as fh:
cls.src = fh.read()
# P20-TD1-F3b: _reflective_debug estratto in unified_loop_helpers.py
# src concatena entrambi i file → tutti gli assertIn rimangono validi
helpers_path = os.path.join(_BACKEND, "agents", "unified_loop_helpers.py")
if os.path.exists(helpers_path):
with open(helpers_path, encoding="utf-8") as fh:
cls.src += "\n" + fh.read()
def test_record_failure_hook_present(self):
"""COG-2: record_failure() è chiamata in unified_loop (hook post-errore)."""
self.assertIn("record_failure", self.src)
def test_record_success_hook_present(self):
"""COG-2: record_success() è chiamata in unified_loop (hook post-successo)."""
self.assertIn("record_success", self.src)
def test_cog2_marker_in_source(self):
"""COG-2: marker COG-2 presente nel codice."""
self.assertIn("COG-2", self.src)
def test_reflection_record_failure_in_reflective_debug(self):
"""COG-2: record_failure chiamata dentro _reflective_debug."""
idx_fn = self.src.find("_reflective_debug")
self.assertGreater(idx_fn, 0, "_reflective_debug non trovato")
idx_rf = self.src.find("record_failure", idx_fn)
self.assertGreater(idx_rf, 0,
"record_failure non trovata dopo _reflective_debug")
# ═══════════════════════════════════════════════════════════════════════════════
# COG-3: TypeScript TDD
# ═══════════════════════════════════════════════════════════════════════════════
_TS_SAMPLE = """\
import { useState } from 'react';
interface Props { name: string; }
const Greet = ({ name }: Props) => {
const [count, setCount] = useState(0);
return <div>{name} {count}</div>;
};
export default Greet;
"""
class TestCOG3TypeScriptTDD(unittest.TestCase):
@classmethod
def setUpClass(cls):
try:
import agents.tdd_runner as tdd
cls.tdd = tdd
except ImportError as e:
raise unittest.SkipTest(f"agents.tdd_runner non importabile: {e}")
def test_should_test_ts_exists(self):
"""COG-3: _should_test_ts() è definita in tdd_runner."""
self.assertTrue(callable(self.tdd._should_test_ts))
def test_should_test_ts_returns_false_for_python_path(self):
"""COG-3: _should_test_ts → False se path termina con .py."""
result = self.tdd._should_test_ts("print('hello')\n" * 6, "main.py")
self.assertFalse(result,
"_should_test_ts deve restituire False per file .py")
def test_should_test_ts_returns_true_for_ts_path_with_code(self):
"""COG-3: _should_test_ts → True per file .ts con >= 5 righe TS."""
result = self.tdd._should_test_ts(_TS_SAMPLE, "App.ts")
self.assertTrue(result,
"_should_test_ts deve restituire True per file .ts con codice TS")
def test_should_test_ts_returns_true_for_tsx(self):
"""COG-3: _should_test_ts → True per file .tsx con >= 5 righe TS."""
result = self.tdd._should_test_ts(_TS_SAMPLE, "App.tsx")
self.assertTrue(result,
"_should_test_ts deve restituire True per file .tsx")
def test_should_test_ts_returns_false_too_few_lines(self):
"""COG-3: _should_test_ts → False per file .ts con < 5 righe non-commento."""
result = self.tdd._should_test_ts("const x: number = 1;", "app.ts")
self.assertFalse(result,
"_should_test_ts deve restituire False per file con poche righe")
def test_should_test_ts_returns_false_no_ts_code_pattern(self):
"""COG-3: _should_test_ts → False se contenuto non ha pattern TS."""
non_ts = "\n".join(["# nessun ts"] * 6)
result = self.tdd._should_test_ts(non_ts, "app.ts")
self.assertFalse(result,
"_should_test_ts deve restituire False senza pattern TS nel contenuto")
def test_run_tdd_check_ts_exists(self):
"""COG-3: run_tdd_check_ts() è definita in tdd_runner."""
self.assertTrue(callable(self.tdd.run_tdd_check_ts))
def test_run_tdd_check_ts_is_coroutine(self):
"""COG-3: run_tdd_check_ts() deve essere async def."""
self.assertTrue(asyncio.iscoroutinefunction(self.tdd.run_tdd_check_ts))
def test_run_tdd_check_ts_skips_non_ts(self):
"""COG-3: run_tdd_check_ts con file .py non chiama on_warn (early return)."""
called = []
async def _fake_on_warn(msg):
called.append(msg)
result = _run(self.tdd.run_tdd_check_ts(
content="print('hello')\n" * 6,
path="main.py",
executor=None,
on_warn=_fake_on_warn,
))
self.assertEqual(called, [],
"run_tdd_check_ts non deve chiamare on_warn su file .py")
# ═══════════════════════════════════════════════════════════════════════════════
# COG-4: Tool Generator
# ═══════════════════════════════════════════════════════════════════════════════
class TestCOG4ToolGenerator(unittest.TestCase):
@classmethod
def setUpClass(cls):
try:
import agents.tool_generator as tg
cls.tg = tg
except ImportError as e:
raise unittest.SkipTest(f"agents.tool_generator non importabile: {e}")
def tearDown(self):
# Reset _GENERATED_COUNT after tests that might increment it
try:
self.tg._GENERATED_COUNT = 0
except Exception:
pass
def test_needs_dynamic_tool_exists(self):
"""COG-4: needs_dynamic_tool() è esportata da tool_generator."""
self.assertTrue(callable(self.tg.needs_dynamic_tool))
def test_needs_dynamic_tool_false_for_network_description(self):
"""COG-4: needs_dynamic_tool → False per descrizioni con operazioni di rete."""
for desc in ("fetch url", "download file via http", "scrape page", "api request"):
with self.subTest(desc=desc):
result = self.tg.needs_dynamic_tool("mio_tool", desc)
self.assertFalse(result,
f"needs_dynamic_tool deve restituire False per '{desc}'")
def test_needs_dynamic_tool_false_for_empty(self):
"""COG-4: needs_dynamic_tool → False per tool name vuoto."""
result = self.tg.needs_dynamic_tool("", "calcola la media")
# Note: empty name + compute desc → True per logica attuale (solo description conta)
# Testiamo solo che non sollevi eccezioni
self.assertIsInstance(result, bool)
def test_needs_dynamic_tool_true_for_compute_description(self):
"""COG-4: needs_dynamic_tool → True per descrizione computazionale locale."""
for desc in ("calcola la media dei valori", "converti JSON in CSV",
"analizza il testo", "estrai le parole chiave"):
with self.subTest(desc=desc):
self.tg._GENERATED_COUNT = 0 # reset guard
result = self.tg.needs_dynamic_tool("mio_tool", desc)
self.assertTrue(result,
f"needs_dynamic_tool deve restituire True per '{desc}'")
def test_needs_dynamic_tool_false_when_limit_reached(self):
"""COG-4: needs_dynamic_tool → False quando _MAX_GENERATED raggiunto."""
self.tg._GENERATED_COUNT = self.tg._MAX_GENERATED
result = self.tg.needs_dynamic_tool("qualsiasi", "calcola la media")
self.assertFalse(result,
"needs_dynamic_tool deve restituire False al limite sessione")
def test_generate_and_register_exists(self):
"""COG-4: generate_and_register() è esportata da tool_generator."""
self.assertTrue(callable(self.tg.generate_and_register))
def test_generate_and_register_is_coroutine(self):
"""COG-4: generate_and_register() deve essere async def."""
self.assertTrue(asyncio.iscoroutinefunction(self.tg.generate_and_register))
def test_generate_and_register_signature(self):
"""COG-4: generate_and_register ha >= 3 param (description, tool_name, llm, ...)."""
sig = inspect.signature(self.tg.generate_and_register)
params = list(sig.parameters)
self.assertGreaterEqual(len(params), 3,
f"generate_and_register deve avere >= 3 params, got {params}")
def test_security_blocked_imports_list_exists(self):
"""COG-4: lista di import bloccati esiste in tool_generator."""
src = inspect.getsource(self.tg)
blocked_keywords = ["requests", "httpx", "socket", "subprocess"]
found = [kw for kw in blocked_keywords if kw in src]
self.assertGreater(len(found), 0,
"Nessun import di rete bloccato trovato — security gate assente")
def test_session_limit_constant_exists(self):
"""COG-4: costante _MAX_GENERATED esiste con valore <= 10."""
self.assertTrue(hasattr(self.tg, "_MAX_GENERATED"),
"_MAX_GENERATED non trovato in tool_generator")
self.assertLessEqual(self.tg._MAX_GENERATED, 10,
"_MAX_GENERATED dovrebbe essere <= 10 per sicurezza")
# ═══════════════════════════════════════════════════════════════════════════════
# COG-4 Wiring: unified_loop.py ha l'elif _s_tool: block
# ═══════════════════════════════════════════════════════════════════════════════
class TestCOG4WiringInUnifiedLoop(unittest.TestCase):
@classmethod
def setUpClass(cls):
ul_path = os.path.join(_BACKEND, "agents", "unified_loop.py")
if not os.path.exists(ul_path):
raise unittest.SkipTest("agents/unified_loop.py non trovato")
with open(ul_path, encoding="utf-8") as fh:
cls.src = fh.read()
def test_tool_map_miss_handled(self):
"""COG-4 wiring: dopo TOOL_MAP miss, c'è elif _s_tool: (no silent skip)."""
idx_get = self.src.find("_TOOL_MAP.get(_s_tool")
self.assertGreater(idx_get, 0, "_TOOL_MAP.get(_s_tool non trovato")
block = self.src[idx_get: idx_get + 2000]
self.assertIn("elif _s_tool:", block)
def test_tool_generator_import_in_elif_block(self):
"""COG-4 wiring: import tool_generator presente nel elif _s_tool: block."""
idx_elif = self.src.find("elif _s_tool:")
self.assertGreater(idx_elif, 0)
block = self.src[idx_elif: idx_elif + 800]
self.assertIn("tool_generator", block)
def test_generate_and_register_called_in_elif(self):
"""COG-4 wiring: generate_and_register chiamata nel block elif _s_tool:."""
idx_elif = self.src.find("elif _s_tool:")
self.assertGreater(idx_elif, 0)
block = self.src[idx_elif: idx_elif + 800]
self.assertIn("generate_and_register", block)
# ═══════════════════════════════════════════════════════════════════════════════
# COG-1 Wiring: unified_loop.py ha il dynamic replan post-gather
# ═══════════════════════════════════════════════════════════════════════════════
class TestCOG1WiringInUnifiedLoop(unittest.TestCase):
@classmethod
def setUpClass(cls):
ul_path = os.path.join(_BACKEND, "agents", "unified_loop.py")
if not os.path.exists(ul_path):
raise unittest.SkipTest("agents/unified_loop.py non trovato")
with open(ul_path, encoding="utf-8") as fh:
cls.src = fh.read()
def test_cog1_real_failures_filter_present(self):
"""COG-1 wiring: _cog1_real_failures filtra exec_warn per keyword."""
self.assertIn("_cog1_real_failures", self.src)
def test_should_replan_called_in_loop(self):
"""COG-1 wiring: should_replan() chiamata in unified_loop post-gather."""
self.assertIn("should_replan", self.src)
def test_dynamic_replanner_imported_in_loop(self):
"""COG-1 wiring: dynamic_replanner importato nel loop."""
self.assertIn("dynamic_replanner", self.src)
def test_replanned_flag_blocks_recursion(self):
"""COG-1 wiring: plan._replanned flag presente per evitare loop infiniti."""
self.assertIn("_replanned", self.src)
def test_replan_result_clears_exec_state(self):
"""COG-1 wiring: exec_done.clear() e exec_warn.clear() presenti nel replan block."""
idx_replan = self.src.find("_cog1_real_failures")
self.assertGreater(idx_replan, 0)
block = self.src[idx_replan: idx_replan + 4000]
self.assertIn("exec_done.clear()", block)
self.assertIn("exec_warn.clear()", block)
# ═══════════════════════════════════════════════════════════════════════════════
# COG-3 Wiring: unified_loop.py chiama run_tdd_check_ts dopo write_file/.ts
# ═══════════════════════════════════════════════════════════════════════════════
class TestCOG3WiringInUnifiedLoop(unittest.TestCase):
@classmethod
def setUpClass(cls):
ul_path = os.path.join(_BACKEND, "agents", "unified_loop.py")
if not os.path.exists(ul_path):
raise unittest.SkipTest("agents/unified_loop.py non trovato")
with open(ul_path, encoding="utf-8") as fh:
cls.src = fh.read()
def test_tdd_runner_imported_near_write_file(self):
"""COG-3 wiring: tdd_runner importato vicino al blocco write_file."""
self.assertIn("tdd_runner", self.src)
def test_run_tdd_check_ts_called_in_write_block(self):
"""COG-3 wiring: run_tdd_check_ts chiamata dopo write_file/apply_patch."""
self.assertIn("run_tdd_check_ts", self.src)
def test_cog3_guard_checks_ts_extension(self):
"""COG-3 wiring: guard che controlla .ts/.tsx prima di chiamare type_check."""
self.assertIn('.endswith((".ts", ".tsx"))', self.src)
def test_cog3_failure_adds_to_exec_warn(self):
"""COG-3 wiring: type_check failure aggiunge warning a exec_warn."""
idx_cog3 = self.src.find("run_tdd_check_ts")
self.assertGreater(idx_cog3, 0)
block = self.src[idx_cog3: idx_cog3 + 1200]
self.assertIn("exec_warn.append", block)
def test_cog3_timeout_protected(self):
"""COG-3 wiring: run_tdd_check_ts protetta da asyncio.wait_for timeout."""
idx_import = self.src.find("from agents.tdd_runner import run_tdd_check_ts")
self.assertGreater(idx_import, 0)
# La chiamata con wait_for viene dopo l'import — finestra più ampia
block = self.src[idx_import: idx_import + 1000]
self.assertIn("wait_for", block)
def test_cog3_scaffold_project_hooked(self):
"""COG-3 wiring: hook post-scaffold_project presente in unified_loop."""
self.assertIn("scaffold_project", self.src)
# Verifica che il hook COG-3 per scaffold sia presente
idx_scaf = self.src.find('rn == "scaffold_project" and _r.get("success")')
self.assertGreater(idx_scaf, 0, "COG-3 scaffold hook non trovato")
def test_cog3_scaffold_uses_ts_stub(self):
"""COG-3 wiring: scaffold hook usa contenuto TS stub per trigger _should_test_ts."""
self.assertIn("_SCAF_TS_STUB", self.src)
# ═══════════════════════════════════════════════════════════════════════════════
# Entrypoint
# ═══════════════════════════════════════════════════════════════════════════════
# ==============================================================================
# COG-5: Goal Drift Detector
# ==============================================================================
class TestCOG5GoalDriftDetector(unittest.TestCase):
@classmethod
def setUpClass(cls):
try:
import agents.goal_drift_detector as gdd
cls.gdd = gdd
except ImportError as e:
raise unittest.SkipTest(f"agents.goal_drift_detector non importabile: {e}")
# ── _extract_keywords ────────────────────────────────────────────────
def test_extract_keywords_filters_short_words(self):
"""COG-5: _extract_keywords esclude parole < 4 char."""
kws = self.gdd._extract_keywords("il lo la di a e un una")
self.assertEqual(len(kws), 0, "Stop-word brevi non devono essere estratte")
def test_extract_keywords_returns_meaningful_words(self):
"""COG-5: _extract_keywords restituisce parole significative."""
kws = self.gdd._extract_keywords("analizza il meteo di Roma oggi")
self.assertIn("analizza", kws)
self.assertIn("meteo", kws)
self.assertIn("roma", kws)
def test_extract_keywords_case_insensitive(self):
"""COG-5: _extract_keywords normalizza a lowercase."""
kws = self.gdd._extract_keywords("Roma METEO Temperatura")
self.assertIn("roma", kws)
self.assertIn("meteo", kws)
self.assertIn("temperatura", kws)
# ── compute_drift_score ──────────────────────────────────────────────
def test_drift_score_zero_for_empty_exec(self):
"""COG-5: drift score = 0.0 se exec_done vuoto."""
score = self.gdd.compute_drift_score("trova la temperatura a Roma", [])
self.assertEqual(score, 0.0)
def test_drift_score_low_when_on_target(self):
"""COG-5: drift score basso quando exec_done copre le keyword del goal."""
exec_done = [
"[subtask 1 — cerca temperatura]: temperatura Roma oggi 22 gradi",
"[subtask 2 — meteo]: previsione meteo Roma domani",
]
score = self.gdd.compute_drift_score("trova la temperatura a Roma", exec_done)
self.assertLess(score, 0.5, f"Score troppo alto per task on-target: {score}")
def test_drift_score_high_when_off_target(self):
"""COG-5: drift score alto quando exec_done non copre il goal."""
exec_done = [
"[subtask 1 — ricetta]: pasta carbonara uova pancetta",
"[subtask 2 — cucina]: aggiungere sale pepe parmigiano",
]
score = self.gdd.compute_drift_score("calcola l'equazione differenziale integrale", exec_done)
self.assertGreater(score, 0.5, f"Score troppo basso per task off-target: {score}")
def test_drift_score_in_range(self):
"""COG-5: drift score sempre in [0.0, 1.0]."""
score = self.gdd.compute_drift_score("test", ["risultato casuale xyz abc"])
self.assertGreaterEqual(score, 0.0)
self.assertLessEqual(score, 1.0)
# ── should_check_drift ───────────────────────────────────────────────
def test_should_check_drift_false_below_min(self):
"""COG-5: should_check_drift → False se step_count < _MIN_EXEC_DONE."""
result = self.gdd.should_check_drift(1, 0)
self.assertFalse(result, "Non deve controllare prima di _MIN_EXEC_DONE step")
def test_should_check_drift_false_too_soon(self):
"""COG-5: should_check_drift → False se non sono passati DRIFT_CHECK_EVERY_N step."""
result = self.gdd.should_check_drift(3, 2)
self.assertFalse(result, "Non deve controllare se non sono passati N step dal check")
def test_should_check_drift_true_at_interval(self):
"""COG-5: should_check_drift → True esattamente ogni DRIFT_CHECK_EVERY_N step."""
n = self.gdd.DRIFT_CHECK_EVERY_N
result = self.gdd.should_check_drift(n, 0)
self.assertTrue(result, f"Deve controllare al passo {n}")
def test_should_check_drift_true_after_prev_check(self):
"""COG-5: should_check_drift → True dopo DRIFT_CHECK_EVERY_N dal precedente check."""
n = self.gdd.DRIFT_CHECK_EVERY_N
result = self.gdd.should_check_drift(6, 3) # 6-3 == 3 == N
self.assertTrue(result)
# ── detect_drift ─────────────────────────────────────────────────────
def test_detect_drift_not_checked_below_interval(self):
"""COG-5: detect_drift.checked = False se intervallo non raggiunto."""
res = self.gdd.detect_drift("goal test qualsiasi", [], 1, 0)
self.assertFalse(res["checked"])
self.assertFalse(res["drifted"])
def test_detect_drift_checked_at_interval(self):
"""COG-5: detect_drift.checked = True quando should_check_drift è True."""
n = self.gdd.DRIFT_CHECK_EVERY_N
exec_done = [f"[subtask {i}]: risultato generico" for i in range(n)]
res = self.gdd.detect_drift("goal test qualsiasi", exec_done, n, 0)
self.assertTrue(res["checked"])
def test_detect_drift_updates_last_check(self):
"""COG-5: detect_drift aggiorna new_last_check al valore step_count."""
n = self.gdd.DRIFT_CHECK_EVERY_N
exec_done = [f"[subtask {i}]: risultato" for i in range(n)]
res = self.gdd.detect_drift("test", exec_done, n, 0)
self.assertEqual(res["new_last_check"], n)
def test_detect_drift_returns_score(self):
"""COG-5: detect_drift restituisce score float in [0,1] quando checked."""
n = self.gdd.DRIFT_CHECK_EVERY_N
exec_done = ["[subtask 1]: cibo ricetta pasta", "[subtask 2]: ingredienti cucina", "[subtask 3]: cottura"]
res = self.gdd.detect_drift("analizza dati finanziari borsa mercato", exec_done, n, 0)
self.assertTrue(res["checked"])
self.assertIsInstance(res["score"], float)
self.assertGreaterEqual(res["score"], 0.0)
self.assertLessEqual(res["score"], 1.0)
def test_detect_drift_no_mutation_on_exec_done(self):
"""COG-5: detect_drift non modifica exec_done (side-effect free)."""
exec_done = ["[subtask 1]: risultato A", "[subtask 2]: risultato B", "[subtask 3]: C"]
original = list(exec_done)
n = self.gdd.DRIFT_CHECK_EVERY_N
self.gdd.detect_drift("goal test", exec_done, n, 0)
self.assertEqual(exec_done, original, "detect_drift non deve mutare exec_done")
def test_detect_drift_reason_filled_on_drift(self):
"""COG-5: detect_drift.reason non vuoto se drifted=True."""
n = self.gdd.DRIFT_CHECK_EVERY_N
# exec_done completamente off-topic rispetto al goal
exec_done = [
"[subtask 1 — poesia]: verso strofa rima metafora",
"[subtask 2 — letteratura]: romanzo narrativa personaggio",
"[subtask 3 — arte]: pittura scultura museo opera",
]
res = self.gdd.detect_drift(
"calcola derivata integrale equazione differenziale", exec_done, n, 0
)
if res["drifted"]:
self.assertGreater(len(res["reason"]), 0, "reason deve essere non vuoto se drifted")
def test_constants_sane(self):
"""COG-5: costanti hanno valori ragionevoli."""
self.assertGreaterEqual(self.gdd.DRIFT_CHECK_EVERY_N, 2)
self.assertLessEqual(self.gdd.DRIFT_CHECK_EVERY_N, 10)
self.assertGreater(self.gdd.DRIFT_OVERLAP_THRESHOLD, 0.0)
self.assertLess(self.gdd.DRIFT_OVERLAP_THRESHOLD, 1.0)
class TestCOG5WiringInUnifiedLoop(unittest.TestCase):
@classmethod
def setUpClass(cls):
ul_path = os.path.join(_BACKEND, "agents", "unified_loop.py")
if not os.path.exists(ul_path):
raise unittest.SkipTest("agents/unified_loop.py non trovato")
with open(ul_path, encoding="utf-8") as fh:
cls.src = fh.read()
def test_cog5_last_check_initialized(self):
"""COG-5 wiring: _cog5_last_check inizializzato nel loop."""
self.assertIn("_cog5_last_check", self.src)
def test_goal_drift_detector_imported(self):
"""COG-5 wiring: goal_drift_detector importato in unified_loop."""
self.assertIn("goal_drift_detector", self.src)
def test_detect_drift_called(self):
"""COG-5 wiring: detect_drift (o alias) chiamato in unified_loop."""
self.assertIn("detect_drift", self.src)
def test_cog5_uses_state_goal(self):
"""COG-5 wiring: detect_drift riceve state.goal come goal originale."""
idx = self.src.find("goal_drift_detector")
self.assertGreater(idx, 0)
block = self.src[idx: idx + 800]
self.assertIn("state.goal", block)
def test_cog5_uses_exec_done(self):
"""COG-5 wiring: detect_drift riceve exec_done come lista risultati."""
idx = self.src.find("goal_drift_detector")
self.assertGreater(idx, 0)
block = self.src[idx: idx + 800]
self.assertIn("exec_done", block)
def test_cog5_drift_appends_to_exec_warn(self):
"""COG-5 wiring: in caso di drift, il messaggio viene aggiunto a exec_warn."""
idx = self.src.find("goal_drift_detector")
self.assertGreater(idx, 0)
block = self.src[idx: idx + 800]
self.assertIn("exec_warn.append", block)
def test_cog5_is_non_blocking(self):
"""COG-5 wiring: detect_drift è dentro try/except (non-blocking)."""
idx = self.src.find("goal_drift_detector")
self.assertGreater(idx, 0)
# La try/except deve precedere l'import
pre_block = self.src[max(0, idx - 200): idx + 800]
self.assertIn("except Exception as _cog5_err", pre_block)
def test_cog5_marker_in_source(self):
"""COG-5 wiring: marker COG-5 presente in unified_loop."""
self.assertIn("COG-5", self.src)
# ==============================================================================
# BUG-6: tool_generator _BLOCKED_RE open() write-mode bypass regression
# ==============================================================================
class TestBUG6BlockedReOpenBypass(unittest.TestCase):
"""
Regressione per BUG-6: _BLOCKED_RE non bloccava open() con modalità
write binaria o con modificatori ('wb', 'w+', 'ab', 'wb+').
Fix: [wa][+bt]* invece di [wa] nel pattern open().
"""
@classmethod
def setUpClass(cls):
try:
import agents.tool_generator as tg
cls.tg = tg
except ImportError as e:
raise unittest.SkipTest(f"agents.tool_generator non importabile: {e}")
def _is_blocked(self, code: str) -> bool:
"""True se il codice è bloccato da _BLOCKED_RE."""
return bool(self.tg._BLOCKED_RE.search(code))
# ── Write-mode bypassati prima del fix ───────────────────────────────
def test_blocks_open_write_binary_wb(self):
"""BUG-6 regression: open('file', 'wb') deve essere bloccato."""
self.assertTrue(
self._is_blocked("open('output.bin', 'wb')"),
"_BLOCKED_RE deve bloccare open() in modalità 'wb'",
)
def test_blocks_open_write_plus(self):
"""BUG-6 regression: open('file', 'w+') deve essere bloccato."""
self.assertTrue(
self._is_blocked("open('data.txt', 'w+')"),
"_BLOCKED_RE deve bloccare open() in modalità 'w+'",
)
def test_blocks_open_append_binary_ab(self):
"""BUG-6 regression: open('file', 'ab') deve essere bloccato."""
self.assertTrue(
self._is_blocked("open('log.bin', 'ab')"),
"_BLOCKED_RE deve bloccare open() in modalità 'ab'",
)
def test_blocks_open_write_binary_plus_wbplus(self):
"""BUG-6 regression: open('file', 'wb+') deve essere bloccato."""
self.assertTrue(
self._is_blocked("open('f', 'wb+')"),
"_BLOCKED_RE deve bloccare open() in modalità 'wb+'",
)
# ── Modalità lecite non devono essere bloccate ───────────────────────
def test_allows_open_read_mode(self):
"""BUG-6: open('file', 'r') NON deve essere bloccato (read-only)."""
self.assertFalse(
self._is_blocked("open('data.csv', 'r')"),
"_BLOCKED_RE non deve bloccare open() in modalità 'r'",
)
def test_allows_open_read_binary(self):
"""BUG-6: open('file', 'rb') NON deve essere bloccato (read binary)."""
self.assertFalse(
self._is_blocked("open('data.bin', 'rb')"),
"_BLOCKED_RE non deve bloccare open() in modalità 'rb'",
)
# ── Pattern storici (non-regressione) ────────────────────────────────
def test_still_blocks_open_write_single(self):
"""BUG-6 non-regressione: open('file', 'w') originale ancora bloccato."""
self.assertTrue(
self._is_blocked("open('out.txt', 'w')"),
"_BLOCKED_RE deve ancora bloccare open() in modalità 'w'",
)
def test_still_blocks_import_requests(self):
"""BUG-6 non-regressione: import requests ancora bloccato."""
self.assertTrue(
self._is_blocked("import requests"),
"_BLOCKED_RE deve ancora bloccare import requests",
)
def test_still_blocks_eval(self):
"""BUG-6 non-regressione: eval() ancora bloccato."""
self.assertTrue(
self._is_blocked("eval('1+1')"),
"_BLOCKED_RE deve ancora bloccare eval()",
)
# ============================================================================
# BUG-10: unified_loop_tools.py _REAL_DATA_PREFIXES had "[PAGINA WEB" but
# _t_read_page() returns "[PAGINA REALE: {url}]" -- prefix mismatch
# means every successful read_page was counted as an error.
# PFX-1..PFX-5 regression tests.
# ============================================================================
class TestBUG10ReadPagePrefixMismatch(unittest.TestCase):
"""
BUG-10: _REAL_DATA_PREFIXES contained "[PAGINA WEB" but _t_read_page()
returns strings starting with "[PAGINA REALE: {url}]".
str.startswith("[PAGINA WEB") always False for those strings -> every
successful read_page was counted as an error, not a success.
Fix: replace "[PAGINA WEB" with "[PAGINA REALE" in _REAL_DATA_PREFIXES.
"""
_OLD_PREFIXES = (
"[RICERCA WEB REALE", "[METEO", "[CALCOLO REALE",
"[IMMAGINE AI GENERATA", "[CODICE PYTHON ESEGUITO",
"[PAGINA WEB",
"[DATI REALI", "[RICERCA APPROFONDITA", "[STRUTTURA PROGETTO",
"[FILE TROVATI", "[NOTIZIE", "[STATO GIT",
)
_FIXED_PREFIXES = (
"[RICERCA WEB REALE", "[METEO", "[CALCOLO REALE",
"[IMMAGINE AI GENERATA", "[CODICE PYTHON ESEGUITO",
"[PAGINA REALE",
"[DATI REALI", "[RICERCA APPROFONDITA", "[STRUTTURA PROGETTO",
"[FILE TROVATI", "[NOTIZIE", "[STATO GIT",
)
_SAMPLE = "[PAGINA REALE: https://example.com]\n(status 200)\nContent here"
# PFX-1: old prefix does not match read_page output
def test_pfx1_old_prefix_does_not_match(self):
"""PFX-1: [PAGINA WEB] does NOT match [PAGINA REALE:...] -- the old bug."""
self.assertFalse(
self._SAMPLE.startswith("[PAGINA WEB"),
"[PAGINA WEB] must NOT match read_page output (demonstrates bug)",
)
# PFX-2: fixed prefix matches read_page output
def test_pfx2_fixed_prefix_matches(self):
"""PFX-2: [PAGINA REALE] DOES match [PAGINA REALE:...] -- fix correct."""
self.assertTrue(
self._SAMPLE.startswith("[PAGINA REALE"),
"[PAGINA REALE] must match read_page output",
)
# PFX-3: n_success = 0 with old broken prefix set
def test_pfx3_old_prefix_gives_zero_success(self):
"""PFX-3: old prefix set => n_success=0 for a read_page result (the bug)."""
results = [self._SAMPLE]
n = sum(1 for r in results if any(r.startswith(p) for p in self._OLD_PREFIXES))
self.assertEqual(n, 0, "Old [PAGINA WEB] prefix => n_success=0 (bug confirmed)")
# PFX-4: n_success = 1 with fixed prefix set
def test_pfx4_fixed_prefix_gives_correct_success(self):
"""PFX-4: fixed prefix set => n_success=1 for a read_page result."""
results = [self._SAMPLE]
n = sum(1 for r in results if any(r.startswith(p) for p in self._FIXED_PREFIXES))
self.assertEqual(n, 1, "Fixed [PAGINA REALE] => n_success=1")
# PFX-5: all other tool prefixes still match correctly (non-regression)
def test_pfx5_other_prefixes_non_regression(self):
"""PFX-5: non-regression -- all other tool prefixes still work."""
cases = [
("[RICERCA WEB REALE: 'AI']\nresult", True),
("[METEO REALE -- Roma]\n22C", True),
("[CALCOLO REALE]\n2+2=4", True),
("[IMMAGINE AI GENERATA]\nURL: https://img", True),
("[CODICE PYTHON ESEGUITO]\noutput", True),
("[PAGINA REALE: https://x.com]\ncontent", True),
("[STRUTTURA PROGETTO: '.']\nfiles", True),
("[FILE TROVATI: pattern='*.py', 3]\nmatches", True),
("[NOTIZIE: 'AI']\narticle", True),
("[STATO GIT (branch: main)\nmodified: x", True),
("[web_search: NESSUN_RISULTATO]", False),
("[read_page: errore -- conn refused]", False),
("[git_status: timeout 8s]", False),
]
for output, expected in cases:
result = any(output.startswith(p) for p in self._FIXED_PREFIXES)
self.assertEqual(
result, expected,
"Prefix check failed for: %r" % output[:60],
)
if __name__ == "__main__":
unittest.main(verbosity=2)