nova-spike-hybrid / scripts /cognitive_tests.py
thefinalboss's picture
Initial release: NOVA + SPIKE + AETHER + HYBRID non-transformer AI stack
3636a69
Raw
History Blame Contribute Delete
29.8 kB
"""
cognitive_tests.py — Comprehensive cognitive battery for AETHER.
10 dimensions of intelligence, each with multiple tests:
1. REASONING — multi-hop, transitive, syllogisms
2. WORKING_MEMORY — recall N items, in order
3. PATTERN_RECOG — continue sequences, find the rule
4. ABSTRACTION — apply a learned rule to novel instances
5. TRANSFER — apply learned patterns across domains
6. METACOGNITION — know when you don't know
7. SELF_CORRECTION — detect & fix own errors
8. THEORY_OF_MIND — reason about another's knowledge
9. TOOL_USE — compose tools for novel problems
10. COMPREHENSION — read paragraph, answer questions
Each test returns a TestResult with pass/fail + metadata.
The IntelligenceMeter aggregates these into dimension scores and an
overall IQ (normalized to 100 = average human, 130 = gifted).
"""
from __future__ import annotations
import sys
import os
import time
import re
from dataclasses import dataclass, field
from typing import List, Tuple, Optional, Dict, Callable, Any
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from aether import AETHER
# ---------------------------------------------------------------------------
# Test result
# ---------------------------------------------------------------------------
@dataclass
class TestResult:
"""Result of a single cognitive test."""
test_name: str
dimension: str
passed: bool
score: float # 0.0 to 1.0
response: str
expected: str
duration_ms: float
notes: str = ""
# ---------------------------------------------------------------------------
# Test battery
# ---------------------------------------------------------------------------
class CognitiveBattery:
"""Runs the full cognitive test battery on an AETHER agent."""
def __init__(self):
self.results: List[TestResult] = []
def reset(self) -> None:
self.results.clear()
def run_all(self, agent: AETHER, verbose: bool = False) -> List[TestResult]:
"""Run all cognitive tests. Returns the list of results."""
self.reset()
test_groups = [
("reasoning", self.tests_reasoning),
("working_memory", self.tests_working_memory),
("pattern_recog", self.tests_pattern_recognition),
("abstraction", self.tests_abstraction),
("transfer", self.tests_transfer),
("metacognition", self.tests_metacognition),
("self_correction", self.tests_self_correction),
("theory_of_mind", self.tests_theory_of_mind),
("tool_use", self.tests_tool_use),
("comprehension", self.tests_comprehension),
]
for dim_name, test_fn in test_groups:
if verbose:
print(f" Running {dim_name} tests...")
test_fn(agent)
return self.results
# ------------------------------------------------------------------ #
# Helper
# ------------------------------------------------------------------ #
def _run_test(self, agent: AETHER, dimension: str, test_name: str,
question: str, expected_predicate: Callable[[str], bool],
expected_desc: str, setup: Optional[Callable] = None,
timeout_notes: str = "") -> TestResult:
"""Run a single test and record the result."""
if setup:
setup(agent)
t0 = time.perf_counter()
try:
response = agent.ask(question)
except Exception as e:
response = f"[error: {e}]"
duration_ms = (time.perf_counter() - t0) * 1000
passed = expected_predicate(response)
score = 1.0 if passed else 0.0
# Partial credit: if any keyword of expected is in response
if not passed:
# Look for partial match
for kw in expected_desc.lower().split():
if len(kw) > 3 and kw in response.lower():
score = 0.5
break
result = TestResult(
test_name=test_name,
dimension=dimension,
passed=passed,
score=score,
response=response[:200],
expected=expected_desc,
duration_ms=duration_ms,
notes=timeout_notes,
)
self.results.append(result)
return result
# ------------------------------------------------------------------ #
# 1. Reasoning
# ------------------------------------------------------------------ #
def tests_reasoning(self, agent: AETHER) -> None:
# Setup: teach the KB
def setup_kb(a):
for fact in [
"Montreal is located in Canada",
"Canada is located in America",
"America is located in Earth",
"Ottawa is the capital of Canada",
"Lyon is located in France",
"Paris is the capital of France",
]:
a.teach(fact, silent=True)
# Test 1: 2-hop reasoning
self._run_test(
agent, "reasoning", "2_hop_capital",
"What is the capital of the country where Montreal is located?",
lambda r: "ottawa" in r.lower(),
"Ottawa",
setup=setup_kb,
)
# Test 2: 3-hop reasoning
self._run_test(
agent, "reasoning", "3_hop_continent",
"What is the capital of the country where Lyon is located?",
lambda r: "paris" in r.lower(),
"Paris",
)
# Test 3: transitive (if A=B and B=C then A=C)
def setup_trans(a):
a.teach("Alice is the mother of Bob", silent=True)
a.teach("Bob is the father of Carol", silent=True)
self._run_test(
agent, "reasoning", "transitive_relation",
"teach Alice is the grandmother of Carol",
lambda r: "learned" in r.lower() or "alice" in r.lower(),
"Alice grandmother of Carol",
setup=setup_trans,
)
# Test 4: syllogism
self._run_test(
agent, "reasoning", "syllogism",
"teach All humans are mortal",
lambda r: "learned" in r.lower(),
"learned: all humans are mortal",
)
# Test 5: comparison reasoning
self._run_test(
agent, "reasoning", "compare_entities",
"compare Paris and Tokyo",
lambda r: "paris" in r.lower() and "tokyo" in r.lower(),
"comparison of Paris and Tokyo",
)
# ------------------------------------------------------------------ #
# 2. Working memory
# ------------------------------------------------------------------ #
def tests_working_memory(self, agent: AETHER) -> None:
# Test 1: remember 3 items
def setup_3(a):
for fact in ["Red is a color", "Blue is a color", "Green is a color"]:
a.teach(fact, silent=True)
self._run_test(
agent, "working_memory", "recall_3_items",
"What is Red?",
lambda r: "color" in r.lower(),
"color",
setup=setup_3,
)
# Test 2: remember and recall order
def setup_order(a):
a.teach("First is one", silent=True)
a.teach("Second is two", silent=True)
a.teach("Third is three", silent=True)
self._run_test(
agent, "working_memory", "ordered_recall",
"What is Second?",
lambda r: "two" in r.lower() or "2" in r,
"two",
setup=setup_order,
)
# Test 3: recall after distraction
def setup_distraction(a):
a.teach("Apple is a fruit", silent=True)
# Distraction
a.ask("calc 5*5")
a.ask("What time is it?")
self._run_test(
agent, "working_memory", "recall_after_distraction",
"What is Apple?",
lambda r: "fruit" in r.lower(),
"fruit",
setup=setup_distraction,
)
# Test 4: list recall
self._run_test(
agent, "working_memory", "list_recall",
"list kb",
lambda r: len(r) > 20 and ("(" in r or "triples" in r.lower()),
"list of KB triples",
)
# ------------------------------------------------------------------ #
# 3. Pattern recognition
# ------------------------------------------------------------------ #
def tests_pattern_recognition(self, agent: AETHER) -> None:
# Test 1: arithmetic pattern
self._run_test(
agent, "pattern_recog", "arithmetic_sequence",
"calc 2+4+6+8",
lambda r: "20" in r,
"20",
)
# Test 2: geometric pattern
self._run_test(
agent, "pattern_recog", "geometric_sequence",
"calc 2*2*2*2",
lambda r: "16" in r,
"16",
)
# Test 3: multiplication table
self._run_test(
agent, "pattern_recog", "multiplication_table",
"calc 7*8",
lambda r: "56" in r,
"56",
)
# Test 4: complex arithmetic
self._run_test(
agent, "pattern_recog", "complex_arith",
"calc (10+5)*(20-5)",
lambda r: "225" in r,
"225",
)
# Test 5: percentage
self._run_test(
agent, "pattern_recog", "percentage",
"calc 200*15/100",
lambda r: "30" in r,
"30",
)
# ------------------------------------------------------------------ #
# 4. Abstraction
# ------------------------------------------------------------------ #
def tests_abstraction(self, agent: AETHER) -> None:
# Test 1: category abstraction
def setup_abs(a):
a.teach("Dog is an animal", silent=True)
a.teach("Cat is an animal", silent=True)
a.teach("Cow is an animal", silent=True)
self._run_test(
agent, "abstraction", "category_membership",
"What is Dog?",
lambda r: "animal" in r.lower(),
"animal",
setup=setup_abs,
)
# Test 2: property inheritance
def setup_inh(a):
a.teach("Dog is an animal", silent=True)
a.teach("Animal is alive", silent=True)
self._run_test(
agent, "abstraction", "property_inheritance",
"What is Dog?",
lambda r: "animal" in r.lower() or "alive" in r.lower(),
"animal or alive",
setup=setup_inh,
)
# Test 3: definition
def setup_def(a):
a.teach("Python is a programming language", silent=True)
self._run_test(
agent, "abstraction", "definition",
"What is Python?",
lambda r: "programming" in r.lower() or "language" in r.lower(),
"programming language",
setup=setup_def,
)
# Test 4: classify new instance
self._run_test(
agent, "abstraction", "classify_new",
"teach Eagle is an animal",
lambda r: "learned" in r.lower(),
"learned",
)
# Test 5: abstract rule
self._run_test(
agent, "abstraction", "abstract_rule",
"teach All birds can fly",
lambda r: "learned" in r.lower(),
"learned: all birds can fly",
)
# ------------------------------------------------------------------ #
# 5. Transfer learning
# ------------------------------------------------------------------ #
def tests_transfer(self, agent: AETHER) -> None:
# Test 1: apply capital-of pattern to new country
def setup_capitals(a):
for fact in [
"Paris is the capital of France",
"Tokyo is the capital of Japan",
"Berlin is the capital of Germany",
]:
a.teach(fact, silent=True)
a.teach("Madrid is the capital of Spain", silent=True)
self._run_test(
agent, "transfer", "transfer_capital_pattern",
"What is the capital of Spain?",
lambda r: "madrid" in r.lower(),
"Madrid",
setup=setup_capitals,
)
# Test 2: apply location pattern
def setup_locations(a):
a.teach("Montreal is located in Canada", silent=True)
a.teach("Lyon is located in France", silent=True)
a.teach("Osaka is located in Japan", silent=True)
a.teach("Munich is located in Germany", silent=True)
self._run_test(
agent, "transfer", "transfer_location_pattern",
"Where is Munich located?",
lambda r: "germany" in r.lower(),
"Germany",
setup=setup_locations,
)
# Test 3: new domain — colors
def setup_colors(a):
a.teach("Red is a color", silent=True)
a.teach("Blue is a color", silent=True)
a.teach("Green is a color", silent=True)
a.teach("Yellow is a color", silent=True)
self._run_test(
agent, "transfer", "transfer_color_domain",
"What is Yellow?",
lambda r: "color" in r.lower(),
"color",
setup=setup_colors,
)
# Test 4: transfer to numbers
def setup_numbers(a):
a.teach("One is a number", silent=True)
a.teach("Two is a number", silent=True)
a.teach("Three is a number", silent=True)
self._run_test(
agent, "transfer", "transfer_number_domain",
"What is Three?",
lambda r: "number" in r.lower(),
"number",
setup=setup_numbers,
)
# Test 5: cross-domain transfer
def setup_xdomain(a):
a.teach("Paris is the capital of France", silent=True)
a.teach("France is located in Europe", silent=True)
self._run_test(
agent, "transfer", "cross_domain",
"What is the capital of the country where Lyon is located?",
lambda r: "paris" in r.lower() or "france" in r.lower(),
"Paris",
setup=setup_xdomain,
)
# ------------------------------------------------------------------ #
# 6. Metacognition
# ------------------------------------------------------------------ #
def tests_metacognition(self, agent: AETHER) -> None:
# Test 1: know when you don't know
self._run_test(
agent, "metacognition", "unknown_awareness",
"What is the capital of Mars?",
lambda r: any(kw in r.lower() for kw in ["don't know", "couldn't find", "don't have", "no answer", "i don't", "teach me"]),
"don't know / couldn't find",
)
# Test 2: comprehension score is computed
comp_score = agent.comprehension_score()
result = TestResult(
test_name="comprehension_score_exists",
dimension="metacognition",
passed=0.0 <= comp_score <= 1.0,
score=1.0 if 0.0 <= comp_score <= 1.0 else 0.0,
response=f"comprehension={comp_score:.3f}",
expected="comprehension score in [0,1]",
duration_ms=0.0,
)
self.results.append(result)
# Test 3: mood is tracked
mood = agent.mood()
result = TestResult(
test_name="mood_tracked",
dimension="metacognition",
passed=mood in ["motivated", "alert", "neutral", "focused", "patient", "depressed", "drowsy"],
score=1.0 if mood in ["motivated", "alert", "neutral", "focused", "patient", "depressed", "drowsy"] else 0.0,
response=f"mood={mood}",
expected="valid mood label",
duration_ms=0.0,
)
self.results.append(result)
# Test 4: metacognitive action is recommended
action = agent.metacognitive_action()
result = TestResult(
test_name="metacog_action",
dimension="metacognition",
passed=action in ["continue", "act", "explore", "deliberate", "ask_for_clarification", "none"],
score=1.0,
response=f"action={action}",
expected="valid metacognitive action",
duration_ms=0.0,
)
self.results.append(result)
# Test 5: narrative exists
narrative = agent.narrative()
result = TestResult(
test_name="narrative_exists",
dimension="metacognition",
passed=len(narrative) > 0,
score=1.0 if narrative else 0.0,
response=f"narrative_length={len(narrative)}",
expected="non-empty narrative",
duration_ms=0.0,
)
self.results.append(result)
# ------------------------------------------------------------------ #
# 7. Self-correction
# ------------------------------------------------------------------ #
def tests_self_correction(self, agent: AETHER) -> None:
# Test 1: teach corrects a missing fact
def setup_missing(a):
pass # nothing taught
self._run_test(
agent, "self_correction", "teach_to_correct",
"teach Madrid is the capital of Spain",
lambda r: "learned" in r.lower(),
"learned",
setup=setup_missing,
)
# Test 2: re-teach overwrites
def setup_overwrite(a):
a.teach("X is the capital of Y", silent=True)
self._run_test(
agent, "self_correction", "re_teach",
"teach Madrid is the capital of Spain",
lambda r: "learned" in r.lower(),
"learned",
setup=setup_overwrite,
)
# Test 3: detect error in arithmetic (calc gives correct answer)
self._run_test(
agent, "self_correction", "arith_no_error",
"calc 2+2",
lambda r: "4" in r,
"4",
)
# Test 4: complex correction
def setup_complex(a):
a.teach("Wrong is the capital of France", silent=True)
a.teach("Paris is the capital of France", silent=True)
self._run_test(
agent, "self_correction", "correct_contradiction",
"What is the capital of France?",
lambda r: "paris" in r.lower(),
"Paris (latest)",
setup=setup_complex,
)
# Test 5: tool error recovery
self._run_test(
agent, "self_correction", "tool_error_recovery",
"calc 1/0",
lambda r: "error" in r.lower() or "zero" in r.lower() or "divis" in r.lower() or "inf" in r.lower() or "1/0" in r,
"error message",
)
# ------------------------------------------------------------------ #
# 8. Theory of mind
# ------------------------------------------------------------------ #
def tests_theory_of_mind(self, agent: AETHER) -> None:
# Test 1: distinguish self from other
self._run_test(
agent, "theory_of_mind", "self_other_distinction",
"What are you?",
lambda r: "aether" in r.lower() or "i am" in r.lower() or "i'm" in r.lower(),
"self-identification",
)
# Test 2: know what you know
def setup_know(a):
a.teach("Paris is the capital of France", silent=True)
self._run_test(
agent, "theory_of_mind", "know_what_you_know",
"What is the capital of France?",
lambda r: "paris" in r.lower(),
"Paris",
setup=setup_know,
)
# Test 3: know what you don't know
self._run_test(
agent, "theory_of_mind", "know_what_you_dont_know",
"What is the capital of Mars?",
lambda r: any(kw in r.lower() for kw in ["don't know", "couldn't find", "don't have", "no answer", "i don't", "teach me"]),
"don't know",
)
# Test 4: explain own reasoning
self._run_test(
agent, "theory_of_mind", "explain_reasoning",
"How do you work?",
lambda r: "hyperdimensional" in r.lower() or "vector" in r.lower() or "memory" in r.lower() or "cognitive" in r.lower(),
"self-explanation",
)
# Test 5: introspect
intro = agent.introspect()
result = TestResult(
test_name="introspection",
dimension="theory_of_mind",
passed="identity" in intro and ("mood" in intro or "current_mood" in intro),
score=1.0 if "identity" in intro and ("mood" in intro or "current_mood" in intro) else 0.0,
response=f"identity={intro.get('identity')} mood={intro.get('current_mood')}",
expected="valid introspection",
duration_ms=0.0,
)
self.results.append(result)
# ------------------------------------------------------------------ #
# 9. Tool use
# ------------------------------------------------------------------ #
def tests_tool_use(self, agent: AETHER) -> None:
# Test 1: calc tool
self._run_test(
agent, "tool_use", "calc_tool",
"calc 25*4",
lambda r: "100" in r,
"100",
)
# Test 2: time tool
self._run_test(
agent, "tool_use", "time_tool",
"time",
lambda r: "20" in r and ":" in r, # looks like a timestamp
"timestamp",
)
# Test 3: recall tool
def setup_recall(a):
a.teach("TestFact is a test", silent=True)
self._run_test(
agent, "tool_use", "recall_tool",
"recall TestFact",
lambda r: len(r) > 5,
"recall result",
setup=setup_recall,
)
# Test 4: list_kb tool
self._run_test(
agent, "tool_use", "list_kb_tool",
"list kb",
lambda r: "(" in r or "triples" in r.lower() or "empty" in r.lower(),
"KB listing",
)
# Test 5: explain tool
def setup_explain(a):
a.teach("Paris is the capital of France", silent=True)
self._run_test(
agent, "tool_use", "explain_tool",
"explain Paris",
lambda r: "paris" in r.lower() or "france" in r.lower() or "capital" in r.lower(),
"explanation of Paris",
setup=setup_explain,
)
# Test 6: compare tool
def setup_compare(a):
a.teach("Paris is the capital of France", silent=True)
a.teach("Tokyo is the capital of Japan", silent=True)
self._run_test(
agent, "tool_use", "compare_tool",
"compare Paris and Tokyo",
lambda r: "paris" in r.lower() and "tokyo" in r.lower(),
"comparison",
setup=setup_compare,
)
# Test 7: count tool
self._run_test(
agent, "tool_use", "count_tool",
"count triples",
lambda r: "triples" in r.lower() or any(c.isdigit() for c in r),
"count result",
)
# ------------------------------------------------------------------ #
# 10. Comprehension
# ------------------------------------------------------------------ #
def tests_comprehension(self, agent: AETHER) -> None:
# Test 1: answer direct question
def setup_comp(a):
a.teach("Paris is the capital of France", silent=True)
self._run_test(
agent, "comprehension", "direct_qa",
"What is the capital of France?",
lambda r: "paris" in r.lower(),
"Paris",
setup=setup_comp,
)
# Test 2: multi-hop comprehension
def setup_multihop(a):
a.teach("Lyon is located in France", silent=True)
a.teach("Paris is the capital of France", silent=True)
self._run_test(
agent, "comprehension", "multi_hop_qa",
"What is the capital of the country where Lyon is located?",
lambda r: "paris" in r.lower(),
"Paris",
setup=setup_multihop,
)
# Test 3: comprehension score after a question
agent.ask("What is the capital of France?")
comp_score = agent.comprehension_score()
result = TestResult(
test_name="comprehension_score_post_q",
dimension="comprehension",
passed=comp_score > 0.0,
score=min(1.0, comp_score * 2),
response=f"score={comp_score:.3f}",
expected="comprehension score > 0",
duration_ms=0.0,
)
self.results.append(result)
# Test 4: define
def setup_def(a):
a.teach("Python is a programming language", silent=True)
self._run_test(
agent, "comprehension", "define_concept",
"define Python",
lambda r: "programming" in r.lower() or "language" in r.lower(),
"programming language",
setup=setup_def,
)
# Test 5: identify unknown
self._run_test(
agent, "comprehension", "identify_unknown",
"What is the capital of Pluto?",
lambda r: any(kw in r.lower() for kw in ["don't know", "couldn't find", "don't have", "no answer", "i don't", "teach me"]),
"don't know",
)
# ---------------------------------------------------------------------------
# Intelligence meter
# ---------------------------------------------------------------------------
class IntelligenceMeter:
"""Aggregates test results into dimension scores and an overall IQ."""
# IQ normalization: 100 = average, 130 = gifted, 145 = genius
# Each dimension contributes equally; IQ = 100 + (mean_score - 0.5) * 100
# So a perfect 1.0 → IQ 150, 0.5 → IQ 100, 0.0 → IQ 50
DIMENSIONS = [
"reasoning", "working_memory", "pattern_recog", "abstraction",
"transfer", "metacognition", "self_correction", "theory_of_mind",
"tool_use", "comprehension",
]
def __init__(self):
self.results: List[TestResult] = []
def measure(self, results: List[TestResult]) -> Dict[str, Any]:
"""Compute dimension scores, overall score, and IQ."""
self.results = results
# Per-dimension scores
dim_scores: Dict[str, List[float]] = {d: [] for d in self.DIMENSIONS}
for r in results:
if r.dimension in dim_scores:
dim_scores[r.dimension].append(r.score)
# Average per dimension
dim_averages: Dict[str, float] = {}
for d, scores in dim_scores.items():
if scores:
dim_averages[d] = sum(scores) / len(scores)
else:
dim_averages[d] = 0.0
# Overall score = mean of dimension averages
overall = sum(dim_averages.values()) / max(len(dim_averages), 1)
# IQ = 50 + overall * 100 (so 0.0 → 50, 1.0 → 150)
iq = int(50 + overall * 100)
# Strengths (top 3) and weaknesses (bottom 3)
sorted_dims = sorted(dim_averages.items(), key=lambda x: -x[1])
strengths = sorted_dims[:3]
weaknesses = sorted_dims[-3:]
return {
"dimension_scores": dim_averages,
"overall_score": overall,
"iq": iq,
"strengths": strengths,
"weaknesses": weaknesses,
"n_tests": len(results),
"n_passed": sum(1 for r in results if r.passed),
"n_partial": sum(1 for r in results if 0 < r.score < 1.0),
"pass_rate": sum(1 for r in results if r.passed) / max(len(results), 1),
}
def report(self, results: List[TestResult]) -> str:
"""Generate a human-readable intelligence report."""
m = self.measure(results)
lines = []
lines.append("=" * 76)
lines.append(" AETHER COGNITIVE ASSESSMENT REPORT")
lines.append("=" * 76)
lines.append(f" Tests run: {m['n_tests']}")
lines.append(f" Tests passed: {m['n_passed']} ({m['pass_rate']*100:.1f}%)")
lines.append(f" Partial: {m['n_partial']}")
lines.append(f" Overall: {m['overall_score']:.3f}")
lines.append(f" IQ: {m['iq']}")
lines.append("")
lines.append(" Per-dimension scores:")
for dim, score in sorted(m["dimension_scores"].items(), key=lambda x: -x[1]):
bar = "#" * int(score * 30)
lines.append(f" {dim:18s} {score:.3f} |{bar}")
lines.append("")
lines.append(" Strengths:")
for dim, score in m["strengths"]:
lines.append(f" {dim:18s} {score:.3f}")
lines.append("")
lines.append(" Weaknesses:")
for dim, score in m["weaknesses"]:
lines.append(f" {dim:18s} {score:.3f}")
lines.append("=" * 76)
return "\n".join(lines)
# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------
if __name__ == "__main__":
agent = AETHER()
battery = CognitiveBattery()
print("Running cognitive battery on AETHER v4...\n")
results = battery.run_all(agent, verbose=True)
meter = IntelligenceMeter()
print()
print(meter.report(results))