Text Generation
PyTorch
English
French
hyperdimensional-computing
spiking-neural-networks
hdc
snn
lif
stdp
r-stdp
brain-inspired
cognitive-architecture
agentic
cpu-only
no-transformer
no-gpu
non-transformer
sparse-distributed-memory
kanerva
attractor-networks
global-workspace-theory
predictive-coding
neuromodulators
consciousness
kuramoto
vector-symbolic-architecture
vsa
one-shot-learning
instant-learning
pure-python
numpy
scipy
fastapi
web-dashboard
multi-modal
bpe
benchmark
beam-search
attention
reinforcement-learning
n-gram
kneser-ney
generative-ai
reasoning
creative-writing
research
prototype
| """ | |
| cognitive_tests.py — Comprehensive cognitive battery for AETHER. | |
| 10 dimensions of intelligence, each with multiple tests: | |
| 1. REASONING — multi-hop, transitive, syllogisms | |
| 2. WORKING_MEMORY — recall N items, in order | |
| 3. PATTERN_RECOG — continue sequences, find the rule | |
| 4. ABSTRACTION — apply a learned rule to novel instances | |
| 5. TRANSFER — apply learned patterns across domains | |
| 6. METACOGNITION — know when you don't know | |
| 7. SELF_CORRECTION — detect & fix own errors | |
| 8. THEORY_OF_MIND — reason about another's knowledge | |
| 9. TOOL_USE — compose tools for novel problems | |
| 10. COMPREHENSION — read paragraph, answer questions | |
| Each test returns a TestResult with pass/fail + metadata. | |
| The IntelligenceMeter aggregates these into dimension scores and an | |
| overall IQ (normalized to 100 = average human, 130 = gifted). | |
| """ | |
| from __future__ import annotations | |
| import sys | |
| import os | |
| import time | |
| import re | |
| from dataclasses import dataclass, field | |
| from typing import List, Tuple, Optional, Dict, Callable, Any | |
| sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) | |
| from aether import AETHER | |
| # --------------------------------------------------------------------------- | |
| # Test result | |
| # --------------------------------------------------------------------------- | |
| class TestResult: | |
| """Result of a single cognitive test.""" | |
| test_name: str | |
| dimension: str | |
| passed: bool | |
| score: float # 0.0 to 1.0 | |
| response: str | |
| expected: str | |
| duration_ms: float | |
| notes: str = "" | |
| # --------------------------------------------------------------------------- | |
| # Test battery | |
| # --------------------------------------------------------------------------- | |
| class CognitiveBattery: | |
| """Runs the full cognitive test battery on an AETHER agent.""" | |
| def __init__(self): | |
| self.results: List[TestResult] = [] | |
| def reset(self) -> None: | |
| self.results.clear() | |
| def run_all(self, agent: AETHER, verbose: bool = False) -> List[TestResult]: | |
| """Run all cognitive tests. Returns the list of results.""" | |
| self.reset() | |
| test_groups = [ | |
| ("reasoning", self.tests_reasoning), | |
| ("working_memory", self.tests_working_memory), | |
| ("pattern_recog", self.tests_pattern_recognition), | |
| ("abstraction", self.tests_abstraction), | |
| ("transfer", self.tests_transfer), | |
| ("metacognition", self.tests_metacognition), | |
| ("self_correction", self.tests_self_correction), | |
| ("theory_of_mind", self.tests_theory_of_mind), | |
| ("tool_use", self.tests_tool_use), | |
| ("comprehension", self.tests_comprehension), | |
| ] | |
| for dim_name, test_fn in test_groups: | |
| if verbose: | |
| print(f" Running {dim_name} tests...") | |
| test_fn(agent) | |
| return self.results | |
| # ------------------------------------------------------------------ # | |
| # Helper | |
| # ------------------------------------------------------------------ # | |
| def _run_test(self, agent: AETHER, dimension: str, test_name: str, | |
| question: str, expected_predicate: Callable[[str], bool], | |
| expected_desc: str, setup: Optional[Callable] = None, | |
| timeout_notes: str = "") -> TestResult: | |
| """Run a single test and record the result.""" | |
| if setup: | |
| setup(agent) | |
| t0 = time.perf_counter() | |
| try: | |
| response = agent.ask(question) | |
| except Exception as e: | |
| response = f"[error: {e}]" | |
| duration_ms = (time.perf_counter() - t0) * 1000 | |
| passed = expected_predicate(response) | |
| score = 1.0 if passed else 0.0 | |
| # Partial credit: if any keyword of expected is in response | |
| if not passed: | |
| # Look for partial match | |
| for kw in expected_desc.lower().split(): | |
| if len(kw) > 3 and kw in response.lower(): | |
| score = 0.5 | |
| break | |
| result = TestResult( | |
| test_name=test_name, | |
| dimension=dimension, | |
| passed=passed, | |
| score=score, | |
| response=response[:200], | |
| expected=expected_desc, | |
| duration_ms=duration_ms, | |
| notes=timeout_notes, | |
| ) | |
| self.results.append(result) | |
| return result | |
| # ------------------------------------------------------------------ # | |
| # 1. Reasoning | |
| # ------------------------------------------------------------------ # | |
| def tests_reasoning(self, agent: AETHER) -> None: | |
| # Setup: teach the KB | |
| def setup_kb(a): | |
| for fact in [ | |
| "Montreal is located in Canada", | |
| "Canada is located in America", | |
| "America is located in Earth", | |
| "Ottawa is the capital of Canada", | |
| "Lyon is located in France", | |
| "Paris is the capital of France", | |
| ]: | |
| a.teach(fact, silent=True) | |
| # Test 1: 2-hop reasoning | |
| self._run_test( | |
| agent, "reasoning", "2_hop_capital", | |
| "What is the capital of the country where Montreal is located?", | |
| lambda r: "ottawa" in r.lower(), | |
| "Ottawa", | |
| setup=setup_kb, | |
| ) | |
| # Test 2: 3-hop reasoning | |
| self._run_test( | |
| agent, "reasoning", "3_hop_continent", | |
| "What is the capital of the country where Lyon is located?", | |
| lambda r: "paris" in r.lower(), | |
| "Paris", | |
| ) | |
| # Test 3: transitive (if A=B and B=C then A=C) | |
| def setup_trans(a): | |
| a.teach("Alice is the mother of Bob", silent=True) | |
| a.teach("Bob is the father of Carol", silent=True) | |
| self._run_test( | |
| agent, "reasoning", "transitive_relation", | |
| "teach Alice is the grandmother of Carol", | |
| lambda r: "learned" in r.lower() or "alice" in r.lower(), | |
| "Alice grandmother of Carol", | |
| setup=setup_trans, | |
| ) | |
| # Test 4: syllogism | |
| self._run_test( | |
| agent, "reasoning", "syllogism", | |
| "teach All humans are mortal", | |
| lambda r: "learned" in r.lower(), | |
| "learned: all humans are mortal", | |
| ) | |
| # Test 5: comparison reasoning | |
| self._run_test( | |
| agent, "reasoning", "compare_entities", | |
| "compare Paris and Tokyo", | |
| lambda r: "paris" in r.lower() and "tokyo" in r.lower(), | |
| "comparison of Paris and Tokyo", | |
| ) | |
| # ------------------------------------------------------------------ # | |
| # 2. Working memory | |
| # ------------------------------------------------------------------ # | |
| def tests_working_memory(self, agent: AETHER) -> None: | |
| # Test 1: remember 3 items | |
| def setup_3(a): | |
| for fact in ["Red is a color", "Blue is a color", "Green is a color"]: | |
| a.teach(fact, silent=True) | |
| self._run_test( | |
| agent, "working_memory", "recall_3_items", | |
| "What is Red?", | |
| lambda r: "color" in r.lower(), | |
| "color", | |
| setup=setup_3, | |
| ) | |
| # Test 2: remember and recall order | |
| def setup_order(a): | |
| a.teach("First is one", silent=True) | |
| a.teach("Second is two", silent=True) | |
| a.teach("Third is three", silent=True) | |
| self._run_test( | |
| agent, "working_memory", "ordered_recall", | |
| "What is Second?", | |
| lambda r: "two" in r.lower() or "2" in r, | |
| "two", | |
| setup=setup_order, | |
| ) | |
| # Test 3: recall after distraction | |
| def setup_distraction(a): | |
| a.teach("Apple is a fruit", silent=True) | |
| # Distraction | |
| a.ask("calc 5*5") | |
| a.ask("What time is it?") | |
| self._run_test( | |
| agent, "working_memory", "recall_after_distraction", | |
| "What is Apple?", | |
| lambda r: "fruit" in r.lower(), | |
| "fruit", | |
| setup=setup_distraction, | |
| ) | |
| # Test 4: list recall | |
| self._run_test( | |
| agent, "working_memory", "list_recall", | |
| "list kb", | |
| lambda r: len(r) > 20 and ("(" in r or "triples" in r.lower()), | |
| "list of KB triples", | |
| ) | |
| # ------------------------------------------------------------------ # | |
| # 3. Pattern recognition | |
| # ------------------------------------------------------------------ # | |
| def tests_pattern_recognition(self, agent: AETHER) -> None: | |
| # Test 1: arithmetic pattern | |
| self._run_test( | |
| agent, "pattern_recog", "arithmetic_sequence", | |
| "calc 2+4+6+8", | |
| lambda r: "20" in r, | |
| "20", | |
| ) | |
| # Test 2: geometric pattern | |
| self._run_test( | |
| agent, "pattern_recog", "geometric_sequence", | |
| "calc 2*2*2*2", | |
| lambda r: "16" in r, | |
| "16", | |
| ) | |
| # Test 3: multiplication table | |
| self._run_test( | |
| agent, "pattern_recog", "multiplication_table", | |
| "calc 7*8", | |
| lambda r: "56" in r, | |
| "56", | |
| ) | |
| # Test 4: complex arithmetic | |
| self._run_test( | |
| agent, "pattern_recog", "complex_arith", | |
| "calc (10+5)*(20-5)", | |
| lambda r: "225" in r, | |
| "225", | |
| ) | |
| # Test 5: percentage | |
| self._run_test( | |
| agent, "pattern_recog", "percentage", | |
| "calc 200*15/100", | |
| lambda r: "30" in r, | |
| "30", | |
| ) | |
| # ------------------------------------------------------------------ # | |
| # 4. Abstraction | |
| # ------------------------------------------------------------------ # | |
| def tests_abstraction(self, agent: AETHER) -> None: | |
| # Test 1: category abstraction | |
| def setup_abs(a): | |
| a.teach("Dog is an animal", silent=True) | |
| a.teach("Cat is an animal", silent=True) | |
| a.teach("Cow is an animal", silent=True) | |
| self._run_test( | |
| agent, "abstraction", "category_membership", | |
| "What is Dog?", | |
| lambda r: "animal" in r.lower(), | |
| "animal", | |
| setup=setup_abs, | |
| ) | |
| # Test 2: property inheritance | |
| def setup_inh(a): | |
| a.teach("Dog is an animal", silent=True) | |
| a.teach("Animal is alive", silent=True) | |
| self._run_test( | |
| agent, "abstraction", "property_inheritance", | |
| "What is Dog?", | |
| lambda r: "animal" in r.lower() or "alive" in r.lower(), | |
| "animal or alive", | |
| setup=setup_inh, | |
| ) | |
| # Test 3: definition | |
| def setup_def(a): | |
| a.teach("Python is a programming language", silent=True) | |
| self._run_test( | |
| agent, "abstraction", "definition", | |
| "What is Python?", | |
| lambda r: "programming" in r.lower() or "language" in r.lower(), | |
| "programming language", | |
| setup=setup_def, | |
| ) | |
| # Test 4: classify new instance | |
| self._run_test( | |
| agent, "abstraction", "classify_new", | |
| "teach Eagle is an animal", | |
| lambda r: "learned" in r.lower(), | |
| "learned", | |
| ) | |
| # Test 5: abstract rule | |
| self._run_test( | |
| agent, "abstraction", "abstract_rule", | |
| "teach All birds can fly", | |
| lambda r: "learned" in r.lower(), | |
| "learned: all birds can fly", | |
| ) | |
| # ------------------------------------------------------------------ # | |
| # 5. Transfer learning | |
| # ------------------------------------------------------------------ # | |
| def tests_transfer(self, agent: AETHER) -> None: | |
| # Test 1: apply capital-of pattern to new country | |
| def setup_capitals(a): | |
| for fact in [ | |
| "Paris is the capital of France", | |
| "Tokyo is the capital of Japan", | |
| "Berlin is the capital of Germany", | |
| ]: | |
| a.teach(fact, silent=True) | |
| a.teach("Madrid is the capital of Spain", silent=True) | |
| self._run_test( | |
| agent, "transfer", "transfer_capital_pattern", | |
| "What is the capital of Spain?", | |
| lambda r: "madrid" in r.lower(), | |
| "Madrid", | |
| setup=setup_capitals, | |
| ) | |
| # Test 2: apply location pattern | |
| def setup_locations(a): | |
| a.teach("Montreal is located in Canada", silent=True) | |
| a.teach("Lyon is located in France", silent=True) | |
| a.teach("Osaka is located in Japan", silent=True) | |
| a.teach("Munich is located in Germany", silent=True) | |
| self._run_test( | |
| agent, "transfer", "transfer_location_pattern", | |
| "Where is Munich located?", | |
| lambda r: "germany" in r.lower(), | |
| "Germany", | |
| setup=setup_locations, | |
| ) | |
| # Test 3: new domain — colors | |
| def setup_colors(a): | |
| a.teach("Red is a color", silent=True) | |
| a.teach("Blue is a color", silent=True) | |
| a.teach("Green is a color", silent=True) | |
| a.teach("Yellow is a color", silent=True) | |
| self._run_test( | |
| agent, "transfer", "transfer_color_domain", | |
| "What is Yellow?", | |
| lambda r: "color" in r.lower(), | |
| "color", | |
| setup=setup_colors, | |
| ) | |
| # Test 4: transfer to numbers | |
| def setup_numbers(a): | |
| a.teach("One is a number", silent=True) | |
| a.teach("Two is a number", silent=True) | |
| a.teach("Three is a number", silent=True) | |
| self._run_test( | |
| agent, "transfer", "transfer_number_domain", | |
| "What is Three?", | |
| lambda r: "number" in r.lower(), | |
| "number", | |
| setup=setup_numbers, | |
| ) | |
| # Test 5: cross-domain transfer | |
| def setup_xdomain(a): | |
| a.teach("Paris is the capital of France", silent=True) | |
| a.teach("France is located in Europe", silent=True) | |
| self._run_test( | |
| agent, "transfer", "cross_domain", | |
| "What is the capital of the country where Lyon is located?", | |
| lambda r: "paris" in r.lower() or "france" in r.lower(), | |
| "Paris", | |
| setup=setup_xdomain, | |
| ) | |
| # ------------------------------------------------------------------ # | |
| # 6. Metacognition | |
| # ------------------------------------------------------------------ # | |
| def tests_metacognition(self, agent: AETHER) -> None: | |
| # Test 1: know when you don't know | |
| self._run_test( | |
| agent, "metacognition", "unknown_awareness", | |
| "What is the capital of Mars?", | |
| lambda r: any(kw in r.lower() for kw in ["don't know", "couldn't find", "don't have", "no answer", "i don't", "teach me"]), | |
| "don't know / couldn't find", | |
| ) | |
| # Test 2: comprehension score is computed | |
| comp_score = agent.comprehension_score() | |
| result = TestResult( | |
| test_name="comprehension_score_exists", | |
| dimension="metacognition", | |
| passed=0.0 <= comp_score <= 1.0, | |
| score=1.0 if 0.0 <= comp_score <= 1.0 else 0.0, | |
| response=f"comprehension={comp_score:.3f}", | |
| expected="comprehension score in [0,1]", | |
| duration_ms=0.0, | |
| ) | |
| self.results.append(result) | |
| # Test 3: mood is tracked | |
| mood = agent.mood() | |
| result = TestResult( | |
| test_name="mood_tracked", | |
| dimension="metacognition", | |
| passed=mood in ["motivated", "alert", "neutral", "focused", "patient", "depressed", "drowsy"], | |
| score=1.0 if mood in ["motivated", "alert", "neutral", "focused", "patient", "depressed", "drowsy"] else 0.0, | |
| response=f"mood={mood}", | |
| expected="valid mood label", | |
| duration_ms=0.0, | |
| ) | |
| self.results.append(result) | |
| # Test 4: metacognitive action is recommended | |
| action = agent.metacognitive_action() | |
| result = TestResult( | |
| test_name="metacog_action", | |
| dimension="metacognition", | |
| passed=action in ["continue", "act", "explore", "deliberate", "ask_for_clarification", "none"], | |
| score=1.0, | |
| response=f"action={action}", | |
| expected="valid metacognitive action", | |
| duration_ms=0.0, | |
| ) | |
| self.results.append(result) | |
| # Test 5: narrative exists | |
| narrative = agent.narrative() | |
| result = TestResult( | |
| test_name="narrative_exists", | |
| dimension="metacognition", | |
| passed=len(narrative) > 0, | |
| score=1.0 if narrative else 0.0, | |
| response=f"narrative_length={len(narrative)}", | |
| expected="non-empty narrative", | |
| duration_ms=0.0, | |
| ) | |
| self.results.append(result) | |
| # ------------------------------------------------------------------ # | |
| # 7. Self-correction | |
| # ------------------------------------------------------------------ # | |
| def tests_self_correction(self, agent: AETHER) -> None: | |
| # Test 1: teach corrects a missing fact | |
| def setup_missing(a): | |
| pass # nothing taught | |
| self._run_test( | |
| agent, "self_correction", "teach_to_correct", | |
| "teach Madrid is the capital of Spain", | |
| lambda r: "learned" in r.lower(), | |
| "learned", | |
| setup=setup_missing, | |
| ) | |
| # Test 2: re-teach overwrites | |
| def setup_overwrite(a): | |
| a.teach("X is the capital of Y", silent=True) | |
| self._run_test( | |
| agent, "self_correction", "re_teach", | |
| "teach Madrid is the capital of Spain", | |
| lambda r: "learned" in r.lower(), | |
| "learned", | |
| setup=setup_overwrite, | |
| ) | |
| # Test 3: detect error in arithmetic (calc gives correct answer) | |
| self._run_test( | |
| agent, "self_correction", "arith_no_error", | |
| "calc 2+2", | |
| lambda r: "4" in r, | |
| "4", | |
| ) | |
| # Test 4: complex correction | |
| def setup_complex(a): | |
| a.teach("Wrong is the capital of France", silent=True) | |
| a.teach("Paris is the capital of France", silent=True) | |
| self._run_test( | |
| agent, "self_correction", "correct_contradiction", | |
| "What is the capital of France?", | |
| lambda r: "paris" in r.lower(), | |
| "Paris (latest)", | |
| setup=setup_complex, | |
| ) | |
| # Test 5: tool error recovery | |
| self._run_test( | |
| agent, "self_correction", "tool_error_recovery", | |
| "calc 1/0", | |
| lambda r: "error" in r.lower() or "zero" in r.lower() or "divis" in r.lower() or "inf" in r.lower() or "1/0" in r, | |
| "error message", | |
| ) | |
| # ------------------------------------------------------------------ # | |
| # 8. Theory of mind | |
| # ------------------------------------------------------------------ # | |
| def tests_theory_of_mind(self, agent: AETHER) -> None: | |
| # Test 1: distinguish self from other | |
| self._run_test( | |
| agent, "theory_of_mind", "self_other_distinction", | |
| "What are you?", | |
| lambda r: "aether" in r.lower() or "i am" in r.lower() or "i'm" in r.lower(), | |
| "self-identification", | |
| ) | |
| # Test 2: know what you know | |
| def setup_know(a): | |
| a.teach("Paris is the capital of France", silent=True) | |
| self._run_test( | |
| agent, "theory_of_mind", "know_what_you_know", | |
| "What is the capital of France?", | |
| lambda r: "paris" in r.lower(), | |
| "Paris", | |
| setup=setup_know, | |
| ) | |
| # Test 3: know what you don't know | |
| self._run_test( | |
| agent, "theory_of_mind", "know_what_you_dont_know", | |
| "What is the capital of Mars?", | |
| lambda r: any(kw in r.lower() for kw in ["don't know", "couldn't find", "don't have", "no answer", "i don't", "teach me"]), | |
| "don't know", | |
| ) | |
| # Test 4: explain own reasoning | |
| self._run_test( | |
| agent, "theory_of_mind", "explain_reasoning", | |
| "How do you work?", | |
| lambda r: "hyperdimensional" in r.lower() or "vector" in r.lower() or "memory" in r.lower() or "cognitive" in r.lower(), | |
| "self-explanation", | |
| ) | |
| # Test 5: introspect | |
| intro = agent.introspect() | |
| result = TestResult( | |
| test_name="introspection", | |
| dimension="theory_of_mind", | |
| passed="identity" in intro and ("mood" in intro or "current_mood" in intro), | |
| score=1.0 if "identity" in intro and ("mood" in intro or "current_mood" in intro) else 0.0, | |
| response=f"identity={intro.get('identity')} mood={intro.get('current_mood')}", | |
| expected="valid introspection", | |
| duration_ms=0.0, | |
| ) | |
| self.results.append(result) | |
| # ------------------------------------------------------------------ # | |
| # 9. Tool use | |
| # ------------------------------------------------------------------ # | |
| def tests_tool_use(self, agent: AETHER) -> None: | |
| # Test 1: calc tool | |
| self._run_test( | |
| agent, "tool_use", "calc_tool", | |
| "calc 25*4", | |
| lambda r: "100" in r, | |
| "100", | |
| ) | |
| # Test 2: time tool | |
| self._run_test( | |
| agent, "tool_use", "time_tool", | |
| "time", | |
| lambda r: "20" in r and ":" in r, # looks like a timestamp | |
| "timestamp", | |
| ) | |
| # Test 3: recall tool | |
| def setup_recall(a): | |
| a.teach("TestFact is a test", silent=True) | |
| self._run_test( | |
| agent, "tool_use", "recall_tool", | |
| "recall TestFact", | |
| lambda r: len(r) > 5, | |
| "recall result", | |
| setup=setup_recall, | |
| ) | |
| # Test 4: list_kb tool | |
| self._run_test( | |
| agent, "tool_use", "list_kb_tool", | |
| "list kb", | |
| lambda r: "(" in r or "triples" in r.lower() or "empty" in r.lower(), | |
| "KB listing", | |
| ) | |
| # Test 5: explain tool | |
| def setup_explain(a): | |
| a.teach("Paris is the capital of France", silent=True) | |
| self._run_test( | |
| agent, "tool_use", "explain_tool", | |
| "explain Paris", | |
| lambda r: "paris" in r.lower() or "france" in r.lower() or "capital" in r.lower(), | |
| "explanation of Paris", | |
| setup=setup_explain, | |
| ) | |
| # Test 6: compare tool | |
| def setup_compare(a): | |
| a.teach("Paris is the capital of France", silent=True) | |
| a.teach("Tokyo is the capital of Japan", silent=True) | |
| self._run_test( | |
| agent, "tool_use", "compare_tool", | |
| "compare Paris and Tokyo", | |
| lambda r: "paris" in r.lower() and "tokyo" in r.lower(), | |
| "comparison", | |
| setup=setup_compare, | |
| ) | |
| # Test 7: count tool | |
| self._run_test( | |
| agent, "tool_use", "count_tool", | |
| "count triples", | |
| lambda r: "triples" in r.lower() or any(c.isdigit() for c in r), | |
| "count result", | |
| ) | |
| # ------------------------------------------------------------------ # | |
| # 10. Comprehension | |
| # ------------------------------------------------------------------ # | |
| def tests_comprehension(self, agent: AETHER) -> None: | |
| # Test 1: answer direct question | |
| def setup_comp(a): | |
| a.teach("Paris is the capital of France", silent=True) | |
| self._run_test( | |
| agent, "comprehension", "direct_qa", | |
| "What is the capital of France?", | |
| lambda r: "paris" in r.lower(), | |
| "Paris", | |
| setup=setup_comp, | |
| ) | |
| # Test 2: multi-hop comprehension | |
| def setup_multihop(a): | |
| a.teach("Lyon is located in France", silent=True) | |
| a.teach("Paris is the capital of France", silent=True) | |
| self._run_test( | |
| agent, "comprehension", "multi_hop_qa", | |
| "What is the capital of the country where Lyon is located?", | |
| lambda r: "paris" in r.lower(), | |
| "Paris", | |
| setup=setup_multihop, | |
| ) | |
| # Test 3: comprehension score after a question | |
| agent.ask("What is the capital of France?") | |
| comp_score = agent.comprehension_score() | |
| result = TestResult( | |
| test_name="comprehension_score_post_q", | |
| dimension="comprehension", | |
| passed=comp_score > 0.0, | |
| score=min(1.0, comp_score * 2), | |
| response=f"score={comp_score:.3f}", | |
| expected="comprehension score > 0", | |
| duration_ms=0.0, | |
| ) | |
| self.results.append(result) | |
| # Test 4: define | |
| def setup_def(a): | |
| a.teach("Python is a programming language", silent=True) | |
| self._run_test( | |
| agent, "comprehension", "define_concept", | |
| "define Python", | |
| lambda r: "programming" in r.lower() or "language" in r.lower(), | |
| "programming language", | |
| setup=setup_def, | |
| ) | |
| # Test 5: identify unknown | |
| self._run_test( | |
| agent, "comprehension", "identify_unknown", | |
| "What is the capital of Pluto?", | |
| lambda r: any(kw in r.lower() for kw in ["don't know", "couldn't find", "don't have", "no answer", "i don't", "teach me"]), | |
| "don't know", | |
| ) | |
| # --------------------------------------------------------------------------- | |
| # Intelligence meter | |
| # --------------------------------------------------------------------------- | |
| class IntelligenceMeter: | |
| """Aggregates test results into dimension scores and an overall IQ.""" | |
| # IQ normalization: 100 = average, 130 = gifted, 145 = genius | |
| # Each dimension contributes equally; IQ = 100 + (mean_score - 0.5) * 100 | |
| # So a perfect 1.0 → IQ 150, 0.5 → IQ 100, 0.0 → IQ 50 | |
| DIMENSIONS = [ | |
| "reasoning", "working_memory", "pattern_recog", "abstraction", | |
| "transfer", "metacognition", "self_correction", "theory_of_mind", | |
| "tool_use", "comprehension", | |
| ] | |
| def __init__(self): | |
| self.results: List[TestResult] = [] | |
| def measure(self, results: List[TestResult]) -> Dict[str, Any]: | |
| """Compute dimension scores, overall score, and IQ.""" | |
| self.results = results | |
| # Per-dimension scores | |
| dim_scores: Dict[str, List[float]] = {d: [] for d in self.DIMENSIONS} | |
| for r in results: | |
| if r.dimension in dim_scores: | |
| dim_scores[r.dimension].append(r.score) | |
| # Average per dimension | |
| dim_averages: Dict[str, float] = {} | |
| for d, scores in dim_scores.items(): | |
| if scores: | |
| dim_averages[d] = sum(scores) / len(scores) | |
| else: | |
| dim_averages[d] = 0.0 | |
| # Overall score = mean of dimension averages | |
| overall = sum(dim_averages.values()) / max(len(dim_averages), 1) | |
| # IQ = 50 + overall * 100 (so 0.0 → 50, 1.0 → 150) | |
| iq = int(50 + overall * 100) | |
| # Strengths (top 3) and weaknesses (bottom 3) | |
| sorted_dims = sorted(dim_averages.items(), key=lambda x: -x[1]) | |
| strengths = sorted_dims[:3] | |
| weaknesses = sorted_dims[-3:] | |
| return { | |
| "dimension_scores": dim_averages, | |
| "overall_score": overall, | |
| "iq": iq, | |
| "strengths": strengths, | |
| "weaknesses": weaknesses, | |
| "n_tests": len(results), | |
| "n_passed": sum(1 for r in results if r.passed), | |
| "n_partial": sum(1 for r in results if 0 < r.score < 1.0), | |
| "pass_rate": sum(1 for r in results if r.passed) / max(len(results), 1), | |
| } | |
| def report(self, results: List[TestResult]) -> str: | |
| """Generate a human-readable intelligence report.""" | |
| m = self.measure(results) | |
| lines = [] | |
| lines.append("=" * 76) | |
| lines.append(" AETHER COGNITIVE ASSESSMENT REPORT") | |
| lines.append("=" * 76) | |
| lines.append(f" Tests run: {m['n_tests']}") | |
| lines.append(f" Tests passed: {m['n_passed']} ({m['pass_rate']*100:.1f}%)") | |
| lines.append(f" Partial: {m['n_partial']}") | |
| lines.append(f" Overall: {m['overall_score']:.3f}") | |
| lines.append(f" IQ: {m['iq']}") | |
| lines.append("") | |
| lines.append(" Per-dimension scores:") | |
| for dim, score in sorted(m["dimension_scores"].items(), key=lambda x: -x[1]): | |
| bar = "#" * int(score * 30) | |
| lines.append(f" {dim:18s} {score:.3f} |{bar}") | |
| lines.append("") | |
| lines.append(" Strengths:") | |
| for dim, score in m["strengths"]: | |
| lines.append(f" {dim:18s} {score:.3f}") | |
| lines.append("") | |
| lines.append(" Weaknesses:") | |
| for dim, score in m["weaknesses"]: | |
| lines.append(f" {dim:18s} {score:.3f}") | |
| lines.append("=" * 76) | |
| return "\n".join(lines) | |
| # --------------------------------------------------------------------------- | |
| # Main | |
| # --------------------------------------------------------------------------- | |
| if __name__ == "__main__": | |
| agent = AETHER() | |
| battery = CognitiveBattery() | |
| print("Running cognitive battery on AETHER v4...\n") | |
| results = battery.run_all(agent, verbose=True) | |
| meter = IntelligenceMeter() | |
| print() | |
| print(meter.report(results)) | |