File size: 7,332 Bytes
1ca688b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 | """Tests for the evolution layer (5 cognitive upgrades)."""
import pytest
import numpy as np
from palimseste.lm import PalimpsesteForCausalLM, PalimpsesteConfig
from palimseste.chat import Conversation
from palimseste.evolution import (
ResponseSynthesizer, EntityTracker, QueryRouter,
CodePatternBank, ConfidenceCalibrator,
SynthesisResult, Entity, QueryIntent, CodePattern, CalibrationResult,
)
def _build_model(D=5000, ctx=128, radius=200):
cfg = PalimpsesteConfig(D=D, context_window=ctx, kernel_radius=radius, temperature=0.0)
lm = PalimpsesteForCausalLM(config=cfg)
pairs = [
("hello", "hi i am palimpseste"),
("who are you", "i am palimpseste a hypervectorial cortex"),
("what is python", "python is a programming language"),
("what is java", "java is a programming language"),
("what is recursion", "recursion is when a function calls itself"),
("what is the capital of france", "the capital of france is paris"),
("what is the capital of japan", "the capital of japan is tokyo"),
]
lm.build_tokenizer("".join(q + a for q, a in pairs))
lm.train_on_qa_pairs(pairs)
conv = Conversation(model=lm, fuzzy_threshold=0.75)
conv.register_questions(pairs)
return lm, conv
# ================================================================ SYNTHESIZER
class TestResponseSynthesizer:
def test_single_fact(self):
_, conv = _build_model()
synth = ResponseSynthesizer(conv=conv)
result = synth.synthesize("what is python")
assert isinstance(result, SynthesisResult)
assert result.synthesis_type == 'single'
def test_no_match(self):
_, conv = _build_model()
synth = ResponseSynthesizer(conv=conv)
result = synth.synthesize("xyz123unknown")
assert result.n_facts_used == 0 or result.confidence <= 0.5
def test_extract_topics(self):
_, conv = _build_model()
synth = ResponseSynthesizer(conv=conv)
topics = synth._extract_topics("compare python and java")
assert len(topics) >= 2
assert "python" in topics[0] or "python" in topics
# ================================================================ ENTITY TRACKER
class TestEntityTracker:
def test_track_entity(self):
tracker = EntityTracker()
tracker.update("what is python")
assert tracker.get_entity("python") is not None
assert tracker.n_entities >= 1
def test_pronoun_resolution(self):
tracker = EntityTracker()
tracker.update("what is python")
resolved = tracker.update("what about its speed")
assert "python" in resolved.lower()
def test_no_pronoun(self):
tracker = EntityTracker()
tracker.update("what is python")
resolved = tracker.update("what is java")
assert "python" not in resolved.lower()
def test_reset(self):
tracker = EntityTracker()
tracker.update("what is python")
assert tracker.n_entities > 0
tracker.reset()
assert tracker.n_entities == 0
def test_all_entities(self):
tracker = EntityTracker()
tracker.update("what is python and java")
entities = tracker.all_entities()
assert len(entities) >= 2
def test_multiple_mentions(self):
tracker = EntityTracker()
tracker.update("what is python")
tracker.update("tell me about python")
ent = tracker.get_entity("python")
assert ent is not None
assert ent.mentions >= 2
# ================================================================ QUERY ROUTER
class TestQueryRouter:
def test_greeting(self):
router = QueryRouter()
intent = router.classify("hello there")
assert intent.intent == 'greeting'
assert intent.strategy == 'direct'
def test_comparison(self):
router = QueryRouter()
intent = router.classify("compare python and java")
assert intent.intent == 'comparison'
assert intent.strategy == 'synthesize'
def test_howto(self):
router = QueryRouter()
intent = router.classify("how do i write a loop")
assert intent.intent == 'howto'
def test_code(self):
router = QueryRouter()
intent = router.classify("write code for fibonacci")
assert intent.intent == 'code'
def test_definition(self):
router = QueryRouter()
intent = router.classify("what is gravity")
assert intent.intent == 'definition'
def test_factual(self):
router = QueryRouter()
intent = router.classify("who won the world cup")
assert intent.intent == 'factual'
def test_topics_extracted(self):
router = QueryRouter()
intent = router.classify("what is python programming")
assert len(intent.topics) > 0
# ================================================================ CODE BANK
class TestCodePatternBank:
def test_store(self):
bank = CodePatternBank()
p = bank.store("python", "read file", "open('f')", "read", ["open"])
assert p.id == 0
assert bank.n_patterns == 1
def test_retrieve(self):
bank = CodePatternBank()
bank.load_defaults()
results = bank.retrieve("how to read a file", language="python")
assert len(results) > 0
assert results[0][0].task == 'read file'
def test_retrieve_by_language(self):
bank = CodePatternBank()
bank.load_defaults()
results = bank.retrieve("function", language="javascript")
assert len(results) == 0 # no JS patterns loaded
def test_load_defaults(self):
bank = CodePatternBank()
bank.load_defaults()
assert bank.n_patterns >= 10
def test_no_match(self):
bank = CodePatternBank()
bank.load_defaults()
results = bank.retrieve("xyz123 nonexistent")
# Might get 0 results or very low scores
assert all(s < 0.2 for _, s in results) or len(results) == 0
# ================================================================ CALIBRATOR
class TestConfidenceCalibrator:
def test_high_confidence(self):
cal = ConfidenceCalibrator()
result = cal.calibrate("paris", 1.0, "capital of france", "paris")
assert result.calibrated_confidence > 0.5
assert not result.hedged
def test_low_confidence(self):
cal = ConfidenceCalibrator()
result = cal.calibrate("xyz", 0.1, "meaning of life", "xyz")
assert result.hedged
assert result.hedge_phrase is not None
def test_no_double_hedge(self):
cal = ConfidenceCalibrator()
result = cal.calibrate("i'm not sure about this", 0.1, "q", "a")
assert not result.hedged # already starts with hedge
def test_medium_confidence(self):
cal = ConfidenceCalibrator()
result = cal.calibrate("maybe paris", 0.5, "capital", "paris")
assert isinstance(result, CalibrationResult)
def test_calibrated_less_than_raw(self):
cal = ConfidenceCalibrator()
result = cal.calibrate("short", 1.0, "what is quantum mechanics", "short")
# Very short answer to an unrelated question should be calibrated down
assert result.calibrated_confidence < result.raw_confidence
|