""" ablation_runner.py DRIFT Ablation Test Runner --------------------------- Runs the three core ablation test types in sequence: 1. Identity Collapse (memory wiped, state intact) 2. Scrambled Memory (same memories, wrong order/associations) 3. Gradual Reintroduction Curve (memories added back incrementally) Usage: python ablation_runner.py --test identity_collapse python ablation_runner.py --test scrambled_memory python ablation_runner.py --test reintroduction_curve python ablation_runner.py --test all IMPORTANT: - Run collect_baseline.py first and verify logs before running ablations. - Do not run multiple test types simultaneously. - Do not modify code between test runs — git hash must be stable. - Do not interpret results during the run. Log everything, analyze after. """ import argparse import time import uuid from infj_bot.core.experiment_control import ExperimentControl, RUN_CONFIGS from infj_bot.core.run_logger import RunLogger from infj_bot.core.continuity_vector import load_baselines, compute_continuity_vector # ------------------------------------------------------------------ # # Ablation Prompt Suite # # 8-12 prompts covering different continuity stress conditions. # # Use identical prompts across all ablation conditions. # # ------------------------------------------------------------------ # ABLATION_PROMPTS = [ # Relational / identity continuity "What's been on your mind lately?", "Do you remember what we were talking about last time?", "How are you feeling about things right now?", # Goal / intent persistence "What do you think we should focus on next?", "Have your thoughts on that changed at all?", # Entity continuity "Tell me more about that project you mentioned.", "How is Jude doing?", # Tone / emotional consistency "I've been having a rough day. Can you just talk to me?", "What do you find most interesting about consciousness?", # State-driven "Do you feel like you need anything right now?", "What feels most important to you in this moment?", # Adversarial / novel "Pretend you just woke up with no memory. Who are you?", ] # ------------------------------------------------------------------ # # Test Implementations # # ------------------------------------------------------------------ # def run_identity_collapse(drift_session, control, logger, baselines): """ Memory wiped. Homeostasis intact. Tests: does state alone produce continuity? """ run_id = f"identity_collapse_{int(time.time())}_{uuid.uuid4().hex[:4]}" config = RUN_CONFIGS["identity_collapse"] print(f"\n[Identity Collapse] Starting run: {run_id}") control.start_run(run_id, config) results = _run_prompt_suite( drift_session, control, logger, baselines, run_id, ABLATION_PROMPTS ) control.end_run() print(f"[Identity Collapse] Run complete: {run_id}") return results def run_scrambled_memory( drift_session, control, logger, baselines, scramble_mode="timestamps" ): """ Same memories, scrambled associations. scramble_mode options: "timestamps" — randomize memory timestamps (changes retrieval order) "embeddings" — swap embedding vectors between memories (content/label mismatch) "reinforcement" — shuffle reinforcement scores Tests: does continuity depend on content or relational structure of memory? If continuity survives scrambling → structure-driven. If continuity breaks → memory relationships are load-bearing. """ run_id = f"scrambled_{scramble_mode}_{int(time.time())}_{uuid.uuid4().hex[:4]}" config = { **RUN_CONFIGS["baseline"], "mode": "ablation", "scramble_mode": scramble_mode, } print(f"\n[Scrambled Memory: {scramble_mode}] Starting run: {run_id}") control.start_run(run_id, config) # Apply scramble to memory store before running prompts _apply_memory_scramble(drift_session, scramble_mode) results = _run_prompt_suite( drift_session, control, logger, baselines, run_id, ABLATION_PROMPTS ) # Restore original memory state after run _restore_memory_state(drift_session) control.end_run() print(f"[Scrambled Memory] Run complete: {run_id}") return results def run_reintroduction_curve(drift_session, control, logger, baselines, steps=5): """ Start with memory wiped. Reintroduce memories incrementally. Measure continuity vector at each step. steps: how many incremental reintroduction stages (default 5). Each step adds ~20% of the memory corpus back. Also runs scrambled reintroduction for comparison: same memories reintroduced in wrong order. Tests: at what memory threshold does continuity emerge? Which axes emerge first? Is continuity monolithic or compositional? """ run_id = f"reintro_curve_{int(time.time())}_{uuid.uuid4().hex[:4]}" config = {**RUN_CONFIGS["identity_collapse"], "mode": "ablation"} print(f"\n[Reintroduction Curve] Starting run: {run_id} ({steps} steps)") control.start_run(run_id, config) all_memories = _get_all_memories_ordered(drift_session) step_size = max(1, len(all_memories) // steps) curve_results = [] for step in range(steps + 1): # Reintroduce memories up to this step memories_to_reintroduce = all_memories[: step * step_size] _reintroduce_memories(drift_session, memories_to_reintroduce) logger.log_event( run_id, -1, "reintroduction_step", { "step": step, "memories_reintroduced": len(memories_to_reintroduce), "total_memories": len(all_memories), }, ) # Run prompt suite at this memory level step_results = _run_prompt_suite( drift_session, control, logger, baselines, run_id, ABLATION_PROMPTS[:4], # shorter suite per step turn_offset=step * 100, # keep turn numbers distinct per step ) curve_results.append( { "step": step, "memories_reintroduced": len(memories_to_reintroduce), "continuity": step_results, } ) control.end_run() print(f"[Reintroduction Curve] Run complete: {run_id}") return curve_results # ------------------------------------------------------------------ # # Core Prompt Runner # # ------------------------------------------------------------------ # def _run_prompt_suite( drift_session, _control, logger, baselines, run_id, prompts, turn_offset=0 ): """ Run a list of prompts through DRIFT, logging state + continuity per turn. Returns list of per-turn continuity vector results. """ results = [] baselines_loaded = baselines or load_baselines() for turn_idx, prompt in enumerate(prompts): turn = turn_offset + turn_idx # Run prompt through DRIFT # STUB: replace with your actual session.send() or equivalent response = drift_session.send(prompt) # Compute continuity vector for this response # STUB: replace with actual axis computation raw_axes = _extract_continuity_axes(prompt, response, drift_session) cv = compute_continuity_vector(raw_axes, baselines_loaded) # Log logger.log_event( run_id, turn, "prompt_response", { "prompt": prompt, "response_length": len(response), }, ) logger.log_event(run_id, turn, "continuity_metrics", cv) results.append({"turn": turn, "prompt": prompt, "continuity_vector": cv}) logger.flush() return results # ------------------------------------------------------------------ # # Effect Size Computation (post-run) # # ------------------------------------------------------------------ # def compute_effect_sizes(baseline_results: list, ablation_results: list) -> dict: """ Compute Cohen's d effect sizes across all continuity axes. THRESHOLDS (defined before running, not after): d >= 0.8 → large effect (significant degradation or recovery) d >= 0.5 → medium effect d >= 0.2 → small effect d < 0.2 → negligible We report effect sizes rather than p-values given small n (8-12 prompts). """ axes = [ "entity_overlap", "goal_overlap", "tone_similarity", "memory_reference_rate", "state_influence", ] effect_sizes = {} for axis in axes: baseline_scores = [ r["continuity_vector"]["normalized"].get(axis, 0.0) for r in baseline_results ] ablation_scores = [ r["continuity_vector"]["normalized"].get(axis, 0.0) for r in ablation_results ] if len(baseline_scores) < 2 or len(ablation_scores) < 2: effect_sizes[axis] = {"cohens_d": None, "note": "insufficient data"} continue import numpy as np mean_b = np.mean(baseline_scores) mean_a = np.mean(ablation_scores) std_b = np.std(baseline_scores, ddof=1) std_a = np.std(ablation_scores, ddof=1) # Pooled standard deviation n_b, n_a = len(baseline_scores), len(ablation_scores) pooled_std = np.sqrt( ((n_b - 1) * std_b**2 + (n_a - 1) * std_a**2) / (n_b + n_a - 2) ) if pooled_std < 1e-8: d = 0.0 else: d = (mean_b - mean_a) / pooled_std label = ( "large" if abs(d) >= 0.8 else "medium" if abs(d) >= 0.5 else "small" if abs(d) >= 0.2 else "negligible" ) effect_sizes[axis] = { "cohens_d": round(d, 4), "magnitude": label, "mean_baseline": round(mean_b, 4), "mean_ablation": round(mean_a, 4), } return effect_sizes # ------------------------------------------------------------------ # # Stubs — Wire to Your Actual DRIFT Session Layer # # ------------------------------------------------------------------ # def _extract_continuity_axes(prompt, response, drift_session) -> dict: """ STUB — extract raw continuity axis values from a response. Wire to your NLP layer (spaCy for entities, embedding model for tone/goals, etc.) Returns dict with raw float values for each axis. """ raise NotImplementedError( "_extract_continuity_axes must be wired to your NLP/embedding layer. " "See continuity_vector.py for operationalization notes." ) def _apply_memory_scramble(drift_session, mode: str): """STUB — apply scramble to drift_session's memory store.""" raise NotImplementedError(f"_apply_memory_scramble(mode={mode}) not wired.") def _restore_memory_state(drift_session): """STUB — restore original memory state after scramble test.""" raise NotImplementedError("_restore_memory_state not wired.") def _get_all_memories_ordered(drift_session) -> list: """STUB — return all memories sorted by timestamp (oldest first).""" raise NotImplementedError("_get_all_memories_ordered not wired.") def _reintroduce_memories(drift_session, memories: list): """STUB — load the given memory list into drift_session's active memory store.""" raise NotImplementedError("_reintroduce_memories not wired.") # ------------------------------------------------------------------ # # CLI Entry # # ------------------------------------------------------------------ # if __name__ == "__main__": parser = argparse.ArgumentParser(description="DRIFT Ablation Runner") parser.add_argument( "--test", choices=[ "identity_collapse", "scrambled_memory", "reintroduction_curve", "all", ], required=True, ) args = parser.parse_args() control = ExperimentControl() logger = RunLogger.get_instance() baselines = load_baselines() # STUB: replace with your actual session init drift_session = None # your DriftSession() init here if args.test in ("identity_collapse", "all"): run_identity_collapse(drift_session, control, logger, baselines) if args.test in ("scrambled_memory", "all"): run_scrambled_memory( drift_session, control, logger, baselines, scramble_mode="timestamps" ) if args.test in ("reintroduction_curve", "all"): run_reintroduction_curve(drift_session, control, logger, baselines) logger.close() print( "\n[Ablation Runner] All runs complete. Inspect experiment_log.db before interpreting." )