src/scripts/ablation_study.py

import logging
from scripts.evaluate import run_evaluation

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

def run_ablation():
    """
    Automates ablation studies on sample efficiency and temperature scheduling.
    """
    logger.info("Starting System Ablation Studies...")
    
    # 1. Sample Efficiency Ablation
    # Assuming we trained models on different subsets: 100, 300, 500 samples
    sample_sizes = [100, 300, 500]
    for size in sample_sizes:
        lora_path = f"./param_mem_lora_samples_{size}"
        logger.info(f"--- Ablation: Sample Size {size} ---")
        try:
            # We use try/except since paths might not exist yet
            run_evaluation(model_name="meta-llama/Meta-Llama-3-8B-Instruct", lora_path=lora_path, domain="humaneval")
        except Exception as e:
            logger.warning(f"Could not evaluate sample size {size}: {e}")
            
    # 2. Temperature Scheduling Ablation
    # This would require modifying the agent_loop.py parameters dynamically.
    logger.info("--- Ablation: Temperature Scheduling (Dynamic vs Static) ---")
    logger.info("This ablation requires overriding the agent's temp scaling logic via kwargs.")
    # Implementation placeholder for temp scheduling
    
    logger.info("Ablation Studies Complete.")

if __name__ == "__main__":
    run_ablation()