Spaces:
Sleeping
Sleeping
File size: 1,366 Bytes
0279c66 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 | import logging
from scripts.evaluate import run_evaluation
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
def run_ablation():
"""
Automates ablation studies on sample efficiency and temperature scheduling.
"""
logger.info("Starting System Ablation Studies...")
# 1. Sample Efficiency Ablation
# Assuming we trained models on different subsets: 100, 300, 500 samples
sample_sizes = [100, 300, 500]
for size in sample_sizes:
lora_path = f"./param_mem_lora_samples_{size}"
logger.info(f"--- Ablation: Sample Size {size} ---")
try:
# We use try/except since paths might not exist yet
run_evaluation(model_name="meta-llama/Meta-Llama-3-8B-Instruct", lora_path=lora_path, domain="humaneval")
except Exception as e:
logger.warning(f"Could not evaluate sample size {size}: {e}")
# 2. Temperature Scheduling Ablation
# This would require modifying the agent_loop.py parameters dynamically.
logger.info("--- Ablation: Temperature Scheduling (Dynamic vs Static) ---")
logger.info("This ablation requires overriding the agent's temp scaling logic via kwargs.")
# Implementation placeholder for temp scheduling
logger.info("Ablation Studies Complete.")
if __name__ == "__main__":
run_ablation()
|