scripts/run_ablations.py

"""Dependency-checked, restartable lightweight experiments. No API usage."""
import argparse,json,subprocess,sys
from pathlib import Path
import pandas as pd

def call(*args):subprocess.run([sys.executable,'scripts/train_heads.py',*map(str,args)],check=True)

def main():
    parser=argparse.ArgumentParser();parser.add_argument('--skip-evaluation',action='store_true',help='Defer evaluation until the14B intervention cache and replication are complete');args=parser.parse_args()
    ir=pd.read_parquet('data/processed/interventions.parquet');counts=ir[(ir.int_split=='train')&ir.strict].groupby('criterion_index').size().reindex(range(4),fill_value=0)
    budgets=[n for n in [10,25,50,100,250] if n<=counts.min()]
    if not budgets:budgets=[n for n in [1,2,5] if n<=counts.min()]
    if not budgets:raise ValueError('No complete four-criterion strict budget; inspect controllability and reduced criteria')
    main_n=max(n for n in budgets if n<=50)
    reduced_n=max([n for n in [1,2,5,10,25,50] if n<=counts.loc[[0,1,3]].min()],default=0)
    Path('outputs/main_design.json').write_text(json.dumps({'main_n':main_n,'available_strict_counts':counts.to_dict(),'budgets':budgets,'selection':'maximum supported prespecified budget <=50; chosen without test metrics','budget_unit':'edits_per_criterion','reduced_criteria':[0,1,3],'reduced_n':reduced_n,'verifiers':sorted(ir.verifier.dropna().unique().tolist()),'verifier_revision':sorted(ir.verifier_revision.dropna().unique().tolist()),'preferred_verifier':'DeepSeek-V4.1-Flash (unavailable; see engine_decisions.md)','fallback_local_verifier_shares_32b_backbone':True},indent=2))
    for n in budgets:call('--method','ilr','--n',n)
    for filter_name in ['raw','direction_filtered']:call('--method','ilr','--n',main_n,'--filter',filter_name)
    call('--method','ilr','--n',main_n,'--latent-dim',4)
    call('--method','latent','--latent-dim',4)
    call('--method','ilr','--n',main_n,'--arch','mlp')
    call('--method','latent','--arch','mlp')
    call('--method','latent','--shuffle-labels')
    call('--method','ilr','--n',main_n,'--flip-intervention-signs')
    for layer in [16,32,48,56,64]:
        for pooling in ['mean','last']:
            call('--method','latent','--layer',layer,'--pooling',pooling)
            call('--method','ilr','--n',main_n,'--layer',layer,'--pooling',pooling)
    if reduced_n:
        # The k4 unanchored latent fit above is numerically the same baseline;
        # its saved coordinates can be DEV-matched to three criteria by analysis.
        call('--method','ilr','--n',reduced_n,'--latent-dim',4,'--criteria',0,1,3)
    if Path('outputs/features/14b/COMPLETE.json').exists() and Path('outputs/features/14b_interventions/COMPLETE.json').exists():
        call('--method','latent','--backbone','14b');call('--method','ilr','--n',main_n,'--backbone','14b')
    if not args.skip_evaluation:subprocess.run([sys.executable,'scripts/evaluate_heads.py'],check=True)

if __name__=='__main__':main()