scripts/validate_artifacts.py

from __future__ import annotations

import json
from pathlib import Path

import pandas as pd

ROOT = Path(__file__).resolve().parents[1]
ARTIFACT_DIR = ROOT / 'artifacts'

REQUIRED = [
    'feature_catalog.csv',
    'layer_metrics.csv',
    'stability.csv',
    'selection_stability.csv',
    'causal_results_final_token.csv',
    'causal_results_max_active.csv',
    'causal_position_summary.csv',
    'feature_set_results.csv',
    'study_feature_summary.csv',
    'study_summary.json',
    'summary.json',
    'report.md',
]


def _require_columns(path: Path, columns: set[str]) -> None:
    frame = pd.read_csv(path)
    missing = columns.difference(frame.columns)
    if missing:
        raise SystemExit(f'{path.relative_to(ROOT)} missing columns: {sorted(missing)}')
    if frame.empty:
        raise SystemExit(f'{path.relative_to(ROOT)} is empty.')


def main() -> None:
    missing = [name for name in REQUIRED if not (ARTIFACT_DIR / name).exists()]
    if missing:
        raise SystemExit(f'Missing offline-study artifacts: {missing}')

    _require_columns(ARTIFACT_DIR / 'feature_catalog.csv', {'layer','concept','feature_id','train_auroc','auroc','f1'})
    _require_columns(ARTIFACT_DIR / 'selection_stability.csv', {'layer','concept','feature_id','resample_support','median_resample_rank'})
    causal_columns = {
        'task_id','concept','feature_id','position_policy','intervention_token_index',
        'feature_active_at_intervention','feature_active_at_final_token','feature_active_anywhere',
        'intervention','condition','target_mean_logprob_delta','js_divergence',
    }
    _require_columns(ARTIFACT_DIR / 'causal_results_final_token.csv', causal_columns)
    _require_columns(ARTIFACT_DIR / 'causal_results_max_active.csv', causal_columns)
    _require_columns(ARTIFACT_DIR / 'causal_position_summary.csv', {
        'concept','position_policy','feature_active_at_intervention_rate','target_specificity_ratio',
        'target_paired_advantage','target_sign_flip_pvalue',
    })
    _require_columns(ARTIFACT_DIR / 'study_feature_summary.csv', {
        'concept','layer','feature_id','heldout_auroc','heldout_f1','candidate_resample_support',
        'final_target_specificity_ratio','max_active_target_specificity_ratio',
        'final_feature_active_at_intervention_rate','max_active_feature_active_at_intervention_rate',
    })

    summary = json.loads((ARTIFACT_DIR / 'study_summary.json').read_text(encoding='utf-8'))
    if int(summary.get('n_concepts', 0)) < 1:
        raise SystemExit('study_summary.json has no concepts.')
    if summary.get('primary_causal_position_policy') != 'max_feature_activation':
        raise SystemExit('study_summary.json must use max_feature_activation as the primary causal policy.')
    if 'causal task' not in str(summary.get('causal_statistical_unit', '')).lower():
        raise SystemExit('study_summary.json must document causal-task-level inference.')

    required_figures = [
        'feature_auroc.png','layer_diagnostics.png','causal_effects.png','feature_set_effects.png',
        'association_vs_causality.png','causal_position_sensitivity.png',
    ]
    missing_figures = [name for name in required_figures if not (ARTIFACT_DIR / 'figures' / name).exists()]
    if missing_figures:
        raise SystemExit(f'Missing report figures: {missing_figures}')

    print('FeatureLens offline artifact validation: PASS')
    print(f"  concepts: {summary['n_concepts']}")
    print(f"  primary causal policy: {summary['primary_causal_position_policy']}")
    print(f"  statistical unit: {summary['causal_statistical_unit']}")
    print('  report: artifacts/report.md')


if __name__ == '__main__':
    main()