Spaces:
Running on Zero
Running on Zero
File size: 3,694 Bytes
3a2b2e4 b784950 3a2b2e4 b784950 3a2b2e4 b784950 3a2b2e4 b784950 3a2b2e4 b784950 3a2b2e4 b784950 3a2b2e4 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 | from __future__ import annotations
import json
from pathlib import Path
import pandas as pd
ROOT = Path(__file__).resolve().parents[1]
ARTIFACT_DIR = ROOT / 'artifacts'
REQUIRED = [
'feature_catalog.csv',
'layer_metrics.csv',
'stability.csv',
'selection_stability.csv',
'causal_results_final_token.csv',
'causal_results_max_active.csv',
'causal_position_summary.csv',
'feature_set_results.csv',
'study_feature_summary.csv',
'study_summary.json',
'summary.json',
'report.md',
]
def _require_columns(path: Path, columns: set[str]) -> None:
frame = pd.read_csv(path)
missing = columns.difference(frame.columns)
if missing:
raise SystemExit(f'{path.relative_to(ROOT)} missing columns: {sorted(missing)}')
if frame.empty:
raise SystemExit(f'{path.relative_to(ROOT)} is empty.')
def main() -> None:
missing = [name for name in REQUIRED if not (ARTIFACT_DIR / name).exists()]
if missing:
raise SystemExit(f'Missing offline-study artifacts: {missing}')
_require_columns(ARTIFACT_DIR / 'feature_catalog.csv', {'layer','concept','feature_id','train_auroc','auroc','f1'})
_require_columns(ARTIFACT_DIR / 'selection_stability.csv', {'layer','concept','feature_id','resample_support','median_resample_rank'})
causal_columns = {
'task_id','concept','feature_id','position_policy','intervention_token_index',
'feature_active_at_intervention','feature_active_at_final_token','feature_active_anywhere',
'intervention','condition','target_mean_logprob_delta','js_divergence',
}
_require_columns(ARTIFACT_DIR / 'causal_results_final_token.csv', causal_columns)
_require_columns(ARTIFACT_DIR / 'causal_results_max_active.csv', causal_columns)
_require_columns(ARTIFACT_DIR / 'causal_position_summary.csv', {
'concept','position_policy','feature_active_at_intervention_rate','target_specificity_ratio',
'target_paired_advantage','target_sign_flip_pvalue',
})
_require_columns(ARTIFACT_DIR / 'study_feature_summary.csv', {
'concept','layer','feature_id','heldout_auroc','heldout_f1','candidate_resample_support',
'final_target_specificity_ratio','max_active_target_specificity_ratio',
'final_feature_active_at_intervention_rate','max_active_feature_active_at_intervention_rate',
})
summary = json.loads((ARTIFACT_DIR / 'study_summary.json').read_text(encoding='utf-8'))
if int(summary.get('n_concepts', 0)) < 1:
raise SystemExit('study_summary.json has no concepts.')
if summary.get('primary_causal_position_policy') != 'max_feature_activation':
raise SystemExit('study_summary.json must use max_feature_activation as the primary causal policy.')
if 'causal task' not in str(summary.get('causal_statistical_unit', '')).lower():
raise SystemExit('study_summary.json must document causal-task-level inference.')
required_figures = [
'feature_auroc.png','layer_diagnostics.png','causal_effects.png','feature_set_effects.png',
'association_vs_causality.png','causal_position_sensitivity.png',
]
missing_figures = [name for name in required_figures if not (ARTIFACT_DIR / 'figures' / name).exists()]
if missing_figures:
raise SystemExit(f'Missing report figures: {missing_figures}')
print('FeatureLens offline artifact validation: PASS')
print(f" concepts: {summary['n_concepts']}")
print(f" primary causal policy: {summary['primary_causal_position_policy']}")
print(f" statistical unit: {summary['causal_statistical_unit']}")
print(' report: artifacts/report.md')
if __name__ == '__main__':
main()
|