featurelens/study.py

from __future__ import annotations

import json
from pathlib import Path

import pandas as pd


class OfflineStudy:
    """Read committed offline-study artifacts without invoking the model."""

    REQUIRED = (
        'summary.json',
        'study_summary.json',
        'study_feature_summary.csv',
        'causal_position_summary.csv',
        'selection_stability.csv',
        'feature_catalog.csv',
        'layer_metrics.csv',
        'stability.csv',
        'causal_results_final_token.csv',
        'causal_results_max_active.csv',
        'feature_set_results.csv',
        'report.md',
    )

    def __init__(self, artifact_dir: str | Path = 'artifacts') -> None:
        self.artifact_dir = Path(artifact_dir)

    @property
    def missing(self) -> list[str]:
        return [name for name in self.REQUIRED if not (self.artifact_dir / name).exists()]

    @property
    def complete(self) -> bool:
        return not self.missing

    def _json(self, name: str) -> dict:
        path = self.artifact_dir / name
        if not path.exists():
            return {}
        return json.loads(path.read_text(encoding='utf-8'))

    def dataframe(self, name: str) -> pd.DataFrame:
        path = self.artifact_dir / name
        if not path.exists():
            return pd.DataFrame()
        return pd.read_csv(path)

    def figure(self, name: str) -> str | None:
        path = self.artifact_dir / 'figures' / name
        return str(path) if path.exists() else None

    def overview_markdown(self) -> str:
        if not self.complete:
            missing = ', '.join(f'`{name}`' for name in self.missing[:6])
            suffix = '…' if len(self.missing) > 6 else ''
            return (
                '### Offline study not materialized yet\n\n'
                'The live workbench is usable now, but the finalized position-sensitivity study artifacts '
                f'have not been committed. Missing: {missing}{suffix}\n\n'
                'Run `python -m experiments.run_all --resume` or use the full-study Colab notebook '
                'to materialize the measured study artifacts.'
            )

        summary = self._json('summary.json')
        study = self._json('study_summary.json')
        correlations = study.get('correlations', {})
        target_corr = correlations.get('heldout_auroc_vs_max_active_target_specificity', {})
        js_corr = correlations.get('heldout_auroc_vs_max_active_js_specificity', {})
        return (
            '### Offline study results\n\n'
            f"{summary.get('headline', 'Benchmark completed.')}\n\n"
            f"{summary.get('interpretation', '')}\n\n"
            '**Position sensitivity**\n\n'
            f"- Final-token feature coverage: **{float(study.get('final_token_feature_coverage', float('nan'))):.1%}**.\n"
            f"- Max-active feature coverage: **{float(study.get('max_active_feature_coverage', float('nan'))):.1%}**.\n"
            f"- Final-token target specificity: **{float(study.get('final_token_target_specificity_ratio', float('nan'))):.2f}×**.\n"
            f"- Max-active target specificity: **{float(study.get('max_active_target_specificity_ratio', float('nan'))):.2f}×**.\n\n"
            '**Study-level diagnostics**\n\n'
            f"- Selected-feature median resample support: **{float(study.get('median_selected_feature_resample_support', float('nan'))):.1%}**.\n"
            f"- Held-out AUROC ↔ max-active target-specificity Spearman ρ: **{float(target_corr.get('rho', float('nan'))):+.3f}** (n={int(target_corr.get('n', 0))}).\n"
            f"- Held-out AUROC ↔ max-active JS-specificity Spearman ρ: **{float(js_corr.get('rho', float('nan'))):+.3f}** (n={int(js_corr.get('n', 0))}).\n\n"
            'Cross-concept correlations are descriptive; per-concept causal evidence remains anchored to '
            'norm-matched random controls.'
        )

    def readiness_markdown(self) -> str:
        if self.complete:
            return '**Artifact status:** complete and ready for the public study view.'
        return (
            f'**Artifact status:** {len(self.REQUIRED) - len(self.missing)}/{len(self.REQUIRED)} required '
            f'files present; {len(self.missing)} missing.'
        )