Spaces:
Running on Zero
Running on Zero
File size: 4,236 Bytes
3a2b2e4 b784950 3a2b2e4 b784950 3a2b2e4 b784950 ffa621b 3a2b2e4 b784950 3a2b2e4 b784950 3a2b2e4 b784950 3a2b2e4 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 | from __future__ import annotations
import json
from pathlib import Path
import pandas as pd
class OfflineStudy:
"""Read committed offline-study artifacts without invoking the model."""
REQUIRED = (
'summary.json',
'study_summary.json',
'study_feature_summary.csv',
'causal_position_summary.csv',
'selection_stability.csv',
'feature_catalog.csv',
'layer_metrics.csv',
'stability.csv',
'causal_results_final_token.csv',
'causal_results_max_active.csv',
'feature_set_results.csv',
'report.md',
)
def __init__(self, artifact_dir: str | Path = 'artifacts') -> None:
self.artifact_dir = Path(artifact_dir)
@property
def missing(self) -> list[str]:
return [name for name in self.REQUIRED if not (self.artifact_dir / name).exists()]
@property
def complete(self) -> bool:
return not self.missing
def _json(self, name: str) -> dict:
path = self.artifact_dir / name
if not path.exists():
return {}
return json.loads(path.read_text(encoding='utf-8'))
def dataframe(self, name: str) -> pd.DataFrame:
path = self.artifact_dir / name
if not path.exists():
return pd.DataFrame()
return pd.read_csv(path)
def figure(self, name: str) -> str | None:
path = self.artifact_dir / 'figures' / name
return str(path) if path.exists() else None
def overview_markdown(self) -> str:
if not self.complete:
missing = ', '.join(f'`{name}`' for name in self.missing[:6])
suffix = '…' if len(self.missing) > 6 else ''
return (
'### Offline study not materialized yet\n\n'
'The live workbench is usable now, but the finalized position-sensitivity study artifacts '
f'have not been committed. Missing: {missing}{suffix}\n\n'
'Run `python -m experiments.run_all --resume` or use the full-study Colab notebook '
'to materialize the measured study artifacts.'
)
summary = self._json('summary.json')
study = self._json('study_summary.json')
correlations = study.get('correlations', {})
target_corr = correlations.get('heldout_auroc_vs_max_active_target_specificity', {})
js_corr = correlations.get('heldout_auroc_vs_max_active_js_specificity', {})
return (
'### Offline study results\n\n'
f"{summary.get('headline', 'Benchmark completed.')}\n\n"
f"{summary.get('interpretation', '')}\n\n"
'**Position sensitivity**\n\n'
f"- Final-token feature coverage: **{float(study.get('final_token_feature_coverage', float('nan'))):.1%}**.\n"
f"- Max-active feature coverage: **{float(study.get('max_active_feature_coverage', float('nan'))):.1%}**.\n"
f"- Final-token target specificity: **{float(study.get('final_token_target_specificity_ratio', float('nan'))):.2f}×**.\n"
f"- Max-active target specificity: **{float(study.get('max_active_target_specificity_ratio', float('nan'))):.2f}×**.\n\n"
'**Study-level diagnostics**\n\n'
f"- Selected-feature median resample support: **{float(study.get('median_selected_feature_resample_support', float('nan'))):.1%}**.\n"
f"- Held-out AUROC ↔ max-active target-specificity Spearman ρ: **{float(target_corr.get('rho', float('nan'))):+.3f}** (n={int(target_corr.get('n', 0))}).\n"
f"- Held-out AUROC ↔ max-active JS-specificity Spearman ρ: **{float(js_corr.get('rho', float('nan'))):+.3f}** (n={int(js_corr.get('n', 0))}).\n\n"
'Cross-concept correlations are descriptive; per-concept causal evidence remains anchored to '
'norm-matched random controls.'
)
def readiness_markdown(self) -> str:
if self.complete:
return '**Artifact status:** complete and ready for the public study view.'
return (
f'**Artifact status:** {len(self.REQUIRED) - len(self.missing)}/{len(self.REQUIRED)} required '
f'files present; {len(self.missing)} missing.'
)
|