from __future__ import annotations import json import subprocess from collections import Counter from pathlib import Path ROOT = Path(__file__).resolve().parents[1] MAX_FILE_SIZE_BYTES = 5_000_000 # 5 MB REQUIRED = [ 'README.md', 'app.py', 'requirements.txt', 'research_config.json', 'featurelens/runtime.py', 'featurelens/sae.py', 'featurelens/interventions.py', 'featurelens/metrics.py', 'featurelens/stats.py', 'experiments/run_all.py', 'experiments/run_causal.py', 'experiments/run_feature_sets.py', 'data/prompts.jsonl', 'data/causal_tasks.jsonl', 'docs/VALIDATION.md', 'scripts/ui_smoke.py', ] def load_jsonl(path: Path) -> list[dict]: return [ json.loads(line) for line in path.read_text(encoding='utf-8').splitlines() if line.strip() ] def repository_candidates() -> list[Path]: """Return tracked files plus untracked files that are not ignored by Git.""" try: result = subprocess.run( ['git', 'ls-files', '--cached', '--others', '--exclude-standard'], cwd=ROOT, capture_output=True, text=True, check=True, ) except FileNotFoundError as exc: raise SystemExit('Git is required to run the FeatureLens release check.') from exc except subprocess.CalledProcessError as exc: raise SystemExit( f'Could not inspect repository files with Git: {exc.stderr.strip()}' ) from exc paths: list[Path] = [] for relative_path in result.stdout.splitlines(): relative_path = relative_path.strip() if not relative_path: continue path = ROOT / relative_path if path.is_file(): paths.append(path) return paths def check_required_files() -> None: missing = [path for path in REQUIRED if not (ROOT / path).exists()] if missing: raise SystemExit(f'Missing required files: {missing}') def check_config(config: dict) -> None: expected = { 'layers': [4, 14, 26], 'model_id': 'Qwen/Qwen3-1.7B-Base', 'sae_width': 32768, 'dose_response_multipliers': [0.0, 0.5, 1.0, 1.5, 2.0, 3.0], 'feature_set_sizes': [1, 3, 5], 'live_random_controls': 8, 'offline_random_controls_default': 8, 'concept_contrast_prompts_per_concept': 4, 'interaction_feature_limit': 5, 'live_geometry_feature_limit': 8, 'concept_contrast_pooling': 'max activation across non-padding prompt tokens', 'candidate_causal_screen_limit': 8, 'candidate_specificity_limit': 3, } for key, value in expected.items(): if config.get(key) != value: raise SystemExit(f'Unexpected {key}: {config.get(key)!r}. Expected {value!r}.') required_live_v04 = { 'batch_context_null_reference', 'random_control_ensemble', 'individual_vs_joint_interaction_decomposition', 'promptwide_paraphrase_robustness', 'controlled_concept_contrast_scan', 'copy_tables_with_headers', } actual_live_v04 = set(config.get('live_features_v0_4', [])) if actual_live_v04 != required_live_v04: raise SystemExit( 'research_config.json live_features_v0_4 mismatch: ' f'{sorted(actual_live_v04)}' ) required_live_v05 = { 'wide_centered_responsive_layout', 'copy_feedback', 'dynamic_height_reflow_observer', 'promptwide_concept_contrast_scan', 'feature_token_activation_trace', 'contrastive_continuation_preference_test', 'feature_decoder_geometry', } actual_live_v05 = set(config.get('live_features_v0_5', [])) if actual_live_v05 != required_live_v05: raise SystemExit( 'research_config.json live_features_v0_5 mismatch: ' f'{sorted(actual_live_v05)}' ) required_live_v06 = { 'start_here_plain_language_onboarding', 'persistent_workbench_context_banner', 'explicit_per_experiment_feature_selectors', 'plot_fullscreen_and_export_controls', 'consistent_heading_and_table_typography', 'concept_guided_candidate_feature_discovery', 'completion_cue_sensitivity_scan', } actual_live_v06 = set(config.get('live_features_v0_6', [])) if actual_live_v06 != required_live_v06: raise SystemExit( 'research_config.json live_features_v0_6 mismatch: ' f'{sorted(actual_live_v06)}' ) required_live_v07 = { 'cleaned_nonaccordion_experiment_layout', 'focused_fullscreen_modal_for_tables_and_plots', 'descriptive_plot_export_filenames', 'german_language_control_concept', 'balanced_candidate_ranking_and_current_prompt_compatibility', 'click_to_select_candidate_rows', 'completion_cue_context_matrix', } actual_live_v07 = set(config.get('live_features_v0_7', [])) if actual_live_v07 != required_live_v07: raise SystemExit( 'research_config.json live_features_v0_7 mismatch: ' f'{sorted(actual_live_v07)}' ) required_live_v08 = { 'bounded_plot_focus_overlay_with_scroll_restore', 'explicit_result_table_headings', 'standalone_dose_response_target_and_feature_inputs', 'causal_ready_current_token_candidate_ranking', 'cue_dominance_specificity_interpretation', 'muted_cue_context_plot_palette', } actual_live_v08 = set(config.get('live_features_v0_8', [])) if actual_live_v08 != required_live_v08: raise SystemExit( 'research_config.json live_features_v0_8 mismatch: ' f'{sorted(actual_live_v08)}' ) required_live_v09 = { 'in_place_aspect_preserving_plot_and_table_focus', 'compact_table_heading_alignment', 'concise_independent_dose_response_copy', 'batched_candidate_causal_triage', 'gpu_budget_aware_hf_validation_scope', } actual_live_v09 = set(config.get('live_features_v0_9', [])) if actual_live_v09 != required_live_v09: raise SystemExit( 'research_config.json live_features_v0_9 mismatch: ' f'{sorted(actual_live_v09)}' ) required_live_v10 = { 'discovery_to_causality_alignment_table', 'association_evidence_vs_target_effect_scatter', 'descriptive_spearman_concordance_summary', 'target_effect_vs_distribution_shift_rank_separation', 'no_extra_gpu_candidate_synthesis', } actual_live_v10 = set(config.get('live_features_v0_10', [])) if actual_live_v10 != required_live_v10: raise SystemExit( 'research_config.json live_features_v0_10 mismatch: ' f'{sorted(actual_live_v10)}' ) required_live_v11 = { 'controlled_multi_candidate_random_specificity_screen', 'strategic_discovery_target_js_shortlist', 'association_vs_controlled_causality_alignment', 'target_specificity_vs_js_specificity_separation', 'single_new_gpu_call_hf_acceptance', } actual_live_v11 = set(config.get('live_features_v0_11', [])) if actual_live_v11 != required_live_v11: raise SystemExit( 'research_config.json live_features_v0_11 mismatch: ' f'{sorted(actual_live_v11)}' ) required_live_v12 = { 'controlled_evidence_pattern_synthesis', 'split_half_discovery_stability', 'cross_target_candidate_profile', 'missing_discovery_alignment_fallback', 'gpu_budget_aware_touched_path_validation', } actual_live_v12 = set(config.get('live_features_v0_12', [])) if actual_live_v12 != required_live_v12: raise SystemExit( 'research_config.json live_features_v0_12 mismatch: ' f'{sorted(actual_live_v12)}' ) if config.get('cross_target_feature_limit') != 3 or config.get('cross_target_target_limit') != 5: raise SystemExit('Cross-target live limits must be 3 features and 5 targets.') if 'german_language' not in config.get('concepts', []) or 'french_language' in config.get('concepts', []): raise SystemExit('research_config.json must use german_language and must not contain french_language.') def check_datasets(config: dict) -> tuple[list[dict], list[dict]]: prompts = load_jsonl(ROOT / 'data' / 'prompts.jsonl') causal = load_jsonl(ROOT / 'data' / 'causal_tasks.jsonl') if len(prompts) != config.get('discovery_prompts'): raise SystemExit( f'Discovery prompt count mismatch: found {len(prompts)}, ' f'expected {config.get("discovery_prompts")}.' ) if len(causal) != config.get('causal_tasks'): raise SystemExit( f'Causal task count mismatch: found {len(causal)}, ' f'expected {config.get("causal_tasks")}.' ) concept_counts = Counter(row['concept'] for row in prompts) if set(concept_counts) != set(config.get('concepts', [])): raise SystemExit('Discovery dataset concepts do not match research_config.json.') if len(set(concept_counts.values())) != 1: raise SystemExit(f'Discovery concepts are not balanced: {dict(concept_counts)}') pair_counts = Counter(row['pair_id'] for row in prompts) if set(pair_counts.values()) != {2}: raise SystemExit('Every discovery paraphrase pair must contain exactly two prompts.') return prompts, causal def check_oversized_files() -> None: oversized: list[str] = [] for path in repository_candidates(): size_bytes = path.stat().st_size if size_bytes > MAX_FILE_SIZE_BYTES: relative = path.relative_to(ROOT) oversized.append(f'{relative} ({size_bytes / 1_000_000:.1f} MB)') if oversized: formatted = '\n - '.join(oversized) raise SystemExit( 'Repository contains unexpectedly large tracked/unignored candidates:\n' f' - {formatted}\n\n' 'If a file is a legitimate local artifact, add it to .gitignore. Model weights, ' 'SAE checkpoints, activation dumps, virtual environments, and caches should not be committed.' ) def check_readme() -> None: readme = (ROOT / 'README.md').read_text(encoding='utf-8') required_strings = [ 'sdk: gradio', 'sdk_version: "6.24.0"', 'Qwen/Qwen3-1.7B-Base', 'full-continuation', 'feature-set', 'paraphrase', 'random-control ensemble', 'batched zero-edit', 'concept contrast', 'non-additivity', 'contrastive', 'decoder geometry', 'token activation', 'concept-guided candidate', 'completion-cue', 'cue × context', 'balanced selectivity', 'german', 'start here', 'causal-ready', 'cue-dominant', 'batched causal candidate triage', 'in-place', 'gpu', 'discovery–causality alignment', 'spearman', 'rank-shift', 'no additional gpu', 'controlled candidate specificity', 'norm-matched random ensemble', 'association vs controlled causality', 'target-specificity ratio', 'js-specificity ratio', 'controlled evidence patterns', 'split-half', 'cross-target', 'target-profile', ] missing = [value for value in required_strings if value.lower() not in readme.lower()] if missing: raise SystemExit(f'README.md is missing required v0.12 content: {missing}') def check_pyproject() -> None: text = (ROOT / 'pyproject.toml').read_text(encoding='utf-8') if 'version = "0.12.0"' not in text: raise SystemExit('pyproject.toml must declare version 0.12.0.') def main() -> None: check_required_files() config = json.loads((ROOT / 'research_config.json').read_text(encoding='utf-8')) check_config(config) prompts, causal = check_datasets(config) check_oversized_files() check_readme() check_pyproject() print('FeatureLens release check: PASS') print(f' discovery prompts: {len(prompts)}') print(f' causal tasks: {len(causal)}') print(f' layers: {config["layers"]}') print(f' feature-set sizes: {config["feature_set_sizes"]}') print(f' random controls: {config["live_random_controls"]}') print(' release: v0.12.0') if __name__ == '__main__': main()