| { |
| "artifact_kind": "frozen_vibethinker_v1_readout_reference", |
| "claim_boundary": "This reference records the V1 readout results. It excludes causal-assay results and does not establish free-generation steering or a global workspace.", |
| "classification": "validated_readout_only", |
| "coverage": { |
| "locked_test_items": 377, |
| "readout_items_completed": 539, |
| "readout_items_no_eligible_targets": 12, |
| "released_prompts": 551 |
| }, |
| "coverage_scope": "frozen_readout_source_run_not_the_100_prompt_ui_test_pack", |
| "task_definition": { |
| "aggregate_mean_reciprocal_rank": "mean_across_eligible_target_terms_of_reciprocal_best_rank", |
| "eligible_target": "at_least_one_candidate_form_tokenizes_to_exactly_one_token", |
| "final_model": "rank_target_terms_in_the_model_next_token_logits_at_the_score_position", |
| "item": "one_prompt_with_one_or_more_target_terms", |
| "layer_scope_reduction": "best_target_rank_across_layers_in_the_reported_scope", |
| "no_eligible_target_item": "none_of_the_item_target_terms_has_an_eligible_single_token_form", |
| "paired_bootstrap_mean_reciprocal_rank": "within_item_mean_of_reciprocal_best_rank_across_eligible_target_terms", |
| "pass_at_k": "mean_across_items_of_the_fraction_of_item_target_terms_with_best_rank_at_most_k", |
| "score_position": { |
| "default": "final_prompt_token", |
| "poetry": "last_newline_token" |
| }, |
| "split": { |
| "dev_fraction": 0.3, |
| "method": "sha256_stable_split", |
| "seed": "vibethinker-jlens-v1" |
| }, |
| "suites": ["association", "multihop", "multilingual", "order-ops", "poetry", "typo"], |
| "target_candidate_forms": [ |
| "original_lowercase_and_capitalized_forms_with_and_without_leading_space", |
| "order_ops_also_adds_configured_operation_synonyms_and_number_word_digit_forms" |
| ], |
| "target_rank": "best_one_based_vocabulary_rank_among_eligible_target_token_ids" |
| }, |
| "evidence": { |
| "incremental_over_ordinary_logit_lens": false, |
| "layer_mapping_specific_vs_shuffled_jacobian": true, |
| "paired_bootstrap": { |
| "confidence": 0.95, |
| "input": "paired_item_metric_differences", |
| "interval": "percentile", |
| "metrics": ["pass@10", "mean_reciprocal_rank"], |
| "samples": 2000, |
| "scope": { |
| "kind": "selected_band", |
| "source_layers": [24, 26, 28, 30, 32, 34] |
| }, |
| "split": "test" |
| }, |
| "paired_bootstrap_decisions": { |
| "incremental_over_ordinary_logit_lens": { |
| "comparison": "jlens_band_minus_logit_lens_band", |
| "criterion": "at_least_one_lower_bound_greater_than_zero", |
| "met": false |
| }, |
| "layer_mapping_specific_vs_shuffled_jacobian": { |
| "comparison": "jlens_band_minus_shuffled_layer_band", |
| "criterion": "both_lower_bounds_greater_than_zero", |
| "met": true |
| }, |
| "token_specific_vs_shuffled_target": { |
| "comparison": "jlens_band_minus_shuffled_token_band", |
| "criterion": "both_lower_bounds_greater_than_zero", |
| "met": true |
| } |
| }, |
| "specificity_scope": { |
| "kind": "selected_band", |
| "source_layers": [24, 26, 28, 30, 32, 34] |
| }, |
| "token_specific_vs_shuffled_target": true, |
| "validated_readout_signal": true |
| }, |
| "lens_sha256": "8f752032a26a5196c1cb447ef63f01e8a29820ff0c57178dd80c6e26d32b12a9", |
| "lens_variant": "fp32_evaluation_safetensors_included", |
| "released_lens_artifact": { |
| "filename": "evaluation.safetensors", |
| "format": "safetensors", |
| "sha256": "0cc184eb65d273ac8bfee5450a141c8cdf8dd6ec8caa68d7d47260ef621777d1", |
| "size_bytes": 301991904, |
| "source_checkpoint_sha256": "8f752032a26a5196c1cb447ef63f01e8a29820ff0c57178dd80c6e26d32b12a9", |
| "tensor_conversion": "lossless_fp32_reserialization" |
| }, |
| "source_artifact_availability": { |
| "fp32_compatibility_check_output_included": true, |
| "fp32_compatibility_check_output_path": "evaluation_compatibility.json", |
| "fp32_derivation_record_included": true, |
| "fp32_derivation_record_path": "evaluation_provenance.json", |
| "fp32_evaluation_lens_included": true, |
| "fp32_evaluation_lens_path": "evaluation.safetensors", |
| "item_level_evaluation_rows_included": false, |
| "item_level_evaluation_rows_location": "companion_trace_repository:data/evaluation-results/readout-trials.jsonl", |
| "paired_bootstrap_interval_bounds_included": false, |
| "paired_bootstrap_interval_bounds_location": "companion_trace_repository:data/evaluation-results/readout-bootstrap-intervals.json", |
| "source_evaluation_bundle_included": false, |
| "source_hashes_included": true, |
| "source_hashes_location": "evaluation_provenance.json_and_companion_trace_repository:data/readout-reference.json", |
| "release_content": "fp16_trace_lens_fp32_evaluation_lens_aggregate_reference_provenance_and_compatibility" |
| }, |
| "metrics": { |
| "jlens_all_layers": { |
| "mean_reciprocal_rank": 0.027840488103301416, |
| "n_items": 377, |
| "pass_at_1": 0.009283819628647215, |
| "pass_at_10": 0.08819628647214854, |
| "pass_at_50": 0.1655614500442087 |
| }, |
| "jlens_selected_band": { |
| "mean_reciprocal_rank": 0.01888596779741093, |
| "n_items": 377, |
| "pass_at_1": 0.003978779840848806, |
| "pass_at_10": 0.0629973474801061, |
| "pass_at_50": 0.11914235190097258 |
| }, |
| "logit_lens_all_layers": { |
| "mean_reciprocal_rank": 0.027249274540932285, |
| "n_items": 377, |
| "pass_at_1": 0.007294429708222812, |
| "pass_at_10": 0.08377541998231654, |
| "pass_at_50": 0.16114058355437666 |
| }, |
| "shuffled_layer_all_layers": { |
| "mean_reciprocal_rank": 0.042073366910739256, |
| "n_items": 377, |
| "pass_at_1": 0.04509283819628647, |
| "pass_at_10": 0.08819628647214854, |
| "pass_at_50": 0.1823607427055703 |
| }, |
| "final_model": { |
| "mean_reciprocal_rank": 0.019635164837169684, |
| "n_items": 377, |
| "pass_at_1": 0.003978779840848806, |
| "pass_at_10": 0.0665340406719717, |
| "pass_at_50": 0.10587975243147656 |
| } |
| }, |
| "model": "WeiboAI/VibeThinker-3B", |
| "model_revision": "77bd2cced09193c8b9a59a32bd8577bbd1f3e01c", |
| "note": "The recorded readout metrics bind to evaluation.safetensors, the FP32 evaluation lens in this repository. They do not evaluate model.safetensors, the FP16 lens used for the captured traces.", |
| "selected_band": [24, 26, 28, 30, 32, 34], |
| "schema_version": 1 |
| } |
|
|
