| { |
| "format": "blah-jacobian-lens-validation", |
| "version": 1, |
| "status": "pass", |
| "model_revision": "ab1c5be13a12c0feb2d5e2c9af89bd5924a0e8b0", |
| "checkpoint_fingerprint": "sha256:399f776b49acc0c8834ff8a7f2390454e2c5f2d833a264e3f83ff546e973cfec", |
| "weights_fingerprint": "sha256:282087102243eb19cbdffd1dbf19d776b7f2e9f73029283e21b20bf79a5c66d4", |
| "source_repository_commit": "7f7111c18f3fea6ceee9f83f7e51a51e2c2fc0c6", |
| "adapter_commit": "7f7111c18f3fea6ceee9f83f7e51a51e2c2fc0c6", |
| "runtime_version": "@alpha/lens 0.1.0; blah-lens-http/1", |
| "framework": "alpha2 native TypeScript autograd and tensor engine", |
| "operating_system": "linux x64", |
| "device": "helios", |
| "dtype": "float32 validation; bundle export dtype from manifest", |
| "tests": [ |
| { |
| "name": "checkpoint fingerprint verification", |
| "status": "pass", |
| "measurements": { |
| "checkpoint_sha256": "sha256:399f776b49acc0c8834ff8a7f2390454e2c5f2d833a264e3f83ff546e973cfec", |
| "manifest_weights_fingerprint": "sha256:282087102243eb19cbdffd1dbf19d776b7f2e9f73029283e21b20bf79a5c66d4", |
| "loaded_weights_fingerprint": "sha256:282087102243eb19cbdffd1dbf19d776b7f2e9f73029283e21b20bf79a5c66d4", |
| "manifest_config_fingerprint": "sha256:1c0bc9ee48de7c105abb00a189623ea369117e9d2d9dd431f1bd48ca996f6d5f", |
| "loaded_config_fingerprint": "sha256:1c0bc9ee48de7c105abb00a189623ea369117e9d2d9dd431f1bd48ca996f6d5f", |
| "requested_hf_revision": "ab1c5be13a12c0feb2d5e2c9af89bd5924a0e8b0", |
| "resolved_hf_revision": "ab1c5be13a12c0feb2d5e2c9af89bd5924a0e8b0", |
| "hf_error": null, |
| "declared_hf_files": { |
| "model.safetensors": "sha256:a5214ebad501b8bd3b09f7552c0db67417d18c3b66432f66f847de0e723dd688", |
| "config.json": "sha256:d9080edae93005a8738604d86bfafa93429fa839ee2ca4c0e1f8901074c8616a", |
| "generation_config.json": "sha256:097ffca0d0dec269ef98214f0399f091452f88b57cc6334208a9904f68e59fd6", |
| "tokenizer.json": "sha256:37372c9b1bdbf7d9655444e90247bef957018d0d7ff0b668d1330e28d97c44cf", |
| "tokenizer_config.json": "sha256:75b8671684f5643da2c5bf397b5a4f021c12842fb355c6431d409f4e91c60245", |
| "chat_template.jinja": "sha256:8c08c7e2eaec9375356477033e1532b814fb75aff8dcf1cc1bf2b04fd93a00a7" |
| }, |
| "native_reexport_files": { |
| "model.safetensors": "sha256:a5214ebad501b8bd3b09f7552c0db67417d18c3b66432f66f847de0e723dd688", |
| "config.json": "sha256:d9080edae93005a8738604d86bfafa93429fa839ee2ca4c0e1f8901074c8616a", |
| "generation_config.json": "sha256:097ffca0d0dec269ef98214f0399f091452f88b57cc6334208a9904f68e59fd6", |
| "tokenizer.json": "sha256:37372c9b1bdbf7d9655444e90247bef957018d0d7ff0b668d1330e28d97c44cf", |
| "tokenizer_config.json": "sha256:75b8671684f5643da2c5bf397b5a4f021c12842fb355c6431d409f4e91c60245", |
| "chat_template.jinja": "sha256:8c08c7e2eaec9375356477033e1532b814fb75aff8dcf1cc1bf2b04fd93a00a7" |
| }, |
| "remote_revision_files": { |
| "model.safetensors": "sha256:a5214ebad501b8bd3b09f7552c0db67417d18c3b66432f66f847de0e723dd688", |
| "config.json": "sha256:d9080edae93005a8738604d86bfafa93429fa839ee2ca4c0e1f8901074c8616a", |
| "generation_config.json": "sha256:097ffca0d0dec269ef98214f0399f091452f88b57cc6334208a9904f68e59fd6", |
| "tokenizer.json": "sha256:37372c9b1bdbf7d9655444e90247bef957018d0d7ff0b668d1330e28d97c44cf", |
| "tokenizer_config.json": "sha256:75b8671684f5643da2c5bf397b5a4f021c12842fb355c6431d409f4e91c60245", |
| "chat_template.jinja": "sha256:8c08c7e2eaec9375356477033e1532b814fb75aff8dcf1cc1bf2b04fd93a00a7" |
| }, |
| "native_reexport_exact_match": true, |
| "remote_revision_exact_match": true, |
| "native_export_error": null |
| } |
| }, |
| { |
| "name": "tokenizer parity", |
| "status": "pass", |
| "measurements": { |
| "round_trips": [ |
| { |
| "text": "plain ASCII", |
| "ids": [ |
| 521, |
| 429, |
| 334, |
| 7385, |
| 2109 |
| ], |
| "decoded": "plain ASCII", |
| "exact": true |
| }, |
| { |
| "text": " café 🎉", |
| "ids": [ |
| 1410, |
| 102, |
| 2205, |
| 32, |
| 240, |
| 159, |
| 142, |
| 137 |
| ], |
| "decoded": " café 🎉", |
| "exact": true |
| }, |
| { |
| "text": "<|user|> hi <|assistant|>", |
| "ids": [ |
| 256, |
| 299, |
| 105, |
| 32, |
| 257 |
| ], |
| "decoded": "<|user|> hi <|assistant|>", |
| "exact": true |
| }, |
| { |
| "text": "line one\nline two", |
| "ids": [ |
| 1301, |
| 553, |
| 10, |
| 1301, |
| 772 |
| ], |
| "decoded": "line one\nline two", |
| "exact": true |
| }, |
| { |
| "text": "DNA? evidence—claim", |
| "ids": [ |
| 68, |
| 78, |
| 65, |
| 63, |
| 2918, |
| 1350, |
| 10499 |
| ], |
| "decoded": "DNA? evidence—claim", |
| "exact": true |
| } |
| ], |
| "special_tokens": [ |
| "<|user|>", |
| "<|assistant|>", |
| "<|end_of_text|>" |
| ], |
| "bos_inserted_by_native_encode": false, |
| "eos_policy": "atomic end_of_text terminates the conversation", |
| "tokenizer_fingerprint_match": true, |
| "hf_vocab_exact_match": true, |
| "hf_chat_template_exact_match": true, |
| "byte_vocab_raw_identity_exact": true, |
| "byte_vocab_tokens_checked": 12285, |
| "byte_vocab_identity_error": null, |
| "hf_error": null |
| } |
| }, |
| { |
| "name": "final logit parity", |
| "status": "pass", |
| "tolerance": { |
| "max_absolute_error": 0.000001, |
| "top1_agreement": 1 |
| }, |
| "measurements": { |
| "max_absolute_error": 0, |
| "max_relative_error": 0, |
| "top1_agreement": 1, |
| "cases": [ |
| { |
| "text": "raw text", |
| "tokens": 2, |
| "maxAbs": 0, |
| "maxRel": 0, |
| "top1Same": 2, |
| "positions": 2 |
| }, |
| { |
| "text": "<|end_of_text|>", |
| "tokens": 1, |
| "maxAbs": 0, |
| "maxRel": 0, |
| "top1Same": 1, |
| "positions": 1 |
| }, |
| { |
| "text": "Several tokens test the final decoder path.", |
| "tokens": 13, |
| "maxAbs": 0, |
| "maxRel": 0, |
| "top1Same": 13, |
| "positions": 13 |
| }, |
| { |
| "text": "<|user|> What is DNA? <|assistant|>", |
| "tokens": 7, |
| "maxAbs": 0, |
| "maxRel": 0, |
| "top1Same": 7, |
| "positions": 7 |
| }, |
| { |
| "generated_token": 317, |
| "maxAbs": 0, |
| "maxRel": 0, |
| "top1Same": 10, |
| "positions": 10 |
| } |
| ] |
| } |
| }, |
| { |
| "name": "determinism and replicated batch equality", |
| "status": "pass", |
| "tolerance": { |
| "max_absolute_error": 0 |
| }, |
| "measurements": { |
| "repeated_forward_max_absolute_error": 0, |
| "replicated_batch_max_absolute_error": 0, |
| "dropout": "disabled", |
| "stochastic_routing": "not present" |
| } |
| }, |
| { |
| "name": "native inference capture parity", |
| "status": "pass", |
| "tolerance": { |
| "site_max_absolute_error": 0.002, |
| "logit_max_absolute_error": 0.02, |
| "top1_agreement": 1 |
| }, |
| "measurements": { |
| "sites": [ |
| "block.000.post", |
| "block.015.post" |
| ], |
| "prompt_tokens": 10, |
| "generated_token": 370, |
| "prompt_site_max_absolute_error": 0.00012111663818359375, |
| "generated_site_max_absolute_error": 0.00004291534423828125, |
| "prompt_logit_max_absolute_error": 0.00004291534423828125, |
| "generated_logit_max_absolute_error": 0.00001811981201171875, |
| "prompt_top1_agreement": 1, |
| "generated_top1_agreement": 1, |
| "prefix_recomputed_during_generation": false |
| } |
| }, |
| { |
| "name": "VJP finite-difference check", |
| "status": "pass", |
| "tolerance": { |
| "absolute_error": 0.002, |
| "relative_error": 0.05 |
| }, |
| "measurements": { |
| "source_site": "block.008.post", |
| "target_direction": "deterministic dense sinusoidal direction", |
| "dtype": "float32", |
| "device": "helios", |
| "epsilon_values": [ |
| 0.005, |
| 0.002, |
| 0.001 |
| ], |
| "numeric_estimator": "median of central differences across epsilon sweep", |
| "analytic": 0.004693400368284951, |
| "numeric_estimates": [ |
| 0.0045629600238683565, |
| 0.004635362606197901, |
| 0.0072345167465570626 |
| ], |
| "selected_numeric_estimate": 0.004635362606197901, |
| "absolute_error": 0.000058037762087049215, |
| "relative_error": 0.012365823823433417, |
| "cosine_agreement": 1 |
| } |
| }, |
| { |
| "name": "same-position estimator synthetic oracle", |
| "status": "pass", |
| "tolerance": { |
| "maximum_absolute_error": 0.015 |
| }, |
| "measurements": { |
| "synthetic_jacobian": "intentionally causal and position-coupled; diagonal target/source blocks are the oracle", |
| "probes": 8192, |
| "expected_mean_diagonal": [ |
| 3.0600000000000005, |
| 3.26, |
| 3.7600000000000007, |
| 3.96 |
| ], |
| "observed_estimate": [ |
| 3.059297800064087, |
| 3.2597150802612305, |
| 3.7495651245117188, |
| 3.9472463130950928 |
| ], |
| "maximum_absolute_error": 0.012753686904907191 |
| } |
| }, |
| { |
| "name": "matrix orientation", |
| "status": "pass", |
| "measurements": { |
| "matrix": [ |
| [ |
| 1, |
| 2 |
| ], |
| [ |
| 3, |
| 5 |
| ] |
| ], |
| "source": [ |
| 7, |
| 11 |
| ], |
| "stored_convention": "J[output_dimension,input_dimension]", |
| "application": "h @ transpose(J)", |
| "observed": [ |
| 29, |
| 76 |
| ], |
| "expected": [ |
| 29, |
| 76 |
| ], |
| "intentionally_wrong_transpose_result": [ |
| 40, |
| 69 |
| ] |
| } |
| }, |
| { |
| "name": "affine centering application", |
| "status": "pass", |
| "measurements": { |
| "formula": "target_mean + (source - source_mean) @ transpose(J)", |
| "source": [ |
| 7, |
| 11 |
| ], |
| "source_mean": [ |
| 2, |
| 3 |
| ], |
| "target_mean": [ |
| 13, |
| 17 |
| ], |
| "observed": [ |
| 34, |
| 72 |
| ], |
| "expected": [ |
| 34, |
| 72 |
| ] |
| } |
| }, |
| { |
| "name": "final-site identity", |
| "status": "pass", |
| "measurements": { |
| "site": "block.015.post", |
| "target_site": "decoder.final.post", |
| "representation_max_absolute_error": 0, |
| "logit_max_absolute_error": 0, |
| "top1_agreement": 1 |
| } |
| }, |
| { |
| "name": "transport shape validation", |
| "status": "pass", |
| "measurements": { |
| "site_count": 16, |
| "tensor_count": 33, |
| "orientation": "[target_width,source_width]", |
| "centering": { |
| "mode": "affine", |
| "target_mean_key": "mean.target" |
| }, |
| "problems": [] |
| } |
| }, |
| { |
| "name": "exported dtype parity", |
| "status": "pass", |
| "tolerance": { |
| "top1_agreement": 0.99, |
| "top5_overlap": 0.95 |
| }, |
| "measurements": { |
| "float32_reference": "fit-state sums divided by valid prompt count", |
| "exported_dtype": "float16", |
| "relative_matrix_error": 0.0046745214840570965, |
| "relative_activation_mean_error": 0.0003218012301417836, |
| "heldout_top1_agreement": 1, |
| "heldout_top5_overlap": 1, |
| "maximum_logit_error": 0.05675506591796875, |
| "readout_rank_correlation": "reported in split-half convergence", |
| "worst_affected_source_site": "block.003.post" |
| } |
| }, |
| { |
| "name": "split-half convergence", |
| "status": "pass", |
| "measurements": { |
| "split": "even-versus-odd valid fitting prompts", |
| "heldout_prompt": { |
| "source": "/workspace/alpha-lens-artifacts/same-position-affine-d06fc71/fit-prompts.jsonl", |
| "index": 99, |
| "token_count": 92 |
| }, |
| "heldout_top1_agreement": 0.8125, |
| "heldout_top5_overlap": 0.9625, |
| "heldout_readout_rank_correlation": 0.9917821190834761, |
| "per_site": { |
| "block.000.post": { |
| "relative_frobenius_difference": 0.11330720628413433, |
| "mean_row_cosine_similarity": 0.9937496335120167, |
| "heldout_top1_agreement": 1, |
| "heldout_top5_overlap": 1, |
| "heldout_readout_rank_correlation": 0.993633726304403 |
| }, |
| "block.001.post": { |
| "relative_frobenius_difference": 0.11215034288173786, |
| "mean_row_cosine_similarity": 0.9938853727320716, |
| "heldout_top1_agreement": 1, |
| "heldout_top5_overlap": 1, |
| "heldout_readout_rank_correlation": 0.9889375964714634 |
| }, |
| "block.002.post": { |
| "relative_frobenius_difference": 0.11150581338117614, |
| "mean_row_cosine_similarity": 0.9939526843390902, |
| "heldout_top1_agreement": 1, |
| "heldout_top5_overlap": 1, |
| "heldout_readout_rank_correlation": 0.9902789998000545 |
| }, |
| "block.003.post": { |
| "relative_frobenius_difference": 0.11005337951364569, |
| "mean_row_cosine_similarity": 0.9941146433162439, |
| "heldout_top1_agreement": 1, |
| "heldout_top5_overlap": 0.8, |
| "heldout_readout_rank_correlation": 0.9876347950733294 |
| }, |
| "block.004.post": { |
| "relative_frobenius_difference": 0.10760264511474193, |
| "mean_row_cosine_similarity": 0.9943777003488947, |
| "heldout_top1_agreement": 1, |
| "heldout_top5_overlap": 1, |
| "heldout_readout_rank_correlation": 0.9894897853906315 |
| }, |
| "block.005.post": { |
| "relative_frobenius_difference": 0.10669721675285727, |
| "mean_row_cosine_similarity": 0.9944726500068686, |
| "heldout_top1_agreement": 1, |
| "heldout_top5_overlap": 1, |
| "heldout_readout_rank_correlation": 0.986223949034681 |
| }, |
| "block.006.post": { |
| "relative_frobenius_difference": 0.10022025691966024, |
| "mean_row_cosine_similarity": 0.9951309006898356, |
| "heldout_top1_agreement": 1, |
| "heldout_top5_overlap": 1, |
| "heldout_readout_rank_correlation": 0.9907683106950765 |
| }, |
| "block.007.post": { |
| "relative_frobenius_difference": 0.09711572079975297, |
| "mean_row_cosine_similarity": 0.9954422550415949, |
| "heldout_top1_agreement": 0, |
| "heldout_top5_overlap": 0.8, |
| "heldout_readout_rank_correlation": 0.9828038813325833 |
| }, |
| "block.008.post": { |
| "relative_frobenius_difference": 0.09313701365433766, |
| "mean_row_cosine_similarity": 0.995817167219063, |
| "heldout_top1_agreement": 0, |
| "heldout_top5_overlap": 1, |
| "heldout_readout_rank_correlation": 0.9907449197911632 |
| }, |
| "block.009.post": { |
| "relative_frobenius_difference": 0.08822666489273606, |
| "mean_row_cosine_similarity": 0.99625857960613, |
| "heldout_top1_agreement": 0, |
| "heldout_top5_overlap": 1, |
| "heldout_readout_rank_correlation": 0.9899805549567106 |
| }, |
| "block.010.post": { |
| "relative_frobenius_difference": 0.08168784842376356, |
| "mean_row_cosine_similarity": 0.9968006350566984, |
| "heldout_top1_agreement": 1, |
| "heldout_top5_overlap": 1, |
| "heldout_readout_rank_correlation": 0.9937126822805975 |
| }, |
| "block.011.post": { |
| "relative_frobenius_difference": 0.073095319263959, |
| "mean_row_cosine_similarity": 0.9974544951921425, |
| "heldout_top1_agreement": 1, |
| "heldout_top5_overlap": 0.8, |
| "heldout_readout_rank_correlation": 0.9949926405412486 |
| }, |
| "block.012.post": { |
| "relative_frobenius_difference": 0.06848760695039217, |
| "mean_row_cosine_similarity": 0.9977674823272114, |
| "heldout_top1_agreement": 1, |
| "heldout_top5_overlap": 1, |
| "heldout_readout_rank_correlation": 0.9943759981687764 |
| }, |
| "block.013.post": { |
| "relative_frobenius_difference": 0.056115025606344446, |
| "mean_row_cosine_similarity": 0.9985084720136246, |
| "heldout_top1_agreement": 1, |
| "heldout_top5_overlap": 1, |
| "heldout_readout_rank_correlation": 0.9963402025443784 |
| }, |
| "block.014.post": { |
| "relative_frobenius_difference": 0.03242463899426187, |
| "mean_row_cosine_similarity": 0.9994884095095996, |
| "heldout_top1_agreement": 1, |
| "heldout_top5_overlap": 1, |
| "heldout_readout_rank_correlation": 0.9985958631445483 |
| }, |
| "block.015.post": { |
| "relative_frobenius_difference": 0.0000011920968035056725, |
| "mean_row_cosine_similarity": 1, |
| "heldout_top1_agreement": 1, |
| "heldout_top5_overlap": 1, |
| "heldout_readout_rank_correlation": 0.9999999998059744 |
| } |
| }, |
| "informational": true |
| }, |
| "detail": "Informational measurement; no convergence threshold is used to select the checkpoint." |
| }, |
| { |
| "name": "golden fixture reproduction", |
| "status": "pass", |
| "measurements": { |
| "tokens": 8, |
| "selected_sites": [ |
| "block.000.post", |
| "block.008.post", |
| "block.015.post" |
| ], |
| "tensor_artifact": "fixtures/golden.safetensors", |
| "chat_formatted": true |
| } |
| } |
| ], |
| "known_limitations": [ |
| "No low-rank artifacts are exported because dense transports are compact for this architecture.", |
| "The same-position transport uses one deterministic Rademacher position probe per fitting prompt and output dimension; it is an unbiased diagonal-Jacobian estimator, not an exhaustive per-position Jacobian enumeration." |
| ], |
| "created_at": "2026-08-02T00:41:37.182Z" |
| } |
|
|
