| { |
| "meta": { |
| "experiment": "W10_decoder_reading", |
| "n_sentences": 500, |
| "decoder_card": "text_sonar_basic_decoder", |
| "n_decoder_layers": 24, |
| "n_cross_attn_heads": 16, |
| "z_is_single_position": true, |
| "skeptic_note": "z is ONE vector => decoder cross-attends to a single position (softmax over 1 key ~ degenerate). Localist test reframed as z-DIRECTION ablation -> output-token-change vs matched-norm random control." |
| }, |
| "A_layerwise_emergence": { |
| "n_sentences_probed": 300, |
| "n_content_tokens": 4032, |
| "n_surface_tokens": 4413, |
| "median_emerge_frac_content": 0.6086956521739131, |
| "median_emerge_frac_surface": 1.0, |
| "mean_emerge_frac_content": 0.6372390441683919, |
| "mean_emerge_frac_surface": 0.8954768027271205, |
| "note": "emerge_frac = layer index (0..1) at which lens-top1 first locks to the final token and stays. lower=earlier.", |
| "logitlens_agree_content_by_layer": [ |
| 0.0, |
| 0.0, |
| 0.000496031746031746, |
| 0.000496031746031746, |
| 0.001240079365079365, |
| 0.004712301587301587, |
| 0.012152777777777778, |
| 0.023809523809523808, |
| 0.04290674603174603, |
| 0.0689484126984127, |
| 0.13392857142857142, |
| 0.2708333333333333, |
| 0.39211309523809523, |
| 0.5354662698412699, |
| 0.6495535714285714, |
| 0.7090773809523809, |
| 0.7584325396825397, |
| 0.8038194444444444, |
| 0.8353174603174603, |
| 0.8648313492063492, |
| 0.8953373015873016, |
| 0.890625, |
| 0.8871527777777778, |
| 0.9995039682539683 |
| ], |
| "logitlens_agree_surface_by_layer": [ |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.005211874008610922, |
| 0.012236573759347382, |
| 0.015862225243598458, |
| 0.019034670292318152, |
| 0.02605937004305461, |
| 0.04713346929526399, |
| 0.05755721731248584, |
| 0.10786313165646952, |
| 0.12735100838431906, |
| 0.22818944029005211, |
| 0.28030818037616134, |
| 0.4049399501472921, |
| 0.37049626104690686, |
| 0.41536369816451396, |
| 0.4421028778608656, |
| 0.4579651031044641, |
| 0.4400634489009744, |
| 0.27781554498073874, |
| 0.35871289372309084, |
| 0.9435757987763427 |
| ] |
| }, |
| "B_crossattn_read_by_layer": { |
| "note": "mean ||cross_attn_output|| / ||residual_into_crossattn|| per layer; how strongly z is read at each depth.", |
| "xattn_read_frac_by_layer": [ |
| 11.981749674479167, |
| 5.466305143038432, |
| 7.206465433438619, |
| 9.383789989153543, |
| 10.489324464797974, |
| 9.529227778116862, |
| 10.383447993596395, |
| 11.11037944952647, |
| 7.5069301795959475, |
| 10.289530377388001, |
| 10.700671984354654, |
| 10.48009269396464, |
| 11.221453116734823, |
| 12.288406060536703, |
| 8.880138290723165, |
| 8.605154465039572, |
| 9.007333512306213, |
| 8.356222885449727, |
| 8.236391728719076, |
| 5.922921441396078, |
| 5.301507139205933, |
| 3.4041185712814332, |
| 1.5054391316572826, |
| 1.156544069846471 |
| ], |
| "peak_read_layer": 13, |
| "early_third_mean": 9.443836240768434, |
| "mid_third_mean": 9.996547146042188, |
| "late_third_mean": 5.361309809982776 |
| }, |
| "A_lexical_bag_first": { |
| "note": "frac of final content-token IDs already present ANYWHERE in lens-top1 (bag) by layer.", |
| "content_bag_recall_by_layer": [ |
| 0.994312718671078, |
| 0.9931090149673741, |
| 0.9920685508019101, |
| 0.9905864079447673, |
| 0.9803983530229664, |
| 0.950794700182858, |
| 0.9084043334403238, |
| 0.8453719021489914, |
| 0.7591730988023548, |
| 0.6281410301085218, |
| 0.5171946775274948, |
| 0.49037106638345035, |
| 0.5285413721125798, |
| 0.6213316098675999, |
| 0.7044672625419526, |
| 0.7454325638814795, |
| 0.780488143005944, |
| 0.8166848795482686, |
| 0.8439021312221768, |
| 0.8739997629506138, |
| 0.9017810559753284, |
| 0.8959827667172865, |
| 0.8926178860126225, |
| 0.9992361111111111 |
| ], |
| "bag_recall_at_half_depth": 0.5285413721125798 |
| }, |
| "C_zdirection_influence": { |
| "note": "Perturb z along a direction, re-greedy-decode, measure #words changed (symmetric diff vs baseline word set) + Jaccard. Localist => content-word-ablation dirs change FEW, targeted words (the ablated word vanishes) more than a matched-norm random perturbation does.", |
| "perturbation_magnitude": "content_ablate=full word removal from pool; pca/rand=0.5*||z|| along dir", |
| "content_ablate_dir": { |
| "n": 359, |
| "mean_words_changed": 1.1114206128133706, |
| "std": 1.6467602832006853, |
| "mean_jaccard_to_base": 0.9415911581469185, |
| "frac_targeted_word_vanished": 0.2785515320334262 |
| }, |
| "random_dir_control": { |
| "n": 240, |
| "mean_words_changed": 2.4625, |
| "std": 3.0397796658091303, |
| "mean_jaccard_to_base": 0.8827266463037415 |
| }, |
| "pca_dirs": { |
| "pca0": { |
| "mean_words_changed": 11.9, |
| "std": 6.859786196862601, |
| "mean_jaccard_to_base": 0.5001729309860259 |
| }, |
| "pca1": { |
| "mean_words_changed": 6.666666666666667, |
| "std": 5.377938473264845, |
| "mean_jaccard_to_base": 0.7162827654127722 |
| }, |
| "pca2": { |
| "mean_words_changed": 9.45, |
| "std": 6.103072996450231, |
| "mean_jaccard_to_base": 0.5888467391794264 |
| }, |
| "pca3": { |
| "mean_words_changed": 9.716666666666667, |
| "std": 6.258571899580784, |
| "mean_jaccard_to_base": 0.6079678794291621 |
| }, |
| "pca4": { |
| "mean_words_changed": 10.283333333333333, |
| "std": 6.537307464766277, |
| "mean_jaccard_to_base": 0.5825735608825627 |
| }, |
| "pca5": { |
| "mean_words_changed": 8.716666666666667, |
| "std": 5.413229678810567, |
| "mean_jaccard_to_base": 0.6322222442879579 |
| } |
| }, |
| "localist_signal": "content-ablation changes fewer/targeted words than random", |
| "examples": [ |
| { |
| "sentence": "New Bedford, MA - A local man was arrested and charged with heroin distribution after a multi-agency investigation, according to authorities.", |
| "baseline_decode": "New Bedford, MA - A local man was arrested and charged with distributing heroin after a multi-agency investigation, according to authorities.", |
| "examples": [ |
| { |
| "dir": "New", |
| "kind": "content_ablate", |
| "decode": "New Bedford, MA - A local man was arrested and charged with distributing heroin after a multi-agency investigation, according to authorities." |
| }, |
| { |
| "dir": "Bedford", |
| "kind": "content_ablate", |
| "decode": "New Bedford, MA - A local man was arrested and charged with distributing heroin after a multi-agency investigation, according to authorities." |
| }, |
| { |
| "dir": "local", |
| "kind": "content_ablate", |
| "decode": "New Bedford, MA - A man was arrested and charged with distributing heroin after a multi-agency investigation, according to authorities." |
| }, |
| { |
| "dir": "man", |
| "kind": "content_ablate", |
| "decode": "New Bedford, MA - A local man was arrested and charged with distributing heroin after a multi-agency investigation, according to authorities." |
| }, |
| { |
| "dir": "arrested", |
| "kind": "content_ablate", |
| "decode": "New Bedford, MA - A local man was arrested and charged with distribution of heroin after a multi-agency investigation, according to authorities." |
| }, |
| { |
| "dir": "charged", |
| "kind": "content_ablate", |
| "decode": "New Bedford, MA - A local man was arrested and charged with distribution of heroin after a multi-agency investigation, according to authorities." |
| }, |
| { |
| "dir": "pca0", |
| "kind": "pca", |
| "decode": "New Bedford, MA - And our own people are in a position to be confronted by the police and the police." |
| }, |
| { |
| "dir": "pca1", |
| "kind": "pca", |
| "decode": "New Bedford, MAYBROW - A local man was arrested and charged with a heroine after a multi-agent investigation, according to the authorities." |
| } |
| ] |
| }, |
| { |
| "sentence": "The school's annual awards ceremony saw students praised for their academic, artistic, and sporting achievements.", |
| "baseline_decode": "The school's annual awards ceremony saw students being praised for their academic, artistic, and sports achievements.", |
| "examples": [ |
| { |
| "dir": "school", |
| "kind": "content_ablate", |
| "decode": "The school's annual awards ceremony saw students being praised for their academic, artistic, and sports achievements." |
| }, |
| { |
| "dir": "annual", |
| "kind": "content_ablate", |
| "decode": "The school's awards ceremony saw students being praised for their academic, artistic, and sports achievements." |
| }, |
| { |
| "dir": "awards", |
| "kind": "content_ablate", |
| "decode": "The school's annual award ceremony saw students being praised for their academic, artistic, and sports achievements." |
| }, |
| { |
| "dir": "ceremony", |
| "kind": "content_ablate", |
| "decode": "The school's annual awards ceremony saw students being praised for their academic, artistic, and sports achievements." |
| }, |
| { |
| "dir": "saw", |
| "kind": "content_ablate", |
| "decode": "The school's annual awards ceremony saw students being praised for their academic, artistic, and sports achievements." |
| }, |
| { |
| "dir": "students", |
| "kind": "content_ablate", |
| "decode": "The school's annual awards ceremony saw students being praised for their academic, artistic, and sports achievements." |
| }, |
| { |
| "dir": "pca0", |
| "kind": "pca", |
| "decode": "The school's annual awards ceremony sees students praised for their achievements and achievements." |
| }, |
| { |
| "dir": "pca1", |
| "kind": "pca", |
| "decode": "The school's annual awards feast at school, students were praised for their academic, artistic, and sports achievements." |
| } |
| ] |
| }, |
| { |
| "sentence": "As I ascended to the rooftop Gravity Bar, I was treated to stunning views of the city and a unique perspective on the brewing process.", |
| "baseline_decode": "As I climbed up to the rooftop Gravity Bar, I was given a view of the city's amazing and a unique perspective on the brewing process.", |
| "examples": [ |
| { |
| "dir": "ascended", |
| "kind": "content_ablate", |
| "decode": "As I climbed on the rooftop Gravity Bar, I was given a breathtaking view of the city and a unique perspective on the brewing process." |
| }, |
| { |
| "dir": "rooftop", |
| "kind": "content_ablate", |
| "decode": "As I climbed to the Gravity Bar, I was given a view of the city's amazing and a unique perspective on the brewing process." |
| }, |
| { |
| "dir": "Gravity", |
| "kind": "content_ablate", |
| "decode": "As I climbed up to the rooftop Bar of Grace, I was given a view of the city's amazing and a unique perspective on the brewing process." |
| }, |
| { |
| "dir": "Bar", |
| "kind": "content_ablate", |
| "decode": "As I climbed to the rooftop Gravity Bar, I was given a breathtaking view of the city and a unique perspective on the brewing process." |
| }, |
| { |
| "dir": "treated", |
| "kind": "content_ablate", |
| "decode": "As I climbed up on the rooftop Gravity Bar, I was given a breathtaking view of the city and a unique perspective on the brewing process." |
| }, |
| { |
| "dir": "stunning", |
| "kind": "content_ablate", |
| "decode": "As I climbed up to the rooftop Gravity Bar, I was given a view of the city and a unique perspective on the brewing process." |
| }, |
| { |
| "dir": "pca0", |
| "kind": "pca", |
| "decode": "As I get on to the surface on gravity bar, I get to see the beautiful view of the world and an unusual understanding of the process of brewing." |
| }, |
| { |
| "dir": "pca1", |
| "kind": "pca", |
| "decode": "As I climbed up on the rooftop Gravity Bar, I was graced with the breathtaking view of the city and a unique look at the groove of the brew." |
| } |
| ] |
| }, |
| { |
| "sentence": "I'm happy to report that some of the best cookies I've ever had were from beloved institutions like Levain Bakery and Jacques Torres Chocolate.", |
| "baseline_decode": "I am happy to report that some of the best cookies I have ever had were from beloved institutions like Levain Bakery and Jacques Torres Chocolate.", |
| "examples": [ |
| { |
| "dir": "happy", |
| "kind": "content_ablate", |
| "decode": "I am happy to report that some of the best cookies I have ever had were from beloved institutions like Levain Bakery and Jacques Torres Chocolate." |
| }, |
| { |
| "dir": "report", |
| "kind": "content_ablate", |
| "decode": "I'm happy to report that some of the best cookies I've ever had were from beloved institutions like Levain Bakery and Jacques Torres Chocolate." |
| }, |
| { |
| "dir": "some", |
| "kind": "content_ablate", |
| "decode": "I am happy to report that some of the best cookies I have ever had were from beloved institutions like Levain Bakery and Jacques Torres Chocolate." |
| }, |
| { |
| "dir": "best", |
| "kind": "content_ablate", |
| "decode": "I am happy to report that some of the best cookies I have ever had were from beloved institutions like Levain Bakery and Jacques Torres Chocolate." |
| }, |
| { |
| "dir": "cookies", |
| "kind": "content_ablate", |
| "decode": "I am happy to report that some of the best cakes I have ever had were from beloved institutions like Levain Bakery and Jacques Torres Chocolate." |
| }, |
| { |
| "dir": "ever", |
| "kind": "content_ablate", |
| "decode": "I am happy to report that some of the best cookies I've had were from beloved institutions like Levain Bakery and Jacques Torres Chocolate." |
| }, |
| { |
| "dir": "pca0", |
| "kind": "pca", |
| "decode": "I'm happy to tell you that some of the best memories I've ever had are from my parents and my friends." |
| }, |
| { |
| "dir": "pca1", |
| "kind": "pca", |
| "decode": "I'm happy to tell you some of the finest chocolates I've ever had were from the lovely Levain Bakery and Jacques Torres Chocolate." |
| } |
| ] |
| }, |
| { |
| "sentence": "Under the deal, Saudi Arabia and Iran will increase their combined oil production to 12 million barrels per day, from 9 million currently.", |
| "baseline_decode": "Under the deal, Saudi Arabia and Iran will increase their combined oil production to 12 million barrels per day, from 9 million currently.", |
| "examples": [ |
| { |
| "dir": "Under", |
| "kind": "content_ablate", |
| "decode": "Under the deal, Saudi Arabia and Iran will increase their combined oil production to 12 million barrels per day, from 9 million currently." |
| }, |
| { |
| "dir": "deal", |
| "kind": "content_ablate", |
| "decode": "Under the deal, Saudi Arabia and Iran will increase their combined oil production to 12 million barrels per day, from 9 million currently." |
| }, |
| { |
| "dir": "Saudi", |
| "kind": "content_ablate", |
| "decode": "Under the deal, Saudi Arabia and Iran will increase their combined oil production to 12 million barrels per day, from 9 million currently." |
| }, |
| { |
| "dir": "Arabia", |
| "kind": "content_ablate", |
| "decode": "Under the deal, Saudi Arabia and Iran will increase their combined oil production to 12 million barrels per day, from 9 million currently." |
| }, |
| { |
| "dir": "Iran", |
| "kind": "content_ablate", |
| "decode": "Under the deal, Saudi Arabia and Iran will increase their combined oil production to 12 million barrels per day, from 9 million currently." |
| }, |
| { |
| "dir": "increase", |
| "kind": "content_ablate", |
| "decode": "Under the deal, Saudi Arabia and Iran will increase their combined oil production to 12 million barrels per day, from 9 million currently." |
| }, |
| { |
| "dir": "pca0", |
| "kind": "pca", |
| "decode": "Under the deal, we will increase our oil production and oil production to 9 billion barrels a day." |
| }, |
| { |
| "dir": "pca1", |
| "kind": "pca", |
| "decode": "Under the deal, Saudia Arabia and Iran and her combined oil production will rise to 12 million barrels a day, from 9 million barrels today." |
| } |
| ] |
| }, |
| { |
| "sentence": "Perched atop the rolling hills of the Scottish Highlands, Edinburgh is a city that seamlessly blends its rich history with modern architectural splendor.", |
| "baseline_decode": "Edinburgh is situated on the hillside of the Scottish Highlands, an city that seamlessly blends its rich history with modern architectural splendor.", |
| "examples": [ |
| { |
| "dir": "Perched", |
| "kind": "content_ablate", |
| "decode": "Edinburgh is built on the rolling hills of the Scottish Highlands, an city that seamlessly mixes its rich history with modern architectural splendor." |
| }, |
| { |
| "dir": "atop", |
| "kind": "content_ablate", |
| "decode": "Edinburgh is situated on the rolling hills of the Scottish Highlands, an city that seamlessly blends its rich history with modern architectural splendor." |
| }, |
| { |
| "dir": "rolling", |
| "kind": "content_ablate", |
| "decode": "Edinburgh is situated on the hillsides of the Scottish Highlands, an city that seamlessly blends its rich history with modern architectural splendor." |
| }, |
| { |
| "dir": "hills", |
| "kind": "content_ablate", |
| "decode": "Edinburgh is situated on the rocky slopes of the Scottish Highlands, an city that seamlessly blends its rich history with modern architectural splendor." |
| }, |
| { |
| "dir": "Scottish", |
| "kind": "content_ablate", |
| "decode": "Situated on the crest of the rolling hills of the Edinburgh Highlands, Edinburgh is a city that seamlessly blends its rich history with modern architectural splendor." |
| }, |
| { |
| "dir": "Highlands", |
| "kind": "content_ablate", |
| "decode": "Edinburgh is situated on the crest of the rolling hills of Scottish Highlands, an city that seamlessly mixes its rich history with modern architectural splendor." |
| }, |
| { |
| "dir": "pca0", |
| "kind": "pca", |
| "decode": "Is it not true that the present-day world is a mess of uncertainties and uncertainties?" |
| }, |
| { |
| "dir": "pca1", |
| "kind": "pca", |
| "decode": "Edinburgh is nestled on the hillside of the rocky slopes of the Scottish Highlands, Edinburgh is a town that unmistakably mixes its dazzling history with its splendor of modern architecture." |
| } |
| ] |
| } |
| ] |
| }, |
| "SCORING": { |
| "prediction": "decoder reads z DIFFUSELY (single position, no per-output-token z-direction map); reconstructs content/topic EARLY, refines surface form LATE; reading is DISTRIBUTED (no clean token->z-dir map).", |
| "falsifier": "clean per-output-token z-direction decode map (localist handle) OR z merely sets an initial state (trivial reading).", |
| "single_position_confirmed": true, |
| "content_emerges_before_surface": true, |
| "z_read_across_depth_not_just_init": true, |
| "clean_localist_token_to_zdir_map": false, |
| "verdict": "CONFIRMED: diffuse/distributed reading; content early, surface late" |
| } |
| } |
