| { |
| "experiment": "P5 eigen-role probe at PRE-POOL token level (R_role from COMPOSITION operator)", |
| "config": { |
| "n_pairs": 2500, |
| "n_matched_pairs": 2500, |
| "n_span_tokens": 131416, |
| "n_pi_planes": 32, |
| "n_id_planes": 32, |
| "seed": 0, |
| "d": 1024, |
| "role_source": "tae_comp_W_order.pt" |
| }, |
| "R_role_meta": { |
| "orth_dev": 0.0, |
| "source": "tae_comp_W_order.pt S_matrix (z_AB->z_BA), polar-orthogonal part", |
| "singular_value_spread": [ |
| 1.9996, |
| 0.0 |
| ] |
| }, |
| "role_plane_angles": { |
| "pi_subspace_mean_rad": 3.1039, |
| "pi_subspace_min_rad": 3.066, |
| "near_identity_mean_rad": 0.00337, |
| "rand_planes_mean_rad": 0.9819, |
| "max_plane_rad": 3.1416, |
| "median_plane_rad": 0.1141 |
| }, |
| "prediction": "pi-subspace does NOT separate spans better than matched random subspace (R_role = generic content-mixing rotation)", |
| "falsifier": "pi-subspace cleanly separates spans (>0.90 acc/auc) AND beats random subspace", |
| "span_separation": { |
| "raw_full_space": { |
| "acc": 0.9786, |
| "auc": 0.9968, |
| "n_train": 105234, |
| "n_test": 26182 |
| }, |
| "pi_eigen_subspace": { |
| "acc": 0.9284, |
| "auc": 0.9704, |
| "n_train": 105234, |
| "n_test": 26182 |
| }, |
| "near_identity_subspace": { |
| "acc": 0.6298, |
| "auc": 0.6517, |
| "n_train": 105234, |
| "n_test": 26182 |
| }, |
| "random_orthonormal_subspace": { |
| "acc": 0.846, |
| "auc": 0.9007, |
| "n_train": 105234, |
| "n_test": 26182 |
| }, |
| "random_eigenplane_subspace": { |
| "acc": 0.8773, |
| "auc": 0.9362, |
| "n_train": 105234, |
| "n_test": 26182 |
| }, |
| "pi_minus_identity_acc": 0.2986, |
| "pi_minus_random_acc": 0.0824, |
| "pi_minus_random_auc": 0.0697, |
| "subspace_mean_coord_norm": { |
| "pi": 0.1727, |
| "near_identity": 0.237, |
| "random_orthonormal": 0.2088, |
| "random_eigenplane": 0.2006 |
| } |
| }, |
| "confounds": { |
| "pi_after_R_pos_inv": { |
| "acc": 0.6949, |
| "auc": 0.7558, |
| "n_train": 105234, |
| "n_test": 26182 |
| }, |
| "R_pos_inv_acc_drop": 0.2335, |
| "pi_after_R_role_inv": { |
| "acc": 0.9284, |
| "auc": 0.9704, |
| "n_train": 105234, |
| "n_test": 26182 |
| }, |
| "R_role_inv_acc_drop": 0.0, |
| "position_stratified": { |
| "n_tokens": 29302, |
| "n_positions_used": 44, |
| "pi": { |
| "acc": 0.8357, |
| "auc": 0.903, |
| "n_train": 23471, |
| "n_test": 5831 |
| }, |
| "random_orthonormal": { |
| "acc": 0.7652, |
| "auc": 0.8278, |
| "n_train": 23471, |
| "n_test": 5831 |
| }, |
| "raw_full": { |
| "acc": 0.947, |
| "auc": 0.9854, |
| "n_train": 23471, |
| "n_test": 5831 |
| }, |
| "pi_minus_random_acc": 0.0705 |
| } |
| }, |
| "verdict": { |
| "falsified": true, |
| "summary": "FALSIFIED: pi-subspace cleanly separates spans AND beats random -> role reading RESURRECTED at token level (comp-operator R_role)", |
| "key_numbers": { |
| "pi_acc": 0.9284, |
| "pi_auc": 0.9704, |
| "random_acc": 0.846, |
| "random_auc": 0.9007, |
| "near_id_acc": 0.6298, |
| "raw_acc": 0.9786, |
| "pi_minus_random_acc": 0.0824 |
| } |
| }, |
| "examples": [ |
| { |
| "A": "The fate of the ACA repeal bill hangs in the balance, and the impact on women's health is a pressing concern.", |
| "B": "The American people deserve a healthcare system that prioritizes prevention, equality, and access to essential services," |
| }, |
| { |
| "A": "The American people deserve a healthcare system that prioritizes prevention, equality, and access to essential services,", |
| "B": "The outcome of this vote will have far-reaching consequences for the health and well-being of millions of Americans, par" |
| }, |
| { |
| "A": "This news comes as a shock to many Americans, who are already feeling the pinch of rising grocery bills.", |
| "B": "The USDA attributes the price increase to higher costs for corn, soybeans, and wheat, as well as increased demand for me" |
| }, |
| { |
| "A": "According to a recent survey, American beef consumption is shifting towards more grass-fed and leaner options.", |
| "B": "This trend is driven by consumer demand for healthier, more sustainable meat options." |
| } |
| ] |
| } |
