sonar-sae / night5_json /w13_binding_scope.json
nickypro's picture
Upload folder using huggingface_hub
411692f verified
Raw
History Blame Contribute Delete
21.8 kB
{
"experiment": "W13 binding scope across relation types (C3 cross-form dissociation)",
"machinery": {
"rpos": {
"n_rows": 16448,
"V": 16384,
"max_pos": 64,
"init": "sinusoidal"
},
"role": {
"orth_dev": 1e-06,
"source": "tae_comp_W_order.pt S polar-orth"
},
"ci_def": "c_i = R_pos^{-p} h - R_role E(w); cbar = mean_i c_i",
"reuses": "C3 tae_role_binding cross_construction_auc + probe_all_fields"
},
"design": "per relation: FORM1_AB/FORM1_BA/FORM2_AB/FORM2_BA per (A,B); same bag; focal A is REL-HEAD in 2/4 and FIRST in 2/4 (balanced). cross-FORM transfer (train form1 test form2) is the load-bearing thematic-vs-surface test.",
"prereg": {
"prediction": "UNIVERSAL no-binding: all relation types surface-positional, cross-form head AUC near chance/anti-correlated (~.015 like agent/patient). Possible exception: comparison/possession lexical marker ('s/than/-er) may leak a cue.",
"falsified_if": "some relation type bound INVARIANTLY: cross-form head AUC>.7 with z_bag near chance -> z binds SOME relations abstractly; no-binding narrows to agent/patient.",
"agent_patient_anchor_from_C3": {
"within_form_role_auc": [
0.774,
0.792
],
"cross_construction_role_auc": 0.015,
"surface_cross_construction_auc": 0.985
}
},
"per_relation": {
"possession": {
"n": 240,
"n_pairs": 60,
"sanity": {
"z_norm_mean": 0.208,
"cbar_frac_of_z_energy": 1.2573,
"zbag_within_quad_max_dev_mean": 0.0119
},
"probes_within": {
"head_pooled_both_forms": {
"A_zbag": {
"auc_mean": 0.501,
"auc_std": 0.0018,
"n_folds_used": 5
},
"B_full_z": {
"auc_mean": 0.8767,
"auc_std": 0.0727,
"n_folds_used": 5
},
"C_ci_field": {
"auc_mean": 0.8792,
"auc_std": 0.0698,
"n_folds_used": 5
},
"ctrl_tfidf": 0.8938
},
"surface_pooled_both_forms": {
"A_zbag": {
"auc_mean": 0.5021,
"auc_std": 0.0064,
"n_folds_used": 5
},
"B_full_z": {
"auc_mean": 0.6701,
"auc_std": 0.0495,
"n_folds_used": 5
},
"C_ci_field": {
"auc_mean": 0.7944,
"auc_std": 0.0948,
"n_folds_used": 5
},
"ctrl_tfidf": 0.9111
},
"head_form1_only": {
"A_zbag": {
"auc_mean": 0.5181,
"auc_std": 0.0071,
"n_folds_used": 5
},
"B_full_z": {
"auc_mean": 0.8736,
"auc_std": 0.0645,
"n_folds_used": 5
},
"C_ci_field": {
"auc_mean": 0.8819,
"auc_std": 0.0647,
"n_folds_used": 5
},
"ctrl_tfidf": 0.9486
},
"head_form2_only": {
"A_zbag": {
"auc_mean": 0.5014,
"auc_std": 0.0102,
"n_folds_used": 5
},
"B_full_z": {
"auc_mean": 0.8667,
"auc_std": 0.0797,
"n_folds_used": 5
},
"C_ci_field": {
"auc_mean": 0.8972,
"auc_std": 0.0614,
"n_folds_used": 5
},
"ctrl_tfidf": 0.9486
},
"head_z_shuffled_label_null_form1": {
"auc_mean": 0.4724,
"auc_std": 0.1274,
"n_folds_used": 5
}
},
"lexical_marker_check": {
"is_lexically_marked_relation": true,
"zbag_head_auc_form1": 0.5181,
"zbag_head_auc_form2": 0.5014,
"tfidf_head_auc_form1": 0.9486,
"tfidf_head_auc_form2": 0.9486,
"bag_leak_flag": false,
"note": "if z_bag (order-free) reads head above ~.5, a lexical marker leaked into the bag; a cross-form transfer riding that is a cue artifact, not binding."
},
"cross_form": {
"head_label_relational": {
"z": {
"train_form1_test_form2": 1.0,
"train_form2_test_form1": 1.0
},
"z_bag": {
"train_form1_test_form2": 0.4994,
"train_form2_test_form1": 0.5003
},
"c_i": {
"train_form1_test_form2": 1.0,
"train_form2_test_form1": 0.9917
}
},
"surface_label_firstmention": {
"z": {
"train_form1_test_form2": 0.0,
"train_form2_test_form1": 0.0
},
"z_bag": {
"train_form1_test_form2": 0.5,
"train_form2_test_form1": 0.4992
},
"c_i": {
"train_form1_test_form2": 0.0,
"train_form2_test_form1": 0.0083
}
}
},
"verdict": {
"head_auc_z_pooled": 0.8767,
"head_auc_form1_only": 0.8736,
"head_auc_form2_only": 0.8667,
"head_z_shuffled_null": 0.4724,
"zbag_head_auc_negctrl_form1": 0.5181,
"zbag_head_auc_negctrl_form2": 0.5014,
"cross_form_head_auc_z_mean": 1.0,
"cross_form_head_z_detail": {
"train_form1_test_form2": 1.0,
"train_form2_test_form1": 1.0
},
"cross_form_surface_auc_z_mean": 0.0,
"within_form_binding": true,
"thematic_invariant_binding": true,
"thematic_but_lexical_leak": false,
"surface_positional_only": false,
"no_binding_at_all": false,
"summary": "FALSIFIER FIRED: z binds this relation INVARIANTLY (cross-form head AUC>.7, z_bag near chance) -> ABSTRACT relational binding, NOT just surface position."
}
},
"location": {
"n": 240,
"n_pairs": 60,
"sanity": {
"z_norm_mean": 0.218,
"cbar_frac_of_z_energy": 1.1481,
"zbag_within_quad_max_dev_mean": 0.0232
},
"probes_within": {
"head_pooled_both_forms": {
"A_zbag": {
"auc_mean": 0.499,
"auc_std": 0.0037,
"n_folds_used": 5
},
"B_full_z": {
"auc_mean": 0.6184,
"auc_std": 0.0503,
"n_folds_used": 5
},
"C_ci_field": {
"auc_mean": 0.6285,
"auc_std": 0.0563,
"n_folds_used": 5
},
"ctrl_tfidf": 0.7076
},
"surface_pooled_both_forms": {
"A_zbag": {
"auc_mean": 0.5031,
"auc_std": 0.0007,
"n_folds_used": 5
},
"B_full_z": {
"auc_mean": 0.8736,
"auc_std": 0.0618,
"n_folds_used": 5
},
"C_ci_field": {
"auc_mean": 0.8816,
"auc_std": 0.0538,
"n_folds_used": 5
},
"ctrl_tfidf": 0.8347
},
"head_form1_only": {
"A_zbag": {
"auc_mean": 0.5,
"auc_std": 0.0088,
"n_folds_used": 5
},
"B_full_z": {
"auc_mean": 0.8764,
"auc_std": 0.0528,
"n_folds_used": 5
},
"C_ci_field": {
"auc_mean": 0.8819,
"auc_std": 0.0459,
"n_folds_used": 5
},
"ctrl_tfidf": 0.8319
},
"head_form2_only": {
"A_zbag": {
"auc_mean": 0.5069,
"auc_std": 0.0108,
"n_folds_used": 5
},
"B_full_z": {
"auc_mean": 0.8736,
"auc_std": 0.0659,
"n_folds_used": 5
},
"C_ci_field": {
"auc_mean": 0.8903,
"auc_std": 0.0556,
"n_folds_used": 5
},
"ctrl_tfidf": 0.8319
},
"head_z_shuffled_label_null_form1": {
"auc_mean": 0.359,
"auc_std": 0.1125,
"n_folds_used": 5
}
},
"lexical_marker_check": {
"is_lexically_marked_relation": false,
"zbag_head_auc_form1": 0.5,
"zbag_head_auc_form2": 0.5069,
"tfidf_head_auc_form1": 0.8319,
"tfidf_head_auc_form2": 0.8319,
"bag_leak_flag": false,
"note": "if z_bag (order-free) reads head above ~.5, a lexical marker leaked into the bag; a cross-form transfer riding that is a cue artifact, not binding."
},
"cross_form": {
"head_label_relational": {
"z": {
"train_form1_test_form2": 0.0006,
"train_form2_test_form1": 0.0
},
"z_bag": {
"train_form1_test_form2": 0.4997,
"train_form2_test_form1": 0.5
},
"c_i": {
"train_form1_test_form2": 0.0,
"train_form2_test_form1": 0.0003
}
},
"surface_label_firstmention": {
"z": {
"train_form1_test_form2": 0.9994,
"train_form2_test_form1": 1.0
},
"z_bag": {
"train_form1_test_form2": 0.5006,
"train_form2_test_form1": 0.5
},
"c_i": {
"train_form1_test_form2": 1.0,
"train_form2_test_form1": 0.9997
}
}
},
"verdict": {
"head_auc_z_pooled": 0.6184,
"head_auc_form1_only": 0.8764,
"head_auc_form2_only": 0.8736,
"head_z_shuffled_null": 0.359,
"zbag_head_auc_negctrl_form1": 0.5,
"zbag_head_auc_negctrl_form2": 0.5069,
"cross_form_head_auc_z_mean": 0.0003,
"cross_form_head_z_detail": {
"train_form1_test_form2": 0.0006,
"train_form2_test_form1": 0.0
},
"cross_form_surface_auc_z_mean": 0.9997,
"within_form_binding": true,
"thematic_invariant_binding": false,
"thematic_but_lexical_leak": false,
"surface_positional_only": true,
"no_binding_at_all": false,
"summary": "SURFACE-POSITIONAL: within-form head readout tracks FIRST-MENTION; cross-form flips/anti-correlates -> no abstract binding (matches agent/patient)."
}
},
"comparison": {
"n": 240,
"n_pairs": 60,
"sanity": {
"z_norm_mean": 0.197,
"cbar_frac_of_z_energy": 1.336,
"zbag_within_quad_max_dev_mean": 0.0274
},
"probes_within": {
"head_pooled_both_forms": {
"A_zbag": {
"auc_mean": 0.4986,
"auc_std": 0.0028,
"n_folds_used": 5
},
"B_full_z": {
"auc_mean": 0.5542,
"auc_std": 0.048,
"n_folds_used": 5
},
"C_ci_field": {
"auc_mean": 0.5392,
"auc_std": 0.0392,
"n_folds_used": 5
},
"ctrl_tfidf": 0.5177
},
"surface_pooled_both_forms": {
"A_zbag": {
"auc_mean": 0.5,
"auc_std": 0.0022,
"n_folds_used": 5
},
"B_full_z": {
"auc_mean": 0.8375,
"auc_std": 0.2045,
"n_folds_used": 5
},
"C_ci_field": {
"auc_mean": 0.849,
"auc_std": 0.1845,
"n_folds_used": 5
},
"ctrl_tfidf": 0.7972
},
"head_form1_only": {
"A_zbag": {
"auc_mean": 0.5069,
"auc_std": 0.0108,
"n_folds_used": 5
},
"B_full_z": {
"auc_mean": 0.8361,
"auc_std": 0.2053,
"n_folds_used": 5
},
"C_ci_field": {
"auc_mean": 0.8389,
"auc_std": 0.1937,
"n_folds_used": 5
},
"ctrl_tfidf": 0.7778
},
"head_form2_only": {
"A_zbag": {
"auc_mean": 0.4917,
"auc_std": 0.0135,
"n_folds_used": 5
},
"B_full_z": {
"auc_mean": 0.8347,
"auc_std": 0.2081,
"n_folds_used": 5
},
"C_ci_field": {
"auc_mean": 0.8472,
"auc_std": 0.186,
"n_folds_used": 5
},
"ctrl_tfidf": 0.7778
},
"head_z_shuffled_label_null_form1": {
"auc_mean": 0.4832,
"auc_std": 0.0976,
"n_folds_used": 5
}
},
"lexical_marker_check": {
"is_lexically_marked_relation": true,
"zbag_head_auc_form1": 0.5069,
"zbag_head_auc_form2": 0.4917,
"tfidf_head_auc_form1": 0.7778,
"tfidf_head_auc_form2": 0.7778,
"bag_leak_flag": false,
"note": "if z_bag (order-free) reads head above ~.5, a lexical marker leaked into the bag; a cross-form transfer riding that is a cue artifact, not binding."
},
"cross_form": {
"head_label_relational": {
"z": {
"train_form1_test_form2": 0.0,
"train_form2_test_form1": 0.0
},
"z_bag": {
"train_form1_test_form2": 0.5003,
"train_form2_test_form1": 0.5003
},
"c_i": {
"train_form1_test_form2": 0.0,
"train_form2_test_form1": 0.0
}
},
"surface_label_firstmention": {
"z": {
"train_form1_test_form2": 1.0,
"train_form2_test_form1": 1.0
},
"z_bag": {
"train_form1_test_form2": 0.4997,
"train_form2_test_form1": 0.5003
},
"c_i": {
"train_form1_test_form2": 1.0,
"train_form2_test_form1": 1.0
}
}
},
"verdict": {
"head_auc_z_pooled": 0.5542,
"head_auc_form1_only": 0.8361,
"head_auc_form2_only": 0.8347,
"head_z_shuffled_null": 0.4832,
"zbag_head_auc_negctrl_form1": 0.5069,
"zbag_head_auc_negctrl_form2": 0.4917,
"cross_form_head_auc_z_mean": 0.0,
"cross_form_head_z_detail": {
"train_form1_test_form2": 0.0,
"train_form2_test_form1": 0.0
},
"cross_form_surface_auc_z_mean": 1.0,
"within_form_binding": true,
"thematic_invariant_binding": false,
"thematic_but_lexical_leak": false,
"surface_positional_only": true,
"no_binding_at_all": false,
"summary": "SURFACE-POSITIONAL: within-form head readout tracks FIRST-MENTION; cross-form flips/anti-correlates -> no abstract binding (matches agent/patient)."
}
},
"temporal": {
"n": 240,
"n_pairs": 60,
"sanity": {
"z_norm_mean": 0.203,
"cbar_frac_of_z_energy": 1.3022,
"zbag_within_quad_max_dev_mean": 0.0373
},
"probes_within": {
"head_pooled_both_forms": {
"A_zbag": {
"auc_mean": 0.4969,
"auc_std": 0.0028,
"n_folds_used": 5
},
"B_full_z": {
"auc_mean": 0.6691,
"auc_std": 0.069,
"n_folds_used": 5
},
"C_ci_field": {
"auc_mean": 0.6625,
"auc_std": 0.0512,
"n_folds_used": 5
},
"ctrl_tfidf": 0.7639
},
"surface_pooled_both_forms": {
"A_zbag": {
"auc_mean": 0.4941,
"auc_std": 0.0028,
"n_folds_used": 5
},
"B_full_z": {
"auc_mean": 0.9073,
"auc_std": 0.0692,
"n_folds_used": 5
},
"C_ci_field": {
"auc_mean": 0.901,
"auc_std": 0.0705,
"n_folds_used": 5
},
"ctrl_tfidf": 0.9236
},
"head_form1_only": {
"A_zbag": {
"auc_mean": 0.4833,
"auc_std": 0.0113,
"n_folds_used": 5
},
"B_full_z": {
"auc_mean": 0.9139,
"auc_std": 0.066,
"n_folds_used": 5
},
"C_ci_field": {
"auc_mean": 0.9042,
"auc_std": 0.0734,
"n_folds_used": 5
},
"ctrl_tfidf": 0.9278
},
"head_form2_only": {
"A_zbag": {
"auc_mean": 0.4875,
"auc_std": 0.0081,
"n_folds_used": 5
},
"B_full_z": {
"auc_mean": 0.9125,
"auc_std": 0.0673,
"n_folds_used": 5
},
"C_ci_field": {
"auc_mean": 0.9042,
"auc_std": 0.0733,
"n_folds_used": 5
},
"ctrl_tfidf": 0.9278
},
"head_z_shuffled_label_null_form1": {
"auc_mean": 0.5895,
"auc_std": 0.1101,
"n_folds_used": 5
}
},
"lexical_marker_check": {
"is_lexically_marked_relation": false,
"zbag_head_auc_form1": 0.4833,
"zbag_head_auc_form2": 0.4875,
"tfidf_head_auc_form1": 0.9278,
"tfidf_head_auc_form2": 0.9278,
"bag_leak_flag": false,
"note": "if z_bag (order-free) reads head above ~.5, a lexical marker leaked into the bag; a cross-form transfer riding that is a cue artifact, not binding."
},
"cross_form": {
"head_label_relational": {
"z": {
"train_form1_test_form2": 0.0,
"train_form2_test_form1": 0.0
},
"z_bag": {
"train_form1_test_form2": 0.5003,
"train_form2_test_form1": 0.5031
},
"c_i": {
"train_form1_test_form2": 0.0,
"train_form2_test_form1": 0.0
}
},
"surface_label_firstmention": {
"z": {
"train_form1_test_form2": 1.0,
"train_form2_test_form1": 1.0
},
"z_bag": {
"train_form1_test_form2": 0.4997,
"train_form2_test_form1": 0.4975
},
"c_i": {
"train_form1_test_form2": 1.0,
"train_form2_test_form1": 1.0
}
}
},
"verdict": {
"head_auc_z_pooled": 0.6691,
"head_auc_form1_only": 0.9139,
"head_auc_form2_only": 0.9125,
"head_z_shuffled_null": 0.5895,
"zbag_head_auc_negctrl_form1": 0.4833,
"zbag_head_auc_negctrl_form2": 0.4875,
"cross_form_head_auc_z_mean": 0.0,
"cross_form_head_z_detail": {
"train_form1_test_form2": 0.0,
"train_form2_test_form1": 0.0
},
"cross_form_surface_auc_z_mean": 1.0,
"within_form_binding": true,
"thematic_invariant_binding": false,
"thematic_but_lexical_leak": false,
"surface_positional_only": true,
"no_binding_at_all": false,
"summary": "SURFACE-POSITIONAL: within-form head readout tracks FIRST-MENTION; cross-form flips/anti-correlates -> no abstract binding (matches agent/patient)."
}
}
},
"cross_relation_summary": {
"possession": {
"within_form_head_auc": [
0.8736,
0.8667
],
"cross_form_head_auc_mean": 1.0,
"cross_form_surface_auc_mean": 0.0,
"zbag_head_negctrl": [
0.5181,
0.5014
],
"thematic_invariant": true,
"surface_positional": false,
"verdict": "FALSIFIER FIRED: z binds this relation INVARIANTLY (cross-form head AUC>.7, z_bag near chance) -> ABSTRACT relational binding, NOT just surface position."
},
"location": {
"within_form_head_auc": [
0.8764,
0.8736
],
"cross_form_head_auc_mean": 0.0003,
"cross_form_surface_auc_mean": 0.9997,
"zbag_head_negctrl": [
0.5,
0.5069
],
"thematic_invariant": false,
"surface_positional": true,
"verdict": "SURFACE-POSITIONAL: within-form head readout tracks FIRST-MENTION; cross-form flips/anti-correlates -> no abstract binding (matches agent/patient)."
},
"comparison": {
"within_form_head_auc": [
0.8361,
0.8347
],
"cross_form_head_auc_mean": 0.0,
"cross_form_surface_auc_mean": 1.0,
"zbag_head_negctrl": [
0.5069,
0.4917
],
"thematic_invariant": false,
"surface_positional": true,
"verdict": "SURFACE-POSITIONAL: within-form head readout tracks FIRST-MENTION; cross-form flips/anti-correlates -> no abstract binding (matches agent/patient)."
},
"temporal": {
"within_form_head_auc": [
0.9139,
0.9125
],
"cross_form_head_auc_mean": 0.0,
"cross_form_surface_auc_mean": 1.0,
"zbag_head_negctrl": [
0.4833,
0.4875
],
"thematic_invariant": false,
"surface_positional": true,
"verdict": "SURFACE-POSITIONAL: within-form head readout tracks FIRST-MENTION; cross-form flips/anti-correlates -> no abstract binding (matches agent/patient)."
}
},
"falsifier_status": "FIRED",
"headline": "FALSIFIER FIRED for ['possession']: z binds SOME relation type(s) INVARIANTLY (cross-form head AUC>.7, z_bag near chance). No-binding is NOT universal; it narrows toward agent/patient + the surface-positional relations.",
"skeptic_followups": {
"possession_3form": {
"cross_form_head_z": {
"form1->form2": 1.0,
"form1->form3": 0.9817,
"form2->form1": 1.0,
"form2->form3": 0.9733,
"form3->form1": 0.9792,
"form3->form2": 0.9878
},
"zbag_head_per_form": {
"form1": 0.5181,
"form2": 0.5014,
"form3": 0.5042
},
"note": "owner transfers across all 3 encodings incl verbal 'owned by'?"
},
"agentpatient_anchor": {
"cross_construction_role_z_a2p": 0.0156,
"cross_construction_role_z_p2a": 0.0139
}
}
}