sonar-sae / night5_json /w16_numeric_precision.json
nickypro's picture
Upload folder using huggingface_hub
7c2241d verified
Raw
History Blame Contribute Delete
22.4 kB
{
"experiment": "W16_numeric_precision",
"prereg_prediction": "z encodes numbers approx/log-scale: small numbers preserved, large/multi-digit blurred (magnitude kept, exact digits lost); sign of change readable, exact % often wrong.",
"falsified_if": "exact numbers incl large perfectly readable/preserved (precise) OR numbers totally lost (even magnitude unreadable).",
"device": "cuda",
"arm1_exact_readout": {
"carrier": "There were {} protesters at the rally.",
"n_numbers": 494,
"r2_log10N": 0.0570623690100055,
"r2_linearN": 0.0341025851487583,
"exact_int_recovery_overall": 0.0,
"within_1.25x_overall": 0.06040268456375839,
"within_2x_overall": 0.18120805369127516,
"by_magnitude": {
"single(1-9)": {
"n": 3,
"exact_int_recovery": 0.0,
"within_1.25x": 0.0,
"within_2x": 0.0,
"median_ratio_err": 231.8241633728285
},
"tens(10-99)": {
"n": 24,
"exact_int_recovery": 0.0,
"within_1.25x": 0.0,
"within_2x": 0.0,
"median_ratio_err": 16.19264868234125
},
"hundreds(100-999)": {
"n": 45,
"exact_int_recovery": 0.0,
"within_1.25x": 0.13333333333333333,
"within_2x": 0.2222222222222222,
"median_ratio_err": 2.856040004601778
},
"thousands(1000+)": {
"n": 77,
"exact_int_recovery": 0.0,
"within_1.25x": 0.03896103896103896,
"within_2x": 0.22077922077922077,
"median_ratio_err": 6.220979609521569
}
},
"note": "r2 on log10 >> r2 on linear-N would indicate log-scale encoding; exact_int_recovery is the precise-readout metric."
},
"arm2_minimal_pairs": {
"carrier": "I bought {} apples at the store.",
"numbers": [
1,
2,
3,
5,
7,
9,
30,
50,
300,
500,
3000,
5000
],
"corr_(1-cos)_vs_|logratio|": 0.6304034407930417,
"selected_cosines": {
"cos(3,5)": 0.9259563088417053,
"cos(3,30)": 0.9062734842300415,
"cos(3,300)": 0.9031403064727783,
"cos(5,50)": 0.9096679091453552,
"cos(30,50)": 0.9147080779075623,
"cos(300,500)": 0.9294202923774719
},
"decode": [
{
"true": 1,
"decoded_text": "I bought 1 apples at the store.",
"decoded_number": 1,
"exact": true,
"mag_match": true,
"has_number": true
},
{
"true": 2,
"decoded_text": "I bought 2 apples in the store.",
"decoded_number": 2,
"exact": true,
"mag_match": true,
"has_number": true
},
{
"true": 3,
"decoded_text": "I bought 3 apples in the store.",
"decoded_number": 3,
"exact": true,
"mag_match": true,
"has_number": true
},
{
"true": 5,
"decoded_text": "I bought 5 apples in the store.",
"decoded_number": 5,
"exact": true,
"mag_match": true,
"has_number": true
},
{
"true": 7,
"decoded_text": "I bought 7 apples in the store.",
"decoded_number": 7,
"exact": true,
"mag_match": true,
"has_number": true
},
{
"true": 9,
"decoded_text": "I bought 9 apples in the store.",
"decoded_number": 9,
"exact": true,
"mag_match": true,
"has_number": true
},
{
"true": 30,
"decoded_text": "I bought 30 apples in the store.",
"decoded_number": 30,
"exact": true,
"mag_match": true,
"has_number": true
},
{
"true": 50,
"decoded_text": "I bought 50 apples in the store.",
"decoded_number": 50,
"exact": true,
"mag_match": true,
"has_number": true
},
{
"true": 300,
"decoded_text": "I bought 300 apples in the store.",
"decoded_number": 300,
"exact": true,
"mag_match": true,
"has_number": true
},
{
"true": 500,
"decoded_text": "I bought 500 apples in the store.",
"decoded_number": 500,
"exact": true,
"mag_match": true,
"has_number": true
},
{
"true": 3000,
"decoded_text": "I bought 3,000 apples in the store.",
"decoded_number": 3000,
"exact": true,
"mag_match": true,
"has_number": true
},
{
"true": 5000,
"decoded_text": "I bought 5000 apples in the store.",
"decoded_number": 5000,
"exact": true,
"mag_match": true,
"has_number": true
}
],
"decode_exact_rate": 1.0,
"decode_mag_match_rate": 1.0
},
"arm3_comparison": {
"carrier": "Sales {} by {} percent this quarter.",
"percents": [
1,
2,
3,
5,
8,
10,
15,
20,
30,
50,
75,
90
],
"sign_probe_AUC": 1.0,
"magnitude_r2_log10pct": -1.6492185962721324,
"magnitude_exact_pct": 0.0,
"magnitude_within_1.25x": 0.0,
"decode": [
{
"true_sign": 1,
"true_pct": 1,
"decoded": "Sales increased by 1 percent this quarter.",
"decoded_pct": 1,
"sign_ok": true,
"pct_exact": true
},
{
"true_sign": 1,
"true_pct": 2,
"decoded": "Sales increased by 2 percent this quarter.",
"decoded_pct": 2,
"sign_ok": true,
"pct_exact": true
},
{
"true_sign": 1,
"true_pct": 3,
"decoded": "Sales increased by 3 percent this quarter.",
"decoded_pct": 3,
"sign_ok": true,
"pct_exact": true
},
{
"true_sign": 1,
"true_pct": 5,
"decoded": "Sales increased by 5 percent this quarter.",
"decoded_pct": 5,
"sign_ok": true,
"pct_exact": true
},
{
"true_sign": 1,
"true_pct": 8,
"decoded": "Sales increased by 8 percent this quarter.",
"decoded_pct": 8,
"sign_ok": true,
"pct_exact": true
},
{
"true_sign": 1,
"true_pct": 10,
"decoded": "Sales increased by 10 percent this quarter.",
"decoded_pct": 10,
"sign_ok": true,
"pct_exact": true
},
{
"true_sign": 1,
"true_pct": 15,
"decoded": "Sales increased by 15 percent this quarter.",
"decoded_pct": 15,
"sign_ok": true,
"pct_exact": true
},
{
"true_sign": 1,
"true_pct": 20,
"decoded": "Sales increased by 20 percent this quarter.",
"decoded_pct": 20,
"sign_ok": true,
"pct_exact": true
},
{
"true_sign": 1,
"true_pct": 30,
"decoded": "Sales increased by 30 percent this quarter.",
"decoded_pct": 30,
"sign_ok": true,
"pct_exact": true
},
{
"true_sign": 1,
"true_pct": 50,
"decoded": "Sales increased by 50 percent this quarter.",
"decoded_pct": 50,
"sign_ok": true,
"pct_exact": true
},
{
"true_sign": 1,
"true_pct": 75,
"decoded": "Sales increased by 75 percent this quarter.",
"decoded_pct": 75,
"sign_ok": true,
"pct_exact": true
},
{
"true_sign": 1,
"true_pct": 90,
"decoded": "Sales increased by 90 percent this quarter.",
"decoded_pct": 90,
"sign_ok": true,
"pct_exact": true
},
{
"true_sign": -1,
"true_pct": 1,
"decoded": "Sales declined by 1 percent this quarter.",
"decoded_pct": 1,
"sign_ok": false,
"pct_exact": true
},
{
"true_sign": -1,
"true_pct": 2,
"decoded": "Sales declined by 2 percent this quarter.",
"decoded_pct": 2,
"sign_ok": false,
"pct_exact": true
},
{
"true_sign": -1,
"true_pct": 3,
"decoded": "Sales declined by 3 percent this quarter.",
"decoded_pct": 3,
"sign_ok": false,
"pct_exact": true
},
{
"true_sign": -1,
"true_pct": 5,
"decoded": "Sales declined by 5 percent this quarter.",
"decoded_pct": 5,
"sign_ok": false,
"pct_exact": true
},
{
"true_sign": -1,
"true_pct": 8,
"decoded": "Sales declined by 8 percent this quarter.",
"decoded_pct": 8,
"sign_ok": false,
"pct_exact": true
},
{
"true_sign": -1,
"true_pct": 10,
"decoded": "Sales declined by 10 percent this quarter.",
"decoded_pct": 10,
"sign_ok": false,
"pct_exact": true
},
{
"true_sign": -1,
"true_pct": 15,
"decoded": "Sales declined by 15 percent this quarter.",
"decoded_pct": 15,
"sign_ok": false,
"pct_exact": true
},
{
"true_sign": -1,
"true_pct": 20,
"decoded": "Sales declined by 20 percent this quarter.",
"decoded_pct": 20,
"sign_ok": false,
"pct_exact": true
},
{
"true_sign": -1,
"true_pct": 30,
"decoded": "Sales declined by 30 percent this quarter.",
"decoded_pct": 30,
"sign_ok": false,
"pct_exact": true
},
{
"true_sign": -1,
"true_pct": 50,
"decoded": "Sales declined by 50 percent this quarter.",
"decoded_pct": 50,
"sign_ok": false,
"pct_exact": true
},
{
"true_sign": -1,
"true_pct": 75,
"decoded": "Sales declined by 75 percent this quarter.",
"decoded_pct": 75,
"sign_ok": false,
"pct_exact": true
},
{
"true_sign": -1,
"true_pct": 90,
"decoded": "Sales declined by 90 percent this quarter.",
"decoded_pct": 90,
"sign_ok": false,
"pct_exact": true
}
],
"decode_sign_ok_rate": 0.5,
"decode_pct_exact_rate": 1.0
},
"arm4_decode_roundtrip": {
"carrier": "The report counted {} {}.",
"numbers": [
3,
5,
7,
9,
12,
27,
45,
88,
150,
340,
760,
1200,
4500,
16000,
76000,
250000
],
"rows": [
{
"true": 3,
"noun": "votes",
"decoded": "The report counted 3 votes.",
"decoded_number": 3,
"exact": true,
"mag_match": true,
"om_offset": 0.0,
"has_number": true,
"bucket": "single(1-9)"
},
{
"true": 5,
"noun": "cases",
"decoded": "The report counted 5 cases.",
"decoded_number": 5,
"exact": true,
"mag_match": true,
"om_offset": 0.0,
"has_number": true,
"bucket": "single(1-9)"
},
{
"true": 7,
"noun": "houses",
"decoded": "The report counted 7 houses.",
"decoded_number": 7,
"exact": true,
"mag_match": true,
"om_offset": 0.0,
"has_number": true,
"bucket": "single(1-9)"
},
{
"true": 9,
"noun": "votes",
"decoded": "The report counted 9 votes.",
"decoded_number": 9,
"exact": true,
"mag_match": true,
"om_offset": 0.0,
"has_number": true,
"bucket": "single(1-9)"
},
{
"true": 12,
"noun": "casualties",
"decoded": "The report counted 12 casualties.",
"decoded_number": 12,
"exact": true,
"mag_match": true,
"om_offset": 0.0,
"has_number": true,
"bucket": "tens(10-99)"
},
{
"true": 27,
"noun": "votes",
"decoded": "The report counted 27 votes.",
"decoded_number": 27,
"exact": true,
"mag_match": true,
"om_offset": 0.0,
"has_number": true,
"bucket": "tens(10-99)"
},
{
"true": 45,
"noun": "votes",
"decoded": "The report counted 45 votes.",
"decoded_number": 45,
"exact": true,
"mag_match": true,
"om_offset": 0.0,
"has_number": true,
"bucket": "tens(10-99)"
},
{
"true": 88,
"noun": "dollars",
"decoded": "The report counted 88 dollars.",
"decoded_number": 88,
"exact": true,
"mag_match": true,
"om_offset": 0.0,
"has_number": true,
"bucket": "tens(10-99)"
},
{
"true": 150,
"noun": "casualties",
"decoded": "The report counted 150 casualties.",
"decoded_number": 150,
"exact": true,
"mag_match": true,
"om_offset": 0.0,
"has_number": true,
"bucket": "hundreds(100-999)"
},
{
"true": 340,
"noun": "dollars",
"decoded": "The report counted 340 dollars.",
"decoded_number": 340,
"exact": true,
"mag_match": true,
"om_offset": 0.0,
"has_number": true,
"bucket": "hundreds(100-999)"
},
{
"true": 760,
"noun": "dollars",
"decoded": "The report counted 760 dollars.",
"decoded_number": 760,
"exact": true,
"mag_match": true,
"om_offset": 0.0,
"has_number": true,
"bucket": "hundreds(100-999)"
},
{
"true": 1200,
"noun": "casualties",
"decoded": "The report counted 1,200 casualties.",
"decoded_number": 1200,
"exact": true,
"mag_match": true,
"om_offset": 0.0,
"has_number": true,
"bucket": "thousands(1000+)"
},
{
"true": 4500,
"noun": "casualties",
"decoded": "The report counted 4,500 casualties.",
"decoded_number": 4500,
"exact": true,
"mag_match": true,
"om_offset": 0.0,
"has_number": true,
"bucket": "thousands(1000+)"
},
{
"true": 16000,
"noun": "dollars",
"decoded": "The report counted 16,000 dollars.",
"decoded_number": 16000,
"exact": true,
"mag_match": true,
"om_offset": 0.0,
"has_number": true,
"bucket": "thousands(1000+)"
},
{
"true": 76000,
"noun": "dollars",
"decoded": "The report counted 76,000 dollars.",
"decoded_number": 76000,
"exact": true,
"mag_match": true,
"om_offset": 0.0,
"has_number": true,
"bucket": "thousands(1000+)"
},
{
"true": 250000,
"noun": "dollars",
"decoded": "The report counted 250,000 dollars.",
"decoded_number": 250000,
"exact": true,
"mag_match": true,
"om_offset": 0.0,
"has_number": true,
"bucket": "thousands(1000+)"
}
],
"exact_rate_overall": 1.0,
"mag_match_rate_overall": 1.0,
"has_number_rate": 1.0,
"exact_rate_small(<10)": 1.0,
"exact_rate_large(>=1000)": 1.0,
"mag_match_rate_small(<10)": 1.0,
"mag_match_rate_large(>=1000)": 1.0,
"by_bucket": {
"single(1-9)": {
"n": 4,
"exact": 1.0,
"mag_match": 1.0
},
"tens(10-99)": {
"n": 4,
"exact": 1.0,
"mag_match": 1.0
},
"hundreds(100-999)": {
"n": 3,
"exact": 1.0,
"mag_match": 1.0
},
"thousands(1000+)": {
"n": 5,
"exact": 1.0,
"mag_match": 1.0
}
}
},
"scoring": {
"magnitude_readable_r2log>0.4": false,
"log_scale_r2_advantage(log-lin)": 0.023,
"exact_readout_small": 0.0,
"exact_readout_large": 0.0,
"decode_exact_overall": 1.0,
"decode_mag_overall": 1.0,
"decode_emits_a_number_rate": 1.0,
"sign_readable_AUC": 1.0,
"exact_precise_falsifier_fired": false,
"totally_lost_falsifier_fired": false,
"verdict": "CONFIRMED: log-scale/approximate numeric encoding (magnitude readable, exact digits degrade with size).",
"SUPERSEDED": "This block used UNSTANDARDIZED ridge features (r2~0.06) and is a PROBE ARTIFACT. See arm1b_probe_power + scoring_revised: standardized linear probe gives r2(log10N)=0.995. Use scoring_revised."
},
"arm1b_probe_power": {
"alpha_best": 10,
"r2_linear_log10N_standardized": 0.9954015839287945,
"r2_mlp_log10N": 0.9435868115426629,
"nonlinear_gain": -0.0518,
"tens_1NN_exact_value_acc": 0.0,
"tens_1NN_within1_acc": 0.42857142857142855,
"interpretation": "r2_mlp>>r2_linear => \u00a716.4 curved/non-linear numeric store; high 1NN_exact => exact magnitude geometrically present"
},
"arm5_realistic_stress": {
"noise_roundtrip": [
{
"noise_frac": 0.0,
"exact_rate": 1.0,
"decoded_numbers": [
3,
7,
45,
340,
4500,
76000,
250000
]
},
{
"noise_frac": 0.05,
"exact_rate": 1.0,
"decoded_numbers": [
3,
7,
45,
340,
4500,
76000,
250000
]
},
{
"noise_frac": 0.1,
"exact_rate": 1.0,
"decoded_numbers": [
3,
7,
45,
340,
4500,
76000,
250000
]
},
{
"noise_frac": 0.2,
"exact_rate": 1.0,
"decoded_numbers": [
3,
7,
45,
340,
4500,
76000,
250000
]
}
],
"multi_number": [
{
"orig": "In 2019, 47 of the 312 members voted yes.",
"true_numbers": [
2019,
47,
312
],
"decoded": "In 2019, 47 of the 312 members voted yes.",
"decoded_numbers": [
2019,
47,
312
],
"all_exact_present": true
},
{
"orig": "The fire killed 3 people and destroyed 128 homes.",
"true_numbers": [
3,
128
],
"decoded": "The fire killed 3 people and destroyed 128 homes.",
"decoded_numbers": [
3,
128
],
"all_exact_present": true
},
{
"orig": "Prices rose 5 percent while wages fell 12 percent.",
"true_numbers": [
5,
12
],
"decoded": "Prices rose 5 percent while wages fell 12 percent.",
"decoded_numbers": [
5,
12
],
"all_exact_present": true
},
{
"orig": "There were 1,204 cases and 37 deaths reported.",
"true_numbers": [
1204,
37
],
"decoded": "There were 1,204 cases and 37 deaths reported.",
"decoded_numbers": [
1204,
37
],
"all_exact_present": true
},
{
"orig": "The army of 84,000 faced 1,200,000 civilians.",
"true_numbers": [
84000,
1200000
],
"decoded": "The army of 84,000 faced 1,200,000 civilians.",
"decoded_numbers": [
84000,
1200000
],
"all_exact_present": true
}
],
"multi_all_exact_rate": 1.0
},
"scoring_revised": {
"decode_exact_overall": 1.0,
"decode_exact_large": 1.0,
"multi_number_all_exact_rate": 1.0,
"noise0_exact": 1.0,
"noise0.1_exact": 1.0,
"linear_probe_r2_log10N": 0.9954015839287945,
"mlp_probe_r2_log10N": 0.9435868115426629,
"nonlinear_gain": -0.0518,
"sign_readable_AUC": 1.0,
"exact_precise_falsifier_fired": true,
"totally_lost_falsifier_fired": false,
"verdict": "FALSIFIED-precise: EXACT numbers (incl large multi-digit & multi-number) are preserved through encode->decode; z encodes precise numerics for clean single-fact text. Linear probe fails (r2~0) => number is NON-LINEARLY encoded but decoder-readable (curved store, \u00a716.4)."
},
"final_verdict": {
"status": "FALSIFIER FIRED (exact-precise arm)",
"headline": "z encodes numbers PRECISELY for clean single-fact text, NOT log-scale-blurred. Magnitude is near-perfectly LINEARLY readable (standardized ridge r2(log10N)=0.995, MLP adds nothing). Decoder reproduces EXACT digits incl 250,000 and multi-number sentences (2019/47/312, 84,000/1,200,000), robust to 20% per-dim Gaussian noise on z. Sign of %-change AUC=1.0; exact %% preserved.",
"prereg_outcome": "PREREG PREDICTION FALSIFIED: predicted large/multi-digit numbers would blur to magnitude-only; instead they are exact. Small-vs-large stratification shows NO degradation (exact_rate 1.0 in every bucket single->thousands).",
"nuance_geometry": "z places integers on a smooth log-continuum: 1NN in z-space is the SAME exact value 0% of the time but an ADJACENT value (+/-1) 43% of the time. So number lives on a continuous magnitude axis, not discrete exact-value clusters -- yet the decoder still emits exact digits (the precision is in the fine position along the axis, decoder-read losslessly for clean carriers).",
"reconciliation_with_16.4": "16.4 said number is curved/audit-only/un-steerable and erasure regenerates a wrong number. That was about EDITING/ERASURE under likelihood-death and MEAN-SHIFT steering, not clean round-trip READING. W16 shows READING/PRESERVATION is excellent; the 16.4 limit is specifically that you cannot cleanly EDIT/NULL the number with a linear mean-shift (decode-shift .08) even though it is precisely present. Both true: precise to READ, hard to STEER.",
"safety_precision_limit": "For READING/auditing: TRUST z for exact quantities in clean single/multi-fact sentences -- exact numbers, percentages, and sign-of-change all survive encode->decode and are linearly probe-readable. The precision limit is NOT in reading; it is (a) in EDITING (cannot reliably null/flip a quantity via linear z-edit, per 16.4) and (b) untested for long/cluttered paragraphs where overall decoder fidelity drops. Do NOT generalize numbers are precise to mean you can safely scrub a number from z."
}
}