kip-hash / eval /report.json
IO-AIR's picture
kip-hash: THE final release (single version)
72eaba2 verified
Raw History Blame Contribute Delete
5.95 kB
{
"verify": {
"n_eval": 946,
"spearman_fine": 0.1643,
"spearman_zonly": 0.0043,
"mae_fine": 12.03,
"mae_score4_decoded": 12.11,
"auc_junk_fine": 0.6631,
"n_junk_eval": 154
},
"dedup": {
"n_pos": 157,
"n_neg": 9367,
"radius_spec_sweep": {
"sweep": [
{
"spec": "r111111",
"radii": [
1,
1,
1,
1,
1,
1
],
"probe_keys_per_row": 729,
"recall": 0.8344,
"false_merges": 26,
"false_merge_rate": 0.00278,
"F1": 0.8344
},
{
"spec": "r211111",
"radii": [
2,
1,
1,
1,
1,
1
],
"probe_keys_per_row": 1215,
"recall": 0.8471,
"false_merges": 36,
"false_merge_rate": 0.00384,
"F1": 0.816
},
{
"spec": "r221111",
"radii": [
2,
2,
1,
1,
1,
1
],
"probe_keys_per_row": 2025,
"recall": 0.8599,
"false_merges": 44,
"false_merge_rate": 0.0047,
"F1": 0.8036
},
{
"spec": "r222111",
"radii": [
2,
2,
2,
1,
1,
1
],
"probe_keys_per_row": 3375,
"recall": 0.8599,
"false_merges": 59,
"false_merge_rate": 0.0063,
"F1": 0.7692
},
{
"spec": "r222211",
"radii": [
2,
2,
2,
2,
1,
1
],
"probe_keys_per_row": 5625,
"recall": 0.8726,
"false_merges": 72,
"false_merge_rate": 0.00769,
"F1": 0.7486
}
],
"shipped": "r111111"
},
"tau_sweep": [
{
"s": 0.4,
"recall": 0.7389,
"false_merges": 6,
"false_merge_rate": 0.00064,
"F1": 0.8315
},
{
"s": 0.6,
"recall": 0.8153,
"false_merges": 14,
"false_merge_rate": 0.00149,
"F1": 0.8562
},
{
"s": 0.8,
"recall": 0.8344,
"false_merges": 22,
"false_merge_rate": 0.00235,
"F1": 0.8452
},
{
"s": 1.0,
"recall": 0.8344,
"false_merges": 26,
"false_merge_rate": 0.00278,
"F1": 0.8344
},
{
"s": 1.3,
"recall": 0.8344,
"false_merges": 32,
"false_merge_rate": 0.00342,
"F1": 0.8188
},
{
"s": 1.6,
"recall": 0.8344,
"false_merges": 32,
"false_merge_rate": 0.00342,
"F1": 0.8188
}
],
"tau_operating_point": {
"scale": 0.6,
"tau": [
0.3357696175575256,
0.29780099987983705,
0.2868009567260742,
0.1945085048675537,
0.2959071099758148,
0.40350615978240967
]
},
"dup_exact_recall": 0.2739,
"dup_fuzzy_recall": 0.8344,
"dup_confirmed_recall": 0.8153,
"dup_score4_same_level": 0.6561,
"neg_confirmed_false_merge": 0.00149,
"neg_candidate_rate": 0.00363,
"false_merge_zone": {
"n_pairs": 14,
"n_dup": 1,
"n_not_dup": 13,
"v3_merges_not_dup": 0,
"v3_catches_dup": 1,
"mh_sim_range": [
0.148,
0.461
]
},
"occupancy": {
"n_docs": 946,
"n_buckets_used": 911,
"max_load": 3,
"entropy_bits": 9.811
}
},
"compact_width": {
"bin_bits": [
3,
3,
3,
3,
3,
3
],
"word_bits": 22,
"dup_exact_recall": 0.4968,
"dup_fuzzy_recall": 0.9172,
"dup_confirmed_recall": 0.879,
"neg_candidate_rate": 0.04185,
"neg_confirmed_false_merge": 0.00395,
"occupancy": {
"n_docs": 946,
"n_buckets_used": 859,
"max_load": 5,
"entropy_bits": 9.694
}
},
"guard_eval": {
"n_confirmed_pos": 128,
"n_confirmed_neg": 14,
"guard_threshold": 0.25,
"gray_zone": 0.5,
"pos_jac_median": 0.784,
"pos_retained_at_guard": 1.0,
"pos_gray_zone": 0.0156,
"neg_jac_median": 0.0,
"neg_rejected_at_guard": 1.0,
"neg_gray_zone": 0.0,
"fm_after_guard": 0.0
},
"gate_eval": {
"drop_rates": {
"drop_score4_lt_1": 0.0973,
"drop_score4_lt_2": 0.1734,
"drop_score4_lt_3": 0.2389,
"drop_score4_lt_4": 0.2939,
"drop_score4_lt_5": 0.3626
},
"per_threshold": {
"lt_1": {
"n_dropped": 92,
"junk_precision": 0.2935,
"junk_recall": 0.1753
},
"lt_2": {
"n_dropped": 164,
"junk_precision": 0.3049,
"junk_recall": 0.3247
},
"lt_3": {
"n_dropped": 226,
"junk_precision": 0.2788,
"junk_recall": 0.4091
},
"lt_4": {
"n_dropped": 278,
"junk_precision": 0.259,
"junk_recall": 0.4675
},
"lt_5": {
"n_dropped": 343,
"junk_precision": 0.2449,
"junk_recall": 0.5455
}
},
"n_junk_eval": 154
},
"gate_sweep": {
"drop_score4_lt_1": 0.0973,
"drop_score4_lt_2": 0.1734,
"drop_score4_lt_3": 0.2389,
"drop_score4_lt_4": 0.2939,
"drop_score4_lt_5": 0.3626
},
"baselines": {
"random_simhash_20": {
"rand_exact_recall": 0.1274,
"rand_confirmed_recall": 0.3885,
"rand_neg_confirmed_false_merge": 0.0
},
"minhash64": {
"minhash_best_F1": 0.9502,
"minhash_thr": 0.35,
"minhash_neg_false_merge_at_thr": 0.00011,
"s": 5.4
}
},
"v2_reference": {
"spearman_hash_only": 0.424,
"spearman_proj": 0.543,
"junk_auc_bits": 0.755,
"easy_band_F1": 0.983,
"false_merge_zone_merges_of_13": 0,
"artifact_mb": 5.9,
"note": "lineage: the retired 128-bit v2 signature was 16 B/row; the kip-hash word is 3.5 B/row with the score packed in"
},
"throughput": {
"engine": "pure stdlib python, one core",
"n_docs": 40,
"sign_s": 0.18,
"docs_per_sec": 220.1
},
"artifact_kb_b64": 448.0,
"config": {
"D": 32768,
"P": 10,
"bin_bits": [
4,
4,
4,
4,
4,
4
],
"word_bits": 28,
"score_bits": 4,
"tau": [
0.5596160292625427,
0.4963349997997284,
0.47800159454345703,
0.32418084144592285,
0.49317851662635803,
0.6725102663040161
],
"fuzzy_radius": 1,
"fuzzy_radii_shipped": [
1,
1,
1,
1,
1,
1
]
},
"data": {
"labeled_docs": 4839,
"eval_docs": 946,
"train_pairs": 48106,
"eval_pairs": 6524,
"synth_train_pairs": 1300,
"synth_eval_pairs": 180,
"labels_by_run": {
"v2": 794,
"v3": 0,
"v4": 1913,
"v5": 2132
}
}
}