v3_scale: verdict v4.1 m8_v3 vs old_ref

v3_scale/verdicts/m8_v3.json

@@ -0,0 +1,457 @@
+{
+ "tag": "m8_v3",
+ "base": "old_ref",
+ "windows": [
+  "fc2",
+  "fresh_en",
+  "fresh_fr",
+  "test20k",
+  "queries",
+  "test_v3"
+ ],
+ "separate": [
+  "neither_code"
+ ],
+ "rule": "aucune perte significative + ≥1 gain significatif ; significatif = McNemar exact patch v4.1 seul (p<0,05), sens = cand_only − ref_only ; bootstrap indicatif",
+ "decision_criterion": "mcnemar_patch_v41_only",
+ "bootstrap": {
+  "block": 512,
+  "n": 2000,
+  "seed": 777
+ },
+ "vs_base": {
+  "fc2": {
+   "common_spans": 565,
+   "patch_p": 5.774199962615967e-08,
+   "boot_delta_pct": 3.0943627450980395,
+   "boot_ci95_pct": [
+    1.1105047487745099,
+    5.108857996323528
+   ],
+   "byte_range": [
+    4,
+    26320
+   ],
+   "ref_only": 29,
+   "cand_only": 1,
+   "base_only": 29,
+   "discordants": 30,
+   "sens_discordants": "ref",
+   "sig_gain": false,
+   "sig_loss": true,
+   "patch_testable": true,
+   "boot_indicatif": "+",
+   "boot_gain_indicatif": true,
+   "boot_loss_indicatif": false,
+   "boot_conflit_mcnemar": true,
+   "base_zero_hit": false,
+   "cand_hits_inv": 897,
+   "cand_hits_inv_on_common": 4,
+   "ref_hits_inv_on_common": 32,
+   "invocab_mask": true,
+   "acc_base_inv": 11.504241671839436,
+   "acc_cand_inv": 14.451425809569841
+  },
+  "fresh_en": {
+   "common_spans": 1119,
+   "patch_p": 0.10812902140605729,
+   "boot_delta_pct": 1.904296875,
+   "boot_ci95_pct": [
+    1.2786102294921875,
+    2.517776489257812
+   ],
+   "byte_range": [
+    4,
+    32915
+   ],
+   "ref_only": 14,
+   "cand_only": 25,
+   "base_only": 14,
+   "discordants": 39,
+   "sens_discordants": "cand",
+   "sig_gain": false,
+   "sig_loss": false,
+   "patch_testable": true,
+   "boot_indicatif": "+",
+   "boot_gain_indicatif": true,
+   "boot_loss_indicatif": false,
+   "boot_conflit_mcnemar": false,
+   "base_zero_hit": false,
+   "cand_hits_inv": 361,
+   "cand_hits_inv_on_common": 33,
+   "ref_hits_inv_on_common": 22,
+   "invocab_mask": true,
+   "acc_base_inv": 3.3951697584879246,
+   "acc_cand_inv": 5.546166845905669
+  },
+  "fresh_fr": {
+   "common_spans": 1676,
+   "patch_p": 1.0,
+   "boot_delta_pct": 1.515625,
+   "boot_ci95_pct": [
+    0.125,
+    3.7227539062499995
+   ],
+   "byte_range": [
+    5,
+    25684
+   ],
+   "ref_only": 0,
+   "cand_only": 0,
+   "base_only": 0,
+   "discordants": 0,
+   "sens_discordants": "=",
+   "sig_gain": false,
+   "sig_loss": false,
+   "patch_testable": false,
+   "boot_indicatif": "+",
+   "boot_gain_indicatif": true,
+   "boot_loss_indicatif": false,
+   "boot_conflit_mcnemar": false,
+   "base_zero_hit": false,
+   "cand_hits_inv": 140,
+   "cand_hits_inv_on_common": 1,
+   "ref_hits_inv_on_common": 1,
+   "invocab_mask": true,
+   "acc_base_inv": 0.28310789556464294,
+   "acc_cand_inv": 2.034587995930824
+  },
+  "test20k": {
+   "common_spans": 2494,
+   "patch_p": 1.0,
+   "boot_delta_pct": 0.7524539262820513,
+   "boot_ci95_pct": [
+    0.44057992788461536,
+    1.0404146634615383
+   ],
+   "byte_range": [
+    3,
+    80053
+   ],
+   "ref_only": 29,
+   "cand_only": 29,
+   "base_only": 29,
+   "discordants": 58,
+   "sens_discordants": "=",
+   "sig_gain": false,
+   "sig_loss": false,
+   "patch_testable": true,
+   "boot_indicatif": "+",
+   "boot_gain_indicatif": true,
+   "boot_loss_indicatif": false,
+   "boot_conflit_mcnemar": false,
+   "base_zero_hit": false,
+   "cand_hits_inv": 575,
+   "cand_hits_inv_on_common": 39,
+   "ref_hits_inv_on_common": 39,
+   "invocab_mask": true,
+   "acc_base_inv": 2.9000672696016148,
+   "acc_cand_inv": 3.554648862512364
+  },
+  "queries": {
+   "common_spans": 1498,
+   "patch_p": 1.0,
+   "boot_delta_pct": 3.2407407407407405,
+   "boot_ci95_pct": [
+    3.1177662037037037,
+    3.3528645833333335
+   ],
+   "byte_range": [
+    3,
+    27800
+   ],
+   "ref_only": 0,
+   "cand_only": 0,
+   "base_only": 0,
+   "discordants": 0,
+   "sens_discordants": "=",
+   "sig_gain": false,
+   "sig_loss": false,
+   "patch_testable": false,
+   "boot_indicatif": "+",
+   "boot_gain_indicatif": true,
+   "boot_loss_indicatif": false,
+   "boot_conflit_mcnemar": false,
+   "base_zero_hit": true,
+   "cand_hits_inv": 250,
+   "cand_hits_inv_on_common": 0,
+   "ref_hits_inv_on_common": 0,
+   "invocab_mask": true,
+   "acc_base_inv": 0.0,
+   "acc_cand_inv": 4.098360655737705
+  },
+  "test_v3": {
+   "common_spans": 2895,
+   "patch_p": 0.1548157850344286,
+   "boot_delta_pct": -0.08072916666666667,
+   "boot_ci95_pct": [
+    -0.47268880208333336,
+    0.3099283854166665
+   ],
+   "byte_range": [
+    5,
+    77095
+   ],
+   "ref_only": 56,
+   "cand_only": 41,
+   "base_only": 56,
+   "discordants": 97,
+   "sens_discordants": "ref",
+   "sig_gain": false,
+   "sig_loss": false,
+   "patch_testable": true,
+   "boot_indicatif": "0",
+   "boot_gain_indicatif": false,
+   "boot_loss_indicatif": false,
+   "boot_conflit_mcnemar": false,
+   "base_zero_hit": false,
+   "cand_hits_inv": 531,
+   "cand_hits_inv_on_common": 49,
+   "ref_hits_inv_on_common": 64,
+   "invocab_mask": true,
+   "acc_base_inv": 4.117009750812568,
+   "acc_cand_inv": 3.2757557063541025
+  },
+  "neither_code": {
+   "common_spans": 502,
+   "patch_p": 0.00634765625,
+   "boot_delta_pct": 6.678025265957446,
+   "boot_ci95_pct": [
+    4.251059674202128,
+    9.288044381648934
+   ],
+   "byte_range": [
+    4,
+    24456
+   ],
+   "ref_only": 11,
+   "cand_only": 1,
+   "base_only": 11,
+   "discordants": 12,
+   "sens_discordants": "ref",
+   "sig_gain": false,
+   "sig_loss": true,
+   "patch_testable": true,
+   "boot_indicatif": "+",
+   "boot_gain_indicatif": true,
+   "boot_loss_indicatif": false,
+   "boot_conflit_mcnemar": true,
+   "base_zero_hit": false,
+   "cand_hits_inv": 865,
+   "cand_hits_inv_on_common": 10,
+   "ref_hits_inv_on_common": 20,
+   "invocab_mask": true,
+   "acc_base_inv": 6.951274027715691,
+   "acc_cand_inv": 14.115535248041775
+  }
+ },
+ "verdict": {
+  "decision": "NON RETENU",
+  "losses": [
+   "fc2"
+  ],
+  "gains": []
+ },
+ "vs_v2b": {
+  "fc2": {
+   "common_spans": 565,
+   "patch_p": 2.353672812205332e-14,
+   "boot_delta_pct": 3.0330882352941178,
+   "boot_ci95_pct": [
+    0.9459252450980391,
+    5.094784007352934
+   ],
+   "byte_range": [
+    4,
+    26320
+   ],
+   "ref_only": 51,
+   "cand_only": 1,
+   "base_only": 51,
+   "discordants": 52,
+   "sens_discordants": "ref",
+   "sig_gain": false,
+   "sig_loss": true,
+   "patch_testable": true,
+   "boot_indicatif": "+",
+   "boot_gain_indicatif": true,
+   "boot_loss_indicatif": false,
+   "boot_conflit_mcnemar": true,
+   "base_zero_hit": false,
+   "cand_hits_inv": 897,
+   "cand_hits_inv_on_common": 4,
+   "ref_hits_inv_on_common": 54,
+   "invocab_mask": true,
+   "acc_base_inv": 11.938754396854955,
+   "acc_cand_inv": 14.451425809569841
+  },
+  "fresh_en": {
+   "common_spans": 1119,
+   "patch_p": 0.3603776529357674,
+   "boot_delta_pct": 2.1453857421875,
+   "boot_ci95_pct": [
+    1.562347412109375,
+    2.7496337890625
+   ],
+   "byte_range": [
+    4,
+    32915
+   ],
+   "ref_only": 18,
+   "cand_only": 25,
+   "base_only": 18,
+   "discordants": 43,
+   "sens_discordants": "cand",
+   "sig_gain": false,
+   "sig_loss": false,
+   "patch_testable": true,
+   "boot_indicatif": "+",
+   "boot_gain_indicatif": true,
+   "boot_loss_indicatif": false,
+   "boot_conflit_mcnemar": false,
+   "base_zero_hit": false,
+   "cand_hits_inv": 361,
+   "cand_hits_inv_on_common": 33,
+   "ref_hits_inv_on_common": 26,
+   "invocab_mask": true,
+   "acc_base_inv": 3.062653132656633,
+   "acc_cand_inv": 5.546166845905669
+  },
+  "fresh_fr": {
+   "common_spans": 1676,
+   "patch_p": 5.529727786779404e-10,
+   "boot_delta_pct": 1.15234375,
+   "boot_ci95_pct": [
+    -0.2421875,
+    3.34384765625
+   ],
+   "byte_range": [
+    5,
+    25684
+   ],
+   "ref_only": 36,
+   "cand_only": 1,
+   "base_only": 36,
+   "discordants": 37,
+   "sens_discordants": "ref",
+   "sig_gain": false,
+   "sig_loss": true,
+   "patch_testable": true,
+   "boot_indicatif": "0",
+   "boot_gain_indicatif": false,
+   "boot_loss_indicatif": false,
+   "boot_conflit_mcnemar": false,
+   "base_zero_hit": false,
+   "cand_hits_inv": 140,
+   "cand_hits_inv_on_common": 1,
+   "ref_hits_inv_on_common": 36,
+   "invocab_mask": true,
+   "acc_base_inv": 2.13903743315508,
+   "acc_cand_inv": 2.034587995930824
+  },
+  "test20k": {
+   "common_spans": 2494,
+   "patch_p": 0.7948436539828609,
+   "boot_delta_pct": 1.092998798076923,
+   "boot_ci95_pct": [
+    0.8061961638621795,
+    1.3696915064102564
+   ],
+   "byte_range": [
+    3,
+    80053
+   ],
+   "ref_only": 28,
+   "cand_only": 31,
+   "base_only": 28,
+   "discordants": 59,
+   "sens_discordants": "cand",
+   "sig_gain": false,
+   "sig_loss": false,
+   "patch_testable": true,
+   "boot_indicatif": "+",
+   "boot_gain_indicatif": true,
+   "boot_loss_indicatif": false,
+   "boot_conflit_mcnemar": false,
+   "base_zero_hit": false,
+   "cand_hits_inv": 575,
+   "cand_hits_inv_on_common": 39,
+   "ref_hits_inv_on_common": 36,
+   "invocab_mask": true,
+   "acc_base_inv": 2.4665520591972494,
+   "acc_cand_inv": 3.554648862512364
+  },
+  "queries": {
+   "common_spans": 1498,
+   "patch_p": 1.0,
+   "boot_delta_pct": 2.6982060185185186,
+   "boot_ci95_pct": [
+    2.5752314814814814,
+    2.813946759259259
+   ],
+   "byte_range": [
+    3,
+    27800
+   ],
+   "ref_only": 0,
+   "cand_only": 0,
+   "base_only": 0,
+   "discordants": 0,
+   "sens_discordants": "=",
+   "sig_gain": false,
+   "sig_loss": false,
+   "patch_testable": false,
+   "boot_indicatif": "+",
+   "boot_gain_indicatif": true,
+   "boot_loss_indicatif": false,
+   "boot_conflit_mcnemar": false,
+   "base_zero_hit": false,
+   "cand_hits_inv": 250,
+   "cand_hits_inv_on_common": 0,
+   "ref_hits_inv_on_common": 0,
+   "invocab_mask": true,
+   "acc_base_inv": 1.4929829799940282,
+   "acc_cand_inv": 4.098360655737705
+  },
+  "test_v3": {
+   "common_spans": 2895,
+   "patch_p": 1.0,
+   "boot_delta_pct": 0.55859375,
+   "boot_ci95_pct": [
+    0.265625,
+    0.862044270833333
+   ],
+   "byte_range": [
+    5,
+    77095
+   ],
+   "ref_only": 40,
+   "cand_only": 41,
+   "base_only": 40,
+   "discordants": 81,
+   "sens_discordants": "cand",
+   "sig_gain": false,
+   "sig_loss": false,
+   "patch_testable": true,
+   "boot_indicatif": "+",
+   "boot_gain_indicatif": true,
+   "boot_loss_indicatif": false,
+   "boot_conflit_mcnemar": false,
+   "base_zero_hit": false,
+   "cand_hits_inv": 531,
+   "cand_hits_inv_on_common": 49,
+   "ref_hits_inv_on_common": 48,
+   "invocab_mask": true,
+   "acc_base_inv": 3.1960996749729147,
+   "acc_cand_inv": 3.2757557063541025
+  }
+ },
+ "verdict_v2b": {
+  "decision": "NON RETENU",
+  "losses": [
+   "fc2",
+   "fresh_fr"
+  ],
+  "gains": []
+ }
+}

v3_scale/verdicts/m8_v3.md

@@ -0,0 +1,29 @@
+# Verdict v4.1 — m8_v3 contre old_ref
+
+**NON RETENU** (pertes : fc2 ; gains : aucun)
+
+| fenêtre | acc réf | acc cand | spans communs | ref_only (réf juste) | cand_only (cand juste) | sens | p McNemar | DÉCISION | bootstrap Δ % [IC95] (indicatif) |
+|---|---|---|---|---|---|---|---|---|---|
+| fc2 | 11.50 | 14.45 | 565 | 29 | 1 | ref | 5.8e-08 | PERTE ⚠ conflit bootstrap | +3.09 [+1.11 ; +5.11] |
+| fresh_en | 3.40 | 5.55 | 1119 | 14 | 25 | cand | 0.11 | n.s. | +1.90 [+1.28 ; +2.52] |
+| fresh_fr | 0.28 | 2.03 | 1676 | 0 | 0 | = | 1 | n.s. (0 discordant) | +1.52 [+0.12 ; +3.72] |
+| test20k | 2.90 | 3.55 | 2494 | 29 | 29 | = | 1 | n.s. | +0.75 [+0.44 ; +1.04] |
+| queries | 0.00 | 4.10 | 1498 | 0 | 0 | = | 1 | n.s. (0 discordant) (réf 0 hit) | +3.24 [+3.12 ; +3.35] |
+| test_v3 | 4.12 | 3.28 | 2895 | 56 | 41 | ref | 0.15 | n.s. | -0.08 [-0.47 ; +0.31] |
+
+À part (contaminé, hors verdict) :
+
+| fenêtre | acc réf | acc cand | spans communs | ref_only (réf juste) | cand_only (cand juste) | sens | p McNemar | DÉCISION | bootstrap Δ % [IC95] (indicatif) |
+|---|---|---|---|---|---|---|---|---|---|
+| neither_code | 6.95 | 14.12 | 502 | 11 | 1 | ref | 0.0063 | PERTE ⚠ conflit bootstrap | +6.68 [+4.25 ; +9.29] |
+
+Référence m9g v2b (λ=2) : **NON RETENU**
+
+| fenêtre | acc réf | acc cand | spans communs | ref_only (réf juste) | cand_only (cand juste) | sens | p McNemar | DÉCISION | bootstrap Δ % [IC95] (indicatif) |
+|---|---|---|---|---|---|---|---|---|---|
+| fc2 | 11.94 | 14.45 | 565 | 51 | 1 | ref | 2.4e-14 | PERTE ⚠ conflit bootstrap | +3.03 [+0.95 ; +5.09] |
+| fresh_en | 3.06 | 5.55 | 1119 | 18 | 25 | cand | 0.36 | n.s. | +2.15 [+1.56 ; +2.75] |
+| fresh_fr | 2.14 | 2.03 | 1676 | 36 | 1 | ref | 5.5e-10 | PERTE | +1.15 [-0.24 ; +3.34] |
+| test20k | 2.47 | 3.55 | 2494 | 28 | 31 | cand | 0.79 | n.s. | +1.09 [+0.81 ; +1.37] |
+| queries | 1.49 | 4.10 | 1498 | 0 | 0 | = | 1 | n.s. (0 discordant) | +2.70 [+2.58 ; +2.81] |
+| test_v3 | 3.20 | 3.28 | 2895 | 40 | 41 | cand | 1 | n.s. | +0.56 [+0.27 ; +0.86] |