Harden ambiguity token map for WURCS uncertainty markers

README.md

@@ -20,7 +20,7 @@ This repository contains the contrastive Bertose checkpoint used to score ambigu
 
 - `checkpoints/best_v51_contrastive_model.pt` - contrastive ambiguity-resolution checkpoint.
 - `vocab/bpe_vocabulary.json` - WURCS BPE vocabulary.
-- `vocab/bpe_ambiguity_tokens.json` - ambiguous BPE token map used by the resolver.
+- `vocab/bpe_ambiguity_tokens.json` - ambiguous BPE token map used by the resolver, including common WURCS uncertainty-marker tokens.
 - `src/multimodal_glycan_bert_v3.py` - model definition.
 - `src/glycan_bert.py` - base BERT layers used by the multimodal model.
 - `src/wurcs_bpe_tokenizer.py` - WURCS BPE tokenizer.

SHA256SUMS

@@ -20,7 +20,7 @@ eb200fe67e613751c0571950e9a7f22f9f44fde0f85b73a40d392189a203f465  ./.cache/huggi
 e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855  ./.cache/huggingface/upload/vocab/bpe_vocabulary.json.lock
 c00560217b399adfb341aacc38053299c7d4b33b4229e89e68275cd454bb7f5b  ./.cache/huggingface/upload/vocab/bpe_vocabulary.json.metadata
 622368f62c23e97e9137c277eaadcc93ee3901cbb420b591422bb1c2e19689a5  ./.gitattributes
-21912ebe4c2b720eac3164c3628f37a39d6c918221c84e04b76a914fd709752d  ./README.md
+4d7dc814c23492bd75c154e35436fff108221285956106deeb117e561a6c560b  ./README.md
 ae468f4e8c06dc0c3848138a474dc43249aa6d14dfd0df8f58d68fcaad371152  ./checkpoints/best_v51_contrastive_model.pt
 daf55c190fece0678064e41697a9545592beb1285f8aa74e595b933b9d37b4c2  ./config.json
 6a56e6f73b8f874470ecde6e538f3f5029ae23aa6c10559817d1c2a8b59b7c0f  ./requirements.txt
@@ -30,5 +30,5 @@ daf55c190fece0678064e41697a9545592beb1285f8aa74e595b933b9d37b4c2  ./config.json
 b69f14c9976951325e3a0a4e8107a16126e67d410e966650f513f1f538a732bb  ./src/glycan_bert.py
 0d9ce16bf90242f38621d64cd974ea5679bff4c2013bea8d7bffe1b8dd120794  ./src/multimodal_glycan_bert_v3.py
 0bc54399362945601bcfd403441fc80968d173200dd0561f57568b2053a94839  ./src/wurcs_bpe_tokenizer.py
-c68cd003370b2dcdb162f848f766e4e62f2653c6c38d205f8cbe53a9aabe2d74  ./vocab/bpe_ambiguity_tokens.json
+ae6ab1ee4f2be992099ee5766de073954c74ccb005c490179cc70418c587c5b7  ./vocab/bpe_ambiguity_tokens.json
 6a572afdf53f1494ab96c896876b824ca7ea749777352606aa9f96bf270ceecc  ./vocab/bpe_vocabulary.json

vocab/bpe_ambiguity_tokens.json

@@ -1,7 +1,10 @@
 {
   "ambiguous_tokens": {
     "?": 32,
+    "u": 80,
+    "x": 83,
     "?|": 90,
+    "xx": 105,
     "a?|": 108,
     "a?|b": 109,
     "?|c": 110,
@@ -13,354 +16,568 @@
     "?|f": 116,
     "a?|b?|c?|d?|e?|f": 117,
     "?-": 118,
-    "?|g": 119,
-    "a?|b?|c?|d?|e?|f?|g": 120,
+    "?|g": 120,
+    "a?|b?|c?|d?|e?|f?|g": 121,
     "?|h": 122,
     "?|i": 123,
     "?|h?|i": 124,
-    "?|j": 125,
-    "?|h?|i?|j": 126,
+    "?|j": 126,
+    "?|h?|i?|j": 127,
     "?|k": 128,
     "?|h?|i?|j?|k": 129,
     "?|l": 130,
     "?|h?|i?|j?|k?|l": 131,
-    "?|m": 132,
-    "?|h?|i?|j?|k?|l?|m": 133,
+    "?|m": 133,
+    "?|h?|i?|j?|k?|l?|m": 134,
     "?|h?|i?|j?|k?|l?|m?|": 138,
     "n?|": 141,
     "n?|o": 142,
     "?}": 143,
-    "n?|o?|": 146,
-    "n?|o?|p": 147,
+    "n?|o?|": 147,
+    "n?|o?|p": 148,
     "?}-": 149,
     "?}-{": 150,
-    "n?|o?|p?|": 153,
-    "n?|o?|p?|q": 154,
+    "xxxx": 153,
+    "n?|o?|p?|": 154,
+    "n?|o?|p?|q": 155,
     "n?|o?|p?|q?|": 157,
     "n?|o?|p?|q?|r": 158,
+    "a2122h-1x": 159,
+    "u2": 160,
     "n?|o?|p?|q?|r?|": 165,
     "n?|o?|p?|q?|r?|s": 166,
     "n?|o?|p?|q?|r?|s?|": 170,
     "n?|o?|p?|q?|r?|s?|t": 171,
-    "?|u": 189,
-    "a?-": 197,
+    "uxxxx": 174,
+    "h-1x": 176,
+    "a2112h-1x": 179,
+    "uxxxxh": 188,
+    "u2122h": 189,
+    "?|u": 190,
+    "axxxx": 195,
+    "a1122h-1x": 197,
+    "a?-": 198,
     "c?-": 201,
-    "?|u?|": 209,
-    "?|u?|v": 210,
-    "b?-": 211,
+    "axxxxh-1x": 202,
+    "b?-": 209,
+    "?|u?|": 210,
+    "?|u?|v": 211,
     "a?-b1": 213,
     "d?-": 217,
+    "u2112h": 220,
     "b?-c1": 221,
-    "c?-d1": 230,
-    "?|u?|v?|": 231,
-    "?|u?|v?|w": 232,
+    "c?-d1": 229,
+    "u1": 230,
+    "?|u?|v?|": 233,
+    "?|u?|v?|w": 234,
+    "Aad21122h-2x": 235,
+    "a1221m-1x": 241,
     "1-?": 242,
     "d?-e1": 244,
-    "e?-": 245,
+    "e?-": 246,
+    "u1122h": 254,
     "e?-f1": 262,
     "?|u?|v?|w?|": 266,
     "?|u?|v?|w?|x": 267,
-    "f?-": 273,
+    "f?-": 272,
+    "xh": 280,
+    "m-1x": 284,
     "?|u?|v?|w?|x?|": 288,
     "?|u?|v?|w?|x?|y": 289,
-    "g?-": 298,
-    "n?|o?|p?|q?|r?|s?|t?": 304,
+    "g?-": 296,
+    "u1221": 297,
+    "u1221m": 300,
     "i?-": 306,
+    "n?|o?|p?|q?|r?|s?|t?": 307,
     "h?-": 308,
-    "?|u?}-{": 312,
-    "?|u?": 313,
-    "n?|o?|p?|q?|r?}-{": 314,
-    "n?|o?|p?|q?|r?": 315,
-    "f?-g1": 318,
+    "?|u?}-{": 313,
+    "?|u?": 314,
+    "n?|o?|p?|q?|r?}-{": 315,
+    "n?|o?|p?|q?|r?": 316,
+    "f?-g1": 317,
     "n?|o?}-{": 322,
     "n?|o?": 323,
     "?|u?|v?|w?|x?|y?|": 325,
     "?|u?|v?|w?|x?|y?|z": 326,
-    "n?|o?|p?|q?|r?|s?}-{": 328,
-    "n?|o?|p?|q?|r?|s?": 329,
-    "n?|o?|p?}-{": 331,
-    "n?|o?|p?": 332,
+    "n?|o?|p?|q?|r?|s?}-{": 329,
+    "n?|o?|p?|q?|r?|s?": 330,
+    "n?|o?|p?}-{": 332,
+    "n?|o?|p?": 333,
+    "uxxxxm": 335,
     "n?|o?|p?|q?}-{": 336,
     "n?|o?|p?|q?": 337,
     "g?-h1": 339,
-    "?|h?|i?|j?|k?|l?}-{": 342,
-    "?|h?|i?|j?|k?|l?": 343,
-    "n?}-{": 344,
-    "n?": 345,
-    "j?-": 346,
+    "j?-": 342,
+    "?|h?|i?|j?|k?|l?}-{": 343,
+    "?|h?|i?|j?|k?|l?": 344,
+    "n?}-{": 345,
+    "n?": 346,
     "h?-i1": 347,
     "?|u?|v?}-{": 351,
     "?|u?|v?": 352,
     "k?-": 353,
-    "i?-j1": 355,
-    "?|h?|i?|j?|k?|l?|m?": 363,
-    "?|h?|i?|j?|k?}-{": 364,
-    "?|h?|i?|j?|k?": 365,
-    "?|u?|v?|w?|x?|y?|z?|": 369,
+    "i?-j1": 354,
+    "a11221h-1x": 355,
+    "?|h?|i?|j?|k?|l?|m?": 364,
+    "?|h?|i?|j?|k?}-{": 366,
+    "?|h?|i?|j?|k?": 367,
+    "?|u?|v?|w?|x?|y?|z?|": 370,
+    "xh-1": 372,
     "?|h?|i?|j?}-{": 375,
     "?|h?|i?|j?": 376,
-    "l?-": 377,
-    "A?|": 392,
-    "A?|B": 393,
+    "l?-": 379,
+    "dxxxx": 381,
+    "AUdxxxx": 388,
+    "A?|": 393,
+    "A?|B": 394,
+    "A-1x": 395,
+    "ax": 399,
     "j?-k1": 401,
-    "m?-": 404,
+    "m?-": 403,
     "?|h?|i?}-{": 408,
     "?|h?|i?": 409,
-    "k?-l1": 418,
-    "?|h?}-{": 420,
-    "?|h?": 421,
-    "A?|B?|": 424,
-    "A?|B?|C": 425,
-    "a?|b?|c?|d?|e?|f?|g?": 427,
-    "f?-g2": 431,
-    "a?|b?|c?|d?|e?|f?}-{": 437,
-    "a?|b?|c?|d?|e?|f?": 438,
-    "l?-m1": 442,
-    "?|u?|v?|w?}-{": 450,
-    "?|u?|v?|w?": 451,
+    "k?-l1": 417,
+    "f?-g2": 419,
+    "?|h?}-{": 421,
+    "?|h?": 422,
+    "A?|B?|": 425,
+    "A?|B?|C": 426,
+    "Aad1122h-2x": 428,
+    "a?|b?|c?|d?|e?|f?|g?": 429,
+    "a2211m-1x": 431,
+    "a?|b?|c?|d?|e?|f?}-{": 438,
+    "a?|b?|c?|d?|e?|f?": 439,
+    "AUdxxxxxh": 440,
+    "l?-m1": 443,
+    "xh-1u": 445,
+    "axx": 446,
+    "a212h-1x": 448,
+    "?|u?|v?|w?}-{": 451,
+    "?|u?|v?|w?": 452,
+    "a2122A-1x": 461,
     "A?|B?|C?|": 464,
     "A?|B?|C?|D": 465,
-    "a?|b?|c?|d?|e?}-{": 475,
-    "a?|b?|c?|d?|e?": 476,
-    "n?-": 499,
+    "a?|b?|c?|d?|e?}-{": 476,
+    "a?|b?|c?|d?|e?": 477,
+    "u11221h": 492,
+    "n?-": 497,
     "a?|b?|c?|d?}-{": 502,
     "a?|b?|c?|d?": 503,
-    "m?-n1": 518,
-    "A?|B?|C?|D?|": 521,
-    "A?|B?|C?|D?|E": 522,
-    "o?-": 534,
-    "d?-h1": 536,
-    "A?|B?|C?|D?|E?|": 542,
-    "A?|B?|C?|D?|E?|F": 543,
-    "c?-i1": 544,
-    "c?-h1": 549,
+    "uxxxxxh": 508,
+    "u2211": 513,
+    "axxxxm-1x": 515,
+    "m?-n1": 520,
+    "A?|B?|C?|D?|": 525,
+    "A?|B?|C?|D?|E": 526,
+    "xh-1x": 527,
+    "u2211m": 529,
+    "d?-h1": 535,
+    "o?-": 536,
+    "c?-i1": 543,
+    "A?|B?|C?|D?|E?|": 544,
+    "A?|B?|C?|D?|E?|F": 545,
+    "c?-h1": 546,
+    "u2122": 549,
     "A?|B?|C?|D?|E?|F?|": 550,
     "A?|B?|C?|D?|E?|F?|G": 551,
-    "?|u?|v?|w?|x?}-{": 563,
-    "?|u?|v?|w?|x?": 564,
+    "?|u?|v?|w?|x?}-{": 566,
+    "?|u?|v?|w?|x?": 567,
     "a?|b?|c?}-{": 571,
     "a?|b?|c?}-{a?|b?|c": 572,
     "a?|b?|c?}-{a?|b?|c?": 573,
-    "?|H": 581,
-    "2-?": 592,
+    "uxx": 574,
+    "?|H": 582,
+    "xh-1d": 591,
+    "2-?": 593,
+    "u212h": 594,
+    "ax12": 595,
     "?|H?|": 598,
     "?|H?|I": 599,
     "?}*OC": 600,
-    "c?-k1": 607,
+    "c?-k1": 606,
+    "x2": 608,
     "c?-g1": 609,
-    "?|H?|I?|": 615,
-    "?|H?|I?|J": 616,
-    "n?-o1": 617,
-    "d?-g1": 629,
-    "o?-p1": 634,
+    "?|H?|I?|": 617,
+    "?|H?|I?|J": 618,
+    "n?-o1": 619,
+    "d?-g1": 628,
+    "x1": 632,
+    "o?-p1": 633,
+    "u2112": 636,
+    "u2122A": 640,
     "p?-": 646,
-    "?|u?|v?|w?|x?|y?}-{": 653,
-    "?|u?|v?|w?|x?|y?": 654,
-    "b?-c2": 656,
-    "d?-i1": 658,
-    "c?-j1": 691,
-    "?}*OSO": 696,
-    "e?-h1": 701,
-    "q?-": 713,
-    "c?-f1": 720,
-    "i?-j2": 728,
-    "?|h?|i?}": 742,
-    "h?-i2": 747,
-    "g?-h2": 753,
-    "c?-l1": 756,
-    "j?-k2": 758,
-    "?|h?}": 759,
-    "c?-e1": 760,
+    "uxxxxA": 651,
+    "d?-i1": 656,
+    "?|u?|v?|w?|x?|y?}-{": 657,
+    "?|u?|v?|w?|x?|y?": 658,
+    "b?-c2": 659,
+    "a211h-1x": 663,
+    "a2112A-1x": 664,
+    "c?-j1": 690,
+    "a2112m-1x": 695,
+    "e?-h1": 696,
+    "?}*OSO": 699,
+    "q?-": 712,
+    "ax11": 713,
+    "c?-f1": 715,
+    "AUdxxxxh": 716,
+    "ud": 718,
+    "hxh": 720,
+    "i?-j2": 724,
+    "axx2": 739,
+    "h?-i2": 742,
+    "g?-h2": 743,
+    "?|h?|i?}": 744,
+    "j?-k2": 750,
+    "uxxxh": 755,
+    "c?-e1": 756,
+    "a2121A-1x": 757,
+    "c?-l1": 758,
+    "?|h?}": 760,
     "?|H?|I?|J?|": 761,
     "?|H?|I?|J?|K": 762,
-    "a?|b?|c?|d?|e?|f?}": 772,
-    "b?-e1": 774,
+    "ux": 766,
+    "b?-e1": 771,
+    "a?|b?|c?|d?|e?|f?}": 773,
+    "-2x": 777,
     "b?-f1": 791,
-    "d?-f1": 794,
-    "p?-q1": 796,
-    "a?|b?|c?|d?|e?}": 798,
-    "a?-d1": 800,
-    "m?-n2": 803,
-    "e?-g1": 809,
-    "?|h?|i?|j?}": 812,
-    "r?-": 817,
-    "a?-c1": 818,
-    "?|u?|v?|w?|x?|y?|z?": 822,
-    "a?-e1": 826,
-    "d?-j1": 833,
-    "b?-g1": 834,
-    "q?-r1": 847,
-    "d?-e2": 854,
-    "c?-m1": 860,
+    "d?-f1": 792,
+    "p?-q1": 799,
+    "m?-n2": 800,
+    "a?|b?|c?|d?|e?}": 801,
+    "a?-d1": 802,
+    "u2112m": 803,
+    "e?-g1": 804,
+    "r?-": 812,
+    "?|h?|i?|j?}": 815,
+    "u211h": 818,
+    "a?-c1": 820,
+    "a?-e1": 824,
+    "?|u?|v?|w?|x?|y?|z?": 825,
+    "d?-j1": 828,
+    "a11222h-1x": 831,
+    "b?-g1": 835,
+    "a122h-1x": 841,
+    "x21": 845,
+    "q?-r1": 849,
+    "d?-e2": 852,
+    "axx1": 855,
+    "c?-m1": 861,
+    "Aax21": 865,
+    "ax12xh-1u": 867,
+    "ax11xh-1u": 868,
     "a?-f1": 875,
-    "b?-d1": 887,
-    "?|H?|I?|J?|K?|": 892,
-    "?|H?|I?|J?|K?|L": 893,
-    "?|H?|I?|J?|K?|L?|": 894,
-    "?|H?|I?|J?|K?|L?|M": 895,
-    "?|H?|I?|J?|K?|L?|M?|": 896,
-    "a?-l1": 920,
-    "?*OSO/3=O/3=O": 923,
-    "k?-l2": 940,
-    "k?-o1": 942,
-    "N?|": 965,
-    "N?|O": 966,
-    "N?|O?|": 967,
-    "N?|O?|P": 968,
-    "N?|O?|P?|": 969,
-    "N?|O?|P?|Q": 970,
-    "N?|O?|P?|Q?|": 971,
-    "N?|O?|P?|Q?|R": 972,
-    "N?|O?|P?|Q?|R?|": 973,
-    "N?|O?|P?|Q?|R?|S": 974,
-    "N?|O?|P?|Q?|R?|S?|": 975,
-    "N?|O?|P?|Q?|R?|S?|T": 976,
-    "?|U": 977,
-    "?|U?|": 978,
-    "?|U?|V": 979,
-    "c?-d2": 983,
-    "r?-s1": 988,
-    "a?|b?}-{": 995,
-    "a?|b?}-{a?|b": 996,
-    "a?|b?}-{a?|b?": 997,
-    "e?-f2": 1001,
-    "g?-i1": 1006,
-    "i?-l1": 1010,
-    "s?-": 1011,
-    "?|h?|i?|j?|k?}": 1017,
-    "b?-h1": 1034,
-    "a?-j1": 1038,
-    "n?-o2": 1046,
-    "a?-b2": 1069,
-    "e?-i1": 1095,
-    "h?-j1": 1102,
-    "a?-k1": 1108,
-    "i?-k1": 1115,
-    "a?-g1": 1116,
-    "?}*OPO": 1122,
-    "d?-k1": 1129,
-    "a?-m1": 1151,
-    "a?-i1": 1159,
-    "A?}-{": 1174,
-    "A?": 1175,
-    "?}*OCC": 1177,
-    "l?-m2": 1179,
-    "A?|B?}-{": 1180,
-    "A?|B?": 1181,
-    "f?-h1": 1183,
-    "a?-n1": 1189,
-    "p?-q2": 1192,
+    "b?-d1": 877,
+    "axxxxxh-1x": 886,
+    "Aax21122h-2b": 887,
+    "?|H?|I?|J?|K?|": 894,
+    "?|H?|I?|J?|K?|L": 895,
+    "?|H?|I?|J?|K?|L?|": 896,
+    "?|H?|I?|J?|K?|L?|M": 897,
+    "?|H?|I?|J?|K?|L?|M?|": 898,
+    "xxxxh": 913,
+    "a2122m-1x": 917,
+    "ha122h-2x": 919,
+    "xm-1": 923,
+    "a?-l1": 926,
+    "k?-l2": 928,
+    "?*OSO/3=O/3=O": 929,
+    "dx": 933,
+    "k?-o1": 943,
+    "hxxxxh": 946,
+    "u2112A": 947,
+    "a1122m-1x": 953,
+    "axxxh-1x": 956,
+    "u12": 960,
+    "u212": 964,
+    "c?-d2": 967,
+    "N?|": 971,
+    "N?|O": 972,
+    "N?|O?|": 973,
+    "N?|O?|P": 974,
+    "N?|O?|P?|": 975,
+    "N?|O?|P?|Q": 976,
+    "N?|O?|P?|Q?|": 977,
+    "N?|O?|P?|Q?|R": 978,
+    "N?|O?|P?|Q?|R?|": 979,
+    "N?|O?|P?|Q?|R?|S": 980,
+    "N?|O?|P?|Q?|R?|S?|": 981,
+    "N?|O?|P?|Q?|R?|S?|T": 982,
+    "?|U": 983,
+    "?|U?|": 984,
+    "?|U?|V": 985,
+    "r?-s1": 991,
+    "e?-f2": 992,
+    "a?|b?}-{": 996,
+    "a?|b?}-{a?|b": 997,
+    "a?|b?}-{a?|b?": 998,
+    "xm-1u": 1000,
+    "a2222h-1x": 1001,
+    "g?-i1": 1002,
+    "u2122m": 1003,
+    "i?-l1": 1007,
+    "s?-": 1008,
+    "a1x2": 1014,
+    "?|h?|i?|j?|k?}": 1020,
+    "ax12xh-1d": 1031,
+    "n?-o2": 1034,
+    "b?-h1": 1035,
+    "a?-j1": 1040,
+    "hdx": 1050,
+    "hdxdh": 1053,
+    "u22": 1067,
+    "a?-b2": 1073,
+    "axxxxA-1x": 1085,
+    "e?-i1": 1092,
+    "h?-j1": 1100,
+    "u122h": 1108,
+    "a?-k1": 1111,
+    "u21": 1112,
+    "i?-k1": 1113,
+    "a?-g1": 1119,
+    "?}*OPO": 1124,
+    "d?-k1": 1130,
+    "a1x1": 1147,
+    "a?-m1": 1153,
+    "l?-m2": 1154,
+    "xm": 1158,
+    "f?-h1": 1159,
+    "axx1xh-1u": 1160,
+    "a?-i1": 1161,
+    "u11222h": 1162,
+    "a4334h-1x": 1169,
+    "A?}-{": 1179,
+    "A?": 1180,
+    "?}*OCC": 1182,
+    "p?-q2": 1184,
+    "A?|B?}-{": 1185,
+    "A?|B?": 1186,
+    "a1112h-1x": 1189,
+    "u2121A": 1190,
+    "a?-n1": 1191,
+    "axx2xh-1d": 1196,
     "c?-n1": 1197,
-    "?|U?|V?|": 1202,
-    "?|U?|V?|W": 1203,
-    "?|U?|V?|W?|": 1204,
-    "?|U?|V?|W?|X": 1205,
-    "?|U?|V?|W?|X?|": 1206,
-    "?|U?|V?|W?|X?|Y": 1207,
-    "?|a": 1208,
+    "uxx2": 1199,
+    "xA-1": 1200,
+    "?|U?|V?|": 1203,
+    "?|U?|V?|W": 1204,
+    "?|U?|V?|W?|": 1205,
+    "?|U?|V?|W?|X": 1206,
+    "?|U?|V?|W?|X?|": 1207,
+    "?|U?|V?|W?|X?|Y": 1208,
+    "?|a": 1209,
+    "u1122m": 1218,
+    "xxxh": 1220,
     "s?-t1": 1223,
-    "?|h?|i?|j?|k?|l?|m?}": 1228,
-    "g?-j1": 1234,
-    "A?|B?|C?|D?}-{": 1242,
-    "A?|B?|C?|D?": 1243,
-    "a?-h1": 1253,
-    "?|H?|I?|J?}-{": 1257,
-    "?|H?|I?|J?": 1258,
-    "o?-p2": 1261,
-    "b?-i1": 1273,
-    "?|h?|i?|j?|k?|l?}": 1309,
-    "j?-m1": 1317,
-    "c?-o1": 1318,
-    "a?-o1": 1330,
-    "a?|b?|c?}*OC": 1331,
+    "?|h?|i?|j?|k?|l?|m?}": 1229,
+    "g?-j1": 1232,
+    "a1211h-1x": 1233,
+    "o?-p2": 1241,
+    "u4": 1242,
+    "axx2xh-1u": 1245,
+    "A?|B?|C?|D?}-{": 1247,
+    "A?|B?|C?|D?": 1248,
+    "u222h": 1249,
+    "ax22": 1255,
+    "a4344h-1x": 1256,
+    "a?-h1": 1257,
+    "?|H?|I?|J?}-{": 1261,
+    "?|H?|I?|J?": 1262,
+    "b?-i1": 1274,
+    "a1122A-1x": 1278,
+    "a222h-1x": 1283,
+    "a1221h-1x": 1286,
+    "xA-1u": 1287,
+    "ux12": 1305,
+    "u112": 1306,
+    "j?-m1": 1308,
+    "?|h?|i?|j?|k?|l?}": 1311,
+    "xh-2x": 1314,
+    "c?-o1": 1319,
+    "a2211h-1x": 1323,
+    "dxx": 1325,
+    "axxxxx": 1329,
+    "a?-o1": 1331,
+    "a?|b?|c?}*OC": 1332,
+    "uxx2xh": 1345,
     "b?-j1": 1357,
     "a?-r1": 1361,
     "n?}": 1363,
+    "axx21": 1370,
     "A?|B?|C?}-{": 1371,
     "A?|B?|C?": 1372,
-    "m?-p1": 1375,
-    "l?-p1": 1383,
-    "a?-p1": 1444,
-    "k?-n1": 1446,
-    "j?-l1": 1470,
-    "?|U?|V?|W?|X?|Y?|": 1471,
-    "?|U?|V?|W?|X?|Y?|Z": 1472,
-    "?|aa?|": 1473,
-    "?|aa?|a": 1474,
-    "?|aa?|ab": 1475,
-    "?*OPO/3O/3=O": 1476,
-    "l?-q1": 1489,
-    "l?-n1": 1499,
-    "a?-s1": 1517,
-    "k?-m1": 1524,
-    "a?-q1": 1546,
-    "c?-q1": 1547,
-    "t?-": 1551,
-    "a?|b?|c?|d?}*OC": 1565,
-    "f?-i1": 1590,
+    "m?-p1": 1374,
+    "ux12xh": 1378,
+    "l?-p1": 1385,
+    "ax1": 1386,
+    "u2222h": 1411,
+    "ax21": 1419,
+    "k?-n1": 1441,
+    "x2h": 1445,
+    "a2212h-1x": 1446,
+    "a?-p1": 1447,
+    "xxh": 1454,
+    "j?-l1": 1459,
+    "?|U?|V?|W?|X?|Y?|": 1472,
+    "?|U?|V?|W?|X?|Y?|Z": 1473,
+    "?|aa?|": 1474,
+    "?|aa?|a": 1475,
+    "?|aa?|ab": 1476,
+    "?*OPO/3O/3=O": 1477,
+    "a1x2xh-1u": 1483,
+    "ad222m-1x": 1485,
+    "l?-q1": 1487,
+    "x22h": 1489,
+    "l?-n1": 1492,
+    "a3344h-1x": 1493,
+    "122h-2x": 1504,
+    "a?-s1": 1520,
+    "11xh": 1523,
+    "k?-m1": 1526,
+    "a1x1xh-1u": 1527,
+    "axxxxxh-1a": 1529,
+    "t?-": 1544,
+    "a?-q1": 1550,
+    "c?-q1": 1551,
+    "f?-i1": 1553,
+    "a?|b?|c?|d?}*OC": 1567,
+    "a21EEA-1x": 1569,
+    "u1211h": 1573,
+    "a1222h-1x": 1579,
     "c?-p1": 1591,
     "n?-q1": 1593,
-    "?|i?}": 1611,
-    "a?|b?|c?|d?|e?}*OC": 1612,
-    "m?-q1": 1617,
-    "q?-r2": 1623,
-    "l?-o1": 1624,
-    "m?-r1": 1628,
-    "a?-t1": 1630,
+    "q?-r2": 1611,
+    "?|i?}": 1612,
+    "a?|b?|c?|d?|e?}*OC": 1613,
+    "x2xh": 1616,
+    "m?-q1": 1618,
+    "a21eEA-1x": 1622,
+    "l?-o1": 1625,
+    "a21FFA-1x": 1627,
+    "haxx": 1629,
+    "m?-r1": 1630,
+    "adxx": 1631,
+    "a?-t1": 1632,
+    "a2221h-1x": 1634,
+    "ud222m": 1648,
     "a?|b?|c?|d?}*OSO": 1649,
-    "c?-r1": 1675,
-    "1-d?|i?}": 1683,
-    "j?-n1": 1691,
-    "u?-": 1694,
+    "u2-": 1651,
+    "u1221h": 1654,
+    "xm-1x": 1663,
+    "c?-r1": 1676,
+    "u1121h": 1679,
+    "1-d?|i?}": 1684,
+    "a2x": 1689,
+    "j?-n1": 1690,
+    "u?-": 1692,
+    "ax1xx": 1701,
+    "Aadxxxx": 1710,
     "a?|b?|c?|d?|e?}*OSO": 1718,
-    "?*OCC/3=O": 1723,
-    "?%": 1752,
-    "?*OP^XOCCN/3O/3=O": 1770,
-    "t?-u1": 1772,
-    "?*": 1774,
-    "c?-s1": 1775,
-    "a?-u1": 1793,
-    "f?-h2": 1808,
-    "e?-j1": 1811,
-    "c?-t1": 1818,
-    "f1-a?|b?|c?|d?|e?}": 1822,
-    "u?-v1": 1835,
-    "h?-k1": 1841,
-    "?|H?|I?|J?|K?}-{": 1846,
-    "?|H?|I?|J?|K?": 1847,
-    "n?|o?}": 1851,
-    "1-d?|h?}": 1852,
-    "q?-s1": 1872,
-    "%?%": 1880,
-    "b?-g2": 1881,
-    "r?-s2": 1882,
-    "d?-l1": 1898,
-    "v?-": 1917,
-    "b?-k1": 1927,
-    "?|aa?|ab?|a": 1942,
-    "?|aa?|ab?|ac": 1943,
-    "?|aa?|ab?|ac?|": 1944,
-    "?|aa?|ab?|ac?|ad": 1945,
-    "a?|b?}*OC": 1949,
-    "?*OC": 1952,
-    "e?-k1": 1955,
-    "a?-d2": 1999,
-    "s?-t2": 2013,
-    "a?-f2": 2027,
-    "o?-q1": 2030,
-    "?}*OP^XOCCN": 2040,
-    "a?|b?|c?}*OCC": 2047,
-    "m?-o1": 2048,
-    "c?-f2": 2058,
-    "A?|B?|C?|D?|E?|F?|G?": 2060,
-    "a?|b?|c?}*OSO": 2071,
-    "?|U?|V?}-{": 2079,
-    "?|U?|V?": 2080,
-    "c?-u1": 2087
+    "u1112h": 1722,
+    "?*OCC/3=O": 1724,
+    "a344h-1x": 1731,
+    "a2x2": 1734,
+    "ad122m-1x": 1740,
+    "xd": 1742,
+    "?%": 1753,
+    "u2221h": 1754,
+    "ux11xh": 1762,
+    "haxxxh-2x": 1765,
+    "uxx1": 1766,
+    "?*OP^XOCCN/3O/3=O": 1771,
+    "t?-u1": 1773,
+    "?*": 1776,
+    "c?-s1": 1777,
+    "ax11xh-1d": 1785,
+    "xxxm": 1786,
+    "e?-j1": 1787,
+    "u1122": 1793,
+    "a?-u1": 1798,
+    "f?-h2": 1805,
+    "a2221m-1x": 1808,
+    "a21x2": 1815,
+    "c?-t1": 1822,
+    "f1-a?|b?|c?|d?|e?}": 1827,
+    "udxxxm": 1831,
+    "h?-k1": 1832,
+    "r?-s2": 1838,
+    "u?-v1": 1841,
+    "u1222h": 1845,
+    "?|H?|I?|J?|K?}-{": 1851,
+    "?|H?|I?|J?|K?": 1852,
+    "n?|o?}": 1856,
+    "1-d?|h?}": 1857,
+    "a2121h-1x": 1872,
+    "q?-s1": 1874,
+    "hUxxxh": 1878,
+    "%?%": 1882,
+    "b?-g2": 1883,
+    "ud122m": 1887,
+    "ud2": 1890,
+    "u2121h": 1895,
+    "u1x2xh": 1899,
+    "d?-l1": 1900,
+    "Aax1": 1904,
+    "ax12xh-1x": 1908,
+    "v?-": 1912,
+    "d2h-1x": 1914,
+    "u4334h": 1917,
+    "Aadxxxxxh-2x": 1923,
+    "uxx1xh": 1924,
+    "dxh": 1925,
+    "b?-k1": 1929,
+    "a2111h-1x": 1932,
+    "u1122A": 1933,
+    "Aa11122h-2x": 1936,
+    "e?-k1": 1940,
+    "?|aa?|ab?|a": 1944,
+    "?|aa?|ab?|ac": 1945,
+    "?|aa?|ab?|ac?|": 1946,
+    "?|aa?|ab?|ac?|ad": 1947,
+    "axx1xh-1d": 1949,
+    "a?|b?}*OC": 1952,
+    "?*OC": 1957,
+    "a2x1": 1958,
+    "ax22xh-1d": 1959,
+    "u4-": 1963,
+    "u2-t3|t": 1978,
+    "u2-t3|t6": 1979,
+    "s?-t2": 1985,
+    "u2212h": 1991,
+    "u4344h": 1997,
+    "a?-d2": 2001,
+    "a1121h-1x": 2003,
+    "axx21h-1b": 2008,
+    "u2211h": 2014,
+    "a3344m-1x": 2018,
+    "ax12xA-1u": 2025,
+    "ud122h": 2027,
+    "o?-q1": 2028,
+    "a1121A-1x": 2029,
+    "a?-f2": 2030,
+    "a1111h-1x": 2034,
+    "m?-o1": 2035,
+    "ud1": 2041,
+    "?}*OP^XOCCN": 2044,
+    "a?|b?|c?}*OCC": 2049,
+    "a2d12m-1x": 2051,
+    "c?-f2": 2059,
+    "A?|B?|C?|D?|E?|F?|G?": 2061,
+    "ad122h-1x": 2069,
+    "a112h-1x": 2071,
+    "a?|b?|c?}*OSO": 2072,
+    "?|U?|V?}-{": 2080,
+    "?|U?|V?": 2081,
+    "u2221m": 2083,
+    "x22h-2a": 2086,
+    "c?-u1": 2088
   },
   "ambiguous_ids": [
     32,
+    80,
+    83,
     90,
+    105,
     108,
     109,
     110,
@@ -372,73 +589,94 @@
     116,
     117,
     118,
-    119,
     120,
+    121,
     122,
     123,
     124,
-    125,
     126,
+    127,
     128,
     129,
     130,
     131,
-    132,
     133,
+    134,
     138,
     141,
     142,
     143,
-    146,
     147,
+    148,
     149,
     150,
     153,
     154,
+    155,
     157,
     158,
+    159,
+    160,
     165,
     166,
     170,
     171,
+    174,
+    176,
+    179,
+    188,
     189,
+    190,
+    195,
     197,
+    198,
     201,
+    202,
     209,
     210,
     211,
     213,
     217,
+    220,
     221,
+    229,
     230,
-    231,
-    232,
+    233,
+    234,
+    235,
+    241,
     242,
     244,
-    245,
+    246,
+    254,
     262,
     266,
     267,
-    273,
+    272,
+    280,
+    284,
     288,
     289,
-    298,
-    304,
+    296,
+    297,
+    300,
     306,
+    307,
     308,
-    312,
     313,
     314,
     315,
-    318,
+    316,
+    317,
     322,
     323,
     325,
     326,
-    328,
     329,
-    331,
+    330,
     332,
+    333,
+    335,
     336,
     337,
     339,
@@ -451,123 +689,181 @@
     351,
     352,
     353,
+    354,
     355,
-    363,
     364,
-    365,
-    369,
+    366,
+    367,
+    370,
+    372,
     375,
     376,
-    377,
-    392,
+    379,
+    381,
+    388,
     393,
+    394,
+    395,
+    399,
     401,
-    404,
+    403,
     408,
     409,
-    418,
-    420,
+    417,
+    419,
     421,
-    424,
+    422,
     425,
-    427,
+    426,
+    428,
+    429,
     431,
-    437,
     438,
-    442,
-    450,
+    439,
+    440,
+    443,
+    445,
+    446,
+    448,
     451,
+    452,
+    461,
     464,
     465,
-    475,
     476,
-    499,
+    477,
+    492,
+    497,
     502,
     503,
-    518,
-    521,
-    522,
-    534,
+    508,
+    513,
+    515,
+    520,
+    525,
+    526,
+    527,
+    529,
+    535,
     536,
-    542,
     543,
     544,
+    545,
+    546,
     549,
     550,
     551,
-    563,
-    564,
+    566,
+    567,
     571,
     572,
     573,
-    581,
-    592,
+    574,
+    582,
+    591,
+    593,
+    594,
+    595,
     598,
     599,
     600,
-    607,
+    606,
+    608,
     609,
-    615,
-    616,
     617,
-    629,
-    634,
+    618,
+    619,
+    628,
+    632,
+    633,
+    636,
+    640,
     646,
-    653,
-    654,
+    651,
     656,
+    657,
     658,
-    691,
+    659,
+    663,
+    664,
+    690,
+    695,
     696,
-    701,
+    699,
+    712,
     713,
+    715,
+    716,
+    718,
     720,
-    728,
+    724,
+    739,
     742,
-    747,
-    753,
+    743,
+    744,
+    750,
+    755,
     756,
+    757,
     758,
-    759,
     760,
     761,
     762,
-    772,
-    774,
+    766,
+    771,
+    773,
+    777,
     791,
-    794,
-    796,
-    798,
+    792,
+    799,
     800,
+    801,
+    802,
     803,
-    809,
+    804,
     812,
-    817,
+    815,
     818,
-    822,
-    826,
-    833,
-    834,
-    847,
-    854,
-    860,
+    820,
+    824,
+    825,
+    828,
+    831,
+    835,
+    841,
+    845,
+    849,
+    852,
+    855,
+    861,
+    865,
+    867,
+    868,
     875,
+    877,
+    886,
     887,
-    892,
-    893,
     894,
     895,
     896,
-    920,
+    897,
+    898,
+    913,
+    917,
+    919,
     923,
-    940,
-    942,
-    965,
-    966,
+    926,
+    928,
+    929,
+    933,
+    943,
+    946,
+    947,
+    953,
+    956,
+    960,
+    964,
     967,
-    968,
-    969,
-    970,
     971,
     972,
     973,
@@ -577,145 +873,278 @@
     977,
     978,
     979,
+    980,
+    981,
+    982,
     983,
-    988,
-    995,
+    984,
+    985,
+    991,
+    992,
     996,
     997,
+    998,
+    1000,
     1001,
-    1006,
-    1010,
-    1011,
-    1017,
+    1002,
+    1003,
+    1007,
+    1008,
+    1014,
+    1020,
+    1031,
     1034,
-    1038,
-    1046,
-    1069,
-    1095,
-    1102,
+    1035,
+    1040,
+    1050,
+    1053,
+    1067,
+    1073,
+    1085,
+    1092,
+    1100,
     1108,
-    1115,
-    1116,
-    1122,
-    1129,
-    1151,
+    1111,
+    1112,
+    1113,
+    1119,
+    1124,
+    1130,
+    1147,
+    1153,
+    1154,
+    1158,
     1159,
-    1174,
-    1175,
-    1177,
+    1160,
+    1161,
+    1162,
+    1169,
     1179,
     1180,
-    1181,
-    1183,
+    1182,
+    1184,
+    1185,
+    1186,
     1189,
-    1192,
+    1190,
+    1191,
+    1196,
     1197,
-    1202,
+    1199,
+    1200,
     1203,
     1204,
     1205,
     1206,
     1207,
     1208,
+    1209,
+    1218,
+    1220,
     1223,
-    1228,
-    1234,
+    1229,
+    1232,
+    1233,
+    1241,
     1242,
-    1243,
-    1253,
+    1245,
+    1247,
+    1248,
+    1249,
+    1255,
+    1256,
     1257,
-    1258,
     1261,
-    1273,
-    1309,
-    1317,
-    1318,
-    1330,
+    1262,
+    1274,
+    1278,
+    1283,
+    1286,
+    1287,
+    1305,
+    1306,
+    1308,
+    1311,
+    1314,
+    1319,
+    1323,
+    1325,
+    1329,
     1331,
+    1332,
+    1345,
     1357,
     1361,
     1363,
+    1370,
     1371,
     1372,
-    1375,
-    1383,
-    1444,
+    1374,
+    1378,
+    1385,
+    1386,
+    1411,
+    1419,
+    1441,
+    1445,
     1446,
-    1470,
-    1471,
+    1447,
+    1454,
+    1459,
     1472,
     1473,
     1474,
     1475,
     1476,
+    1477,
+    1483,
+    1485,
+    1487,
     1489,
-    1499,
-    1517,
-    1524,
-    1546,
-    1547,
+    1492,
+    1493,
+    1504,
+    1520,
+    1523,
+    1526,
+    1527,
+    1529,
+    1544,
+    1550,
     1551,
-    1565,
-    1590,
+    1553,
+    1567,
+    1569,
+    1573,
+    1579,
     1591,
     1593,
     1611,
     1612,
-    1617,
-    1623,
-    1624,
-    1628,
+    1613,
+    1616,
+    1618,
+    1622,
+    1625,
+    1627,
+    1629,
     1630,
+    1631,
+    1632,
+    1634,
+    1648,
     1649,
-    1675,
-    1683,
-    1691,
-    1694,
+    1651,
+    1654,
+    1663,
+    1676,
+    1679,
+    1684,
+    1689,
+    1690,
+    1692,
+    1701,
+    1710,
     1718,
-    1723,
-    1752,
-    1770,
-    1772,
-    1774,
-    1775,
+    1722,
+    1724,
+    1731,
+    1734,
+    1740,
+    1742,
+    1753,
+    1754,
+    1762,
+    1765,
+    1766,
+    1771,
+    1773,
+    1776,
+    1777,
+    1785,
+    1786,
+    1787,
     1793,
+    1798,
+    1805,
     1808,
-    1811,
-    1818,
+    1815,
     1822,
-    1835,
+    1827,
+    1831,
+    1832,
+    1838,
     1841,
-    1846,
-    1847,
+    1845,
     1851,
     1852,
+    1856,
+    1857,
     1872,
-    1880,
-    1881,
+    1874,
+    1878,
     1882,
-    1898,
+    1883,
+    1887,
+    1890,
+    1895,
+    1899,
+    1900,
+    1904,
+    1908,
+    1912,
+    1914,
     1917,
-    1927,
-    1942,
-    1943,
+    1923,
+    1924,
+    1925,
+    1929,
+    1932,
+    1933,
+    1936,
+    1940,
     1944,
     1945,
+    1946,
+    1947,
     1949,
     1952,
-    1955,
-    1999,
-    2013,
+    1957,
+    1958,
+    1959,
+    1963,
+    1978,
+    1979,
+    1985,
+    1991,
+    1997,
+    2001,
+    2003,
+    2008,
+    2014,
+    2018,
+    2025,
     2027,
+    2028,
+    2029,
     2030,
-    2040,
-    2047,
-    2048,
-    2058,
-    2060,
+    2034,
+    2035,
+    2041,
+    2044,
+    2049,
+    2051,
+    2059,
+    2061,
+    2069,
     2071,
-    2079,
+    2072,
     2080,
-    2087
+    2081,
+    2083,
+    2086,
+    2088
   ],
-  "source_vocab": "data/bpe_vocabulary_clean.json"
-}
+  "source_vocab": "data/bpe_vocabulary_clean.json",
+  "augmentation_rule": "Includes original ambiguity map plus WURCS uncertainty marker tokens containing '?', containing 'x', or starting with lowercase 'u'."
+}