Прогон v2: эмбеддинги, дообучение, варианты квантизации

v2/dumps/encoder_fp32.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:14c674307276b5921ee98565196369551e4e2f8c04b1bcad9f19fa3164994e63
+size 2701647

v2/dumps/encoder_int8_emb_only.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a5a38c5efe400e23089236797d9b4afb7a214b6dd22b4f10e317f18352296687
+size 2701567

v2/dumps/encoder_int8_full.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:724e008b30551a3b38a956c4937b87cd6a12499f424e14f48198ac0e1afaea51
+size 2699856

v2/dumps/encoder_int8_full_perchannel.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:69be1b2a7b7e5d99369207e71dc4b3acfbb8be0bc40f82a21f8bb17ce1bc7d33
+size 2701333

v2/dumps/encoder_torch_fp32.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0d69db4da6cf021232502222f9fb30c47cba48394a1d93df0eea9fb11a188607
+size 2701624

v2/dumps/finetuned_torch_fp32.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d91e45799c47ce0de0fa4a1ad866cfb1949a8ab26c22592fff85941ff36aeec6
+size 632625

v2/encoder_int8_emb_only.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ebc9fc66ba19f62b2a799d0ff47c1a15d5d086106f954d5d4a42d96e54d09a0
+size 181711482

v2/encoder_int8_full.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7a0e292f503a41525ecc675f40f0cd597c272fd6f98d8dc10a932ab21a92ce27
+size 118108055

v2/encoder_int8_full_perchannel.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7dcf738ddfe05812a3ee198ab9626010673977bf7cf709a4a0f5a063318f9241
+size 118315775

v2/report_v2.md

@@ -0,0 +1,19 @@
+# multilingual-e5-small, прогон v2
+
+Значения – доля верных ответов, в скобках 95% доверительный интервал Уилсона.
+
+## Замороженный e5 против корректного дообучения (PyTorch fp32)
+
+| Модель | CV acc | Test top-1 | macro-F1 | Порог | Test с порогом | OOD отклонено | Внешний тест | Сценарии |
+|---|---|---|---|---|---|---|---|---|
+| `e5_frozen_logreg` | 0.803 | 0.881 [0.840; 0.912] | 0.884 | 0.731 | 0.695 [0.642; 0.743] | 0.769 [0.664; 0.849] | 0.627 [0.559; 0.690] | 0.846 [0.739; 0.914] |
+| `e5_finetuned_meanpool` | 0.749 | 0.811 [0.765; 0.851] | 0.806 | 0.379 | 0.701 [0.649; 0.749] | 0.628 [0.517; 0.727] | 0.627 [0.559; 0.690] | 0.862 [0.757; 0.925] |
+
+## Варианты ONNX: размер, память, задержка (CPU, 1 поток), качество
+
+| Вариант | Размер, МБ | Память сессии, МБ | p50, мс | p95, мс | Совпадение top-1 с fp32 | Test top-1 | Сценарии с порогом |
+|---|---|---|---|---|---|---|---|
+| `encoder/fp32` | 448 | 516 | 12.4 | 22.7 | 1.000 | 0.881 [0.840; 0.912] | 0.846 [0.739; 0.914] |
+| `encoder/int8_full` | 113 | 151 | 7.2 | 14.6 | 0.953 | 0.868 [0.826; 0.901] | 0.769 [0.654; 0.855] |
+| `encoder/int8_full_perchannel` | 113 | 147 | 8.1 | 16.7 | 0.968 | 0.884 [0.844; 0.914] | 0.815 [0.704; 0.891] |
+| `encoder/int8_emb_only` | 173 | 242 | 13.5 | 22.6 | 0.997 | 0.881 [0.840; 0.912] | 0.831 [0.722; 0.903] |

v2/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:255f5e32cb320b4dc2b31e0f203832fecafe87ca4cc87c3c41eb8ccae73f441a
+size 17082987

v2/tokenizer_config.json

@@ -0,0 +1,17 @@
+{
+  "add_prefix_space": true,
+  "backend": "tokenizers",
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "is_local": false,
+  "local_files_only": false,
+  "mask_token": "<mask>",
+  "model_max_length": 512,
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "XLMRobertaTokenizer",
+  "unk_token": "<unk>"
+}