F1 0.7756 @ 2026-07-06T19:32:57

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+training_report.png filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,3 +1,48 @@
 ---
-license: mit
+tags: [retrieval, chunk-selection, trm]
 ---
+# TRM chunk selector
+
+Recursive PASS/FAIL gate over retrieved chunks (TRM core + skip head),
+1.91M params on frozen `openai/text-embedding-ada-002` embeddings
+(dim 1539). Decides per chunk whether it belongs in the answer set —
+a variable-size selection instead of a fixed top-k.
+
+![training report](training_report.png)
+
+## Test metrics (threshold 0.884)
+
+micro-P **0.8281** · micro-R **0.7294** ·
+micro-F1 **0.7756** · exact-set **0.1316** ·
+best epoch 13
+
+| bench | groups | P | R | F1 |
+|---|---|---|---|---|
+| gold_easy | 2 | 1.0 | 0.375 | 0.5455 |
+| gold_medium | 2 | 0.975 | 0.975 | 0.975 |
+| gold_hard | 5 | 0.9667 | 0.9667 | 0.9667 |
+| llm_held_out | 29 | 0.5085 | 0.375 | 0.4317 |
+
+`gold_*` tiers are hand-curated deterministic labels (easy = section how-to,
+medium = single-doc, hard = table/matrix incl. reverse lookups); rephrasing
+variants of those questions are in train, so they measure learned question
+types. `llm_held_out` is strict generalization on unseen questions.
+
+## Training data
+
+```json
+{
+  "train_groups": 1163,
+  "gold_train_groups": 27,
+  "test_groups": 38,
+  "gold_test_groups": 9,
+  "train_candidates": 27019,
+  "train_pass": 5510,
+  "train_fail": 21509,
+  "pass_ratio": 0.204,
+  "avg_candidates_per_train_group": 23.2,
+  "avg_pass_per_train_group": 4.7
+}
+```
+
+Trained 2026-07-06T19:32:57 · source: https://github.com/s3777091/recursive_models

best_model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:15f614479f8fcf432b38c2bbcc86463f363cfff0e2fc2c8bf9190ce20cdcb0cb
+size 7633333

eval_metrics.json

@@ -0,0 +1,308 @@
+{
+  "threshold": 0.884,
+  "micro_precision": 0.8281,
+  "micro_recall": 0.7294,
+  "micro_f1": 0.7756,
+  "macro_f1": 0.5659,
+  "exact_set_match": 0.1316,
+  "avg_selected": 5.05,
+  "avg_true_pass": 5.74,
+  "num_groups": 38,
+  "best_epoch": 13,
+  "history": [
+    {
+      "epoch": 1,
+      "loss": 129.936,
+      "lr": 0.003571333554490956,
+      "threshold": 0.805,
+      "micro_precision": 0.4937,
+      "micro_recall": 0.1789,
+      "micro_f1": 0.2626,
+      "macro_f1": 0.3452,
+      "exact_set_match": 0.1316,
+      "avg_selected": 2.08,
+      "avg_true_pass": 5.74,
+      "num_groups": 38
+    },
+    {
+      "epoch": 2,
+      "loss": 102.521,
+      "lr": 0.003571333554490956,
+      "threshold": 0.8,
+      "micro_precision": 0.5778,
+      "micro_recall": 0.2385,
+      "micro_f1": 0.3377,
+      "macro_f1": 0.4038,
+      "exact_set_match": 0.1053,
+      "avg_selected": 2.37,
+      "avg_true_pass": 5.74,
+      "num_groups": 38
+    },
+    {
+      "epoch": 3,
+      "loss": 100.13,
+      "lr": 0.003571333554490956,
+      "threshold": 0.776,
+      "micro_precision": 0.5373,
+      "micro_recall": 0.1651,
+      "micro_f1": 0.2526,
+      "macro_f1": 0.3143,
+      "exact_set_match": 0.1053,
+      "avg_selected": 1.76,
+      "avg_true_pass": 5.74,
+      "num_groups": 38
+    },
+    {
+      "epoch": 4,
+      "loss": 93.977,
+      "lr": 0.003571333554490956,
+      "threshold": 0.922,
+      "micro_precision": 0.5593,
+      "micro_recall": 0.3028,
+      "micro_f1": 0.3929,
+      "macro_f1": 0.4452,
+      "exact_set_match": 0.0526,
+      "avg_selected": 3.11,
+      "avg_true_pass": 5.74,
+      "num_groups": 38
+    },
+    {
+      "epoch": 5,
+      "loss": 92.528,
+      "lr": 0.003571333554490956,
+      "threshold": 0.838,
+      "micro_precision": 0.7445,
+      "micro_recall": 0.4679,
+      "micro_f1": 0.5746,
+      "macro_f1": 0.4234,
+      "exact_set_match": 0.1053,
+      "avg_selected": 3.61,
+      "avg_true_pass": 5.74,
+      "num_groups": 38
+    },
+    {
+      "epoch": 6,
+      "loss": 88.403,
+      "lr": 0.003571333554490956,
+      "threshold": 0.838,
+      "micro_precision": 0.7407,
+      "micro_recall": 0.6422,
+      "micro_f1": 0.688,
+      "macro_f1": 0.5556,
+      "exact_set_match": 0.1053,
+      "avg_selected": 4.97,
+      "avg_true_pass": 5.74,
+      "num_groups": 38
+    },
+    {
+      "epoch": 7,
+      "loss": 87.575,
+      "lr": 0.003571333554490956,
+      "threshold": 0.791,
+      "micro_precision": 0.7151,
+      "micro_recall": 0.6101,
+      "micro_f1": 0.6584,
+      "macro_f1": 0.5496,
+      "exact_set_match": 0.1316,
+      "avg_selected": 4.89,
+      "avg_true_pass": 5.74,
+      "num_groups": 38
+    },
+    {
+      "epoch": 8,
+      "loss": 86.008,
+      "lr": 0.003571333554490956,
+      "threshold": 0.792,
+      "micro_precision": 0.8362,
+      "micro_recall": 0.6789,
+      "micro_f1": 0.7494,
+      "macro_f1": 0.4991,
+      "exact_set_match": 0.1842,
+      "avg_selected": 4.66,
+      "avg_true_pass": 5.74,
+      "num_groups": 38
+    },
+    {
+      "epoch": 9,
+      "loss": 83.574,
+      "lr": 0.003571333554490956,
+      "threshold": 0.783,
+      "micro_precision": 0.7744,
+      "micro_recall": 0.6927,
+      "micro_f1": 0.7312,
+      "macro_f1": 0.5291,
+      "exact_set_match": 0.1316,
+      "avg_selected": 5.13,
+      "avg_true_pass": 5.74,
+      "num_groups": 38
+    },
+    {
+      "epoch": 10,
+      "loss": 80.694,
+      "lr": 0.003571333554490956,
+      "threshold": 0.844,
+      "micro_precision": 0.7882,
+      "micro_recall": 0.7339,
+      "micro_f1": 0.7601,
+      "macro_f1": 0.5412,
+      "exact_set_match": 0.1053,
+      "avg_selected": 5.34,
+      "avg_true_pass": 5.74,
+      "num_groups": 38
+    },
+    {
+      "epoch": 11,
+      "loss": 80.353,
+      "lr": 0.003571333554490956,
+      "threshold": 0.937,
+      "micro_precision": 0.783,
+      "micro_recall": 0.7615,
+      "micro_f1": 0.7721,
+      "macro_f1": 0.6148,
+      "exact_set_match": 0.1842,
+      "avg_selected": 5.58,
+      "avg_true_pass": 5.74,
+      "num_groups": 38
+    },
+    {
+      "epoch": 12,
+      "loss": 79.143,
+      "lr": 0.003571333554490956,
+      "threshold": 0.929,
+      "micro_precision": 0.805,
+      "micro_recall": 0.7385,
+      "micro_f1": 0.7703,
+      "macro_f1": 0.56,
+      "exact_set_match": 0.1316,
+      "avg_selected": 5.26,
+      "avg_true_pass": 5.74,
+      "num_groups": 38
+    },
+    {
+      "epoch": 13,
+      "loss": 76.237,
+      "lr": 0.003571333554490956,
+      "threshold": 0.884,
+      "micro_precision": 0.8281,
+      "micro_recall": 0.7294,
+      "micro_f1": 0.7756,
+      "macro_f1": 0.5659,
+      "exact_set_match": 0.1316,
+      "avg_selected": 5.05,
+      "avg_true_pass": 5.74,
+      "num_groups": 38
+    },
+    {
+      "epoch": 14,
+      "loss": 73.207,
+      "lr": 0.003571333554490956,
+      "threshold": 0.826,
+      "micro_precision": 0.8427,
+      "micro_recall": 0.6881,
+      "micro_f1": 0.7576,
+      "macro_f1": 0.5396,
+      "exact_set_match": 0.1579,
+      "avg_selected": 4.68,
+      "avg_true_pass": 5.74,
+      "num_groups": 38
+    },
+    {
+      "epoch": 15,
+      "loss": 73.388,
+      "lr": 0.003571333554490956,
+      "threshold": 0.87,
+      "micro_precision": 0.8289,
+      "micro_recall": 0.711,
+      "micro_f1": 0.7654,
+      "macro_f1": 0.5568,
+      "exact_set_match": 0.1579,
+      "avg_selected": 4.92,
+      "avg_true_pass": 5.74,
+      "num_groups": 38
+    },
+    {
+      "epoch": 16,
+      "loss": 71.875,
+      "lr": 0.003571333554490956,
+      "threshold": 0.945,
+      "micro_precision": 0.7626,
+      "micro_recall": 0.7661,
+      "micro_f1": 0.7643,
+      "macro_f1": 0.5735,
+      "exact_set_match": 0.1316,
+      "avg_selected": 5.76,
+      "avg_true_pass": 5.74,
+      "num_groups": 38
+    },
+    {
+      "epoch": 17,
+      "loss": 72.722,
+      "lr": 0.003571333554490956,
+      "threshold": 0.902,
+      "micro_precision": 0.805,
+      "micro_recall": 0.7385,
+      "micro_f1": 0.7703,
+      "macro_f1": 0.5696,
+      "exact_set_match": 0.1316,
+      "avg_selected": 5.26,
+      "avg_true_pass": 5.74,
+      "num_groups": 38
+    },
+    {
+      "epoch": 18,
+      "loss": 65.242,
+      "lr": 0.001785666777245478,
+      "threshold": 0.873,
+      "micro_precision": 0.8497,
+      "micro_recall": 0.6743,
+      "micro_f1": 0.7519,
+      "macro_f1": 0.5154,
+      "exact_set_match": 0.1316,
+      "avg_selected": 4.55,
+      "avg_true_pass": 5.74,
+      "num_groups": 38
+    },
+    {
+      "epoch": 19,
+      "loss": 63.143,
+      "lr": 0.001785666777245478,
+      "threshold": 0.916,
+      "micro_precision": 0.8424,
+      "micro_recall": 0.711,
+      "micro_f1": 0.7711,
+      "macro_f1": 0.5543,
+      "exact_set_match": 0.1842,
+      "avg_selected": 4.84,
+      "avg_true_pass": 5.74,
+      "num_groups": 38
+    },
+    {
+      "epoch": 20,
+      "loss": 61.132,
+      "lr": 0.001785666777245478,
+      "threshold": 0.903,
+      "micro_precision": 0.8182,
+      "micro_recall": 0.7018,
+      "micro_f1": 0.7556,
+      "macro_f1": 0.5291,
+      "exact_set_match": 0.1053,
+      "avg_selected": 4.92,
+      "avg_true_pass": 5.74,
+      "num_groups": 38
+    },
+    {
+      "epoch": 21,
+      "loss": 60.204,
+      "lr": 0.001785666777245478,
+      "threshold": 0.914,
+      "micro_precision": 0.8889,
+      "micro_recall": 0.6606,
+      "micro_f1": 0.7579,
+      "macro_f1": 0.526,
+      "exact_set_match": 0.1579,
+      "avg_selected": 4.26,
+      "avg_true_pass": 5.74,
+      "num_groups": 38
+    }
+  ]
+}

model_meta.json

@@ -0,0 +1,66 @@
+{
+  "model": "TRMSelector \u2014 recursive PASS/FAIL chunk gate (skip head)",
+  "params": 1905921,
+  "arch": {
+    "hidden": 256,
+    "H_cycles": 3,
+    "L_cycles": 4,
+    "head_dropout": 0.1
+  },
+  "encoder": "openai/text-embedding-ada-002",
+  "emb_dim": 1539,
+  "threshold": 0.884,
+  "lr": 0.003571,
+  "metrics": {
+    "threshold": 0.884,
+    "micro_precision": 0.8281,
+    "micro_recall": 0.7294,
+    "micro_f1": 0.7756,
+    "macro_f1": 0.5659,
+    "exact_set_match": 0.1316,
+    "avg_selected": 5.05,
+    "avg_true_pass": 5.74,
+    "num_groups": 38,
+    "best_epoch": 13
+  },
+  "bench": {
+    "gold_easy": {
+      "groups": 2,
+      "precision": 1.0,
+      "recall": 0.375,
+      "f1": 0.5455
+    },
+    "gold_medium": {
+      "groups": 2,
+      "precision": 0.975,
+      "recall": 0.975,
+      "f1": 0.975
+    },
+    "gold_hard": {
+      "groups": 5,
+      "precision": 0.9667,
+      "recall": 0.9667,
+      "f1": 0.9667
+    },
+    "llm_held_out": {
+      "groups": 29,
+      "precision": 0.5085,
+      "recall": 0.375,
+      "f1": 0.4317
+    }
+  },
+  "data": {
+    "train_groups": 1163,
+    "gold_train_groups": 27,
+    "test_groups": 38,
+    "gold_test_groups": 9,
+    "train_candidates": 27019,
+    "train_pass": 5510,
+    "train_fail": 21509,
+    "pass_ratio": 0.204,
+    "avg_candidates_per_train_group": 23.2,
+    "avg_pass_per_train_group": 4.7
+  },
+  "trained_at": "2026-07-06T19:32:57",
+  "source": "https://github.com/s3777091/recursive_models"
+}