sync: from ael backend/space/

rerank.py

@@ -69,10 +69,37 @@ ENABLED = os.environ.get("AEL_RERANK", "1") not in ("0", "false", "no")
 # still scored correctly by it far more often than not.
 MODEL = os.environ.get("AEL_RERANK_MODEL", "english")
 
-# Drop a result when the model thinks it is more likely NOT to answer the query
-# than to answer it. Deliberately conservative: 0.5 is the neutral point of a
-# calibrated probability, so this only removes what the model actively rejects.
-DROP_BELOW = float(os.environ.get("AEL_RERANK_DROP", "0.5"))
+# Drop a result the model thinks this unlikely to answer the query.
+#
+# This started at 0.5, on the reasonable-sounding argument that 0.5 is the
+# neutral point of a calibrated probability. That argument is wrong for this
+# model, and the measured score distribution says so. Nine queries against
+# production on 2026-10-02 (`?racerDetails=1`):
+#
+#   genuinely answers the query   0.927 "Homepage - TLD"          (what is .tld)
+#                                 0.838 "Lima"                    (capital of Peru)
+#                                 0.692 "How Does a Heat Pump Work"
+#                                 0.676 "Current World Population"
+#                                 0.522 "The Best Donuts In Seattle"
+#   related topic, no answer      0.329 "Best Donuts San Francisco"
+#                                 0.321 "Will a Heat Pump Work in an Old House?"
+#   unrelated                     0.098 "peruse | Etymology of peruse"
+#                                 0.042 "Earliest Mysticete ... Peru"
+#                                 0.004 "Donald Trump - Wikipedia"
+#
+# The separation is between roughly 0.4 and 0.15, not at 0.5 — so 0.5 was
+# throwing away real answers (the 0.4 band) while the junk it was meant to catch
+# sits far below it anyway. 0.35 keeps the 0.4 band and still removes everything
+# under 0.15.
+#
+# What a threshold cannot fix, and did not try to: the model scores news and
+# retail homepages at 0.65-0.80 for questions they cannot answer ("population of
+# Ascension Island" -> "Current World Population", 0.676). The instructions
+# below already tell it to answer no for homepages and category listings and it
+# does not always. That is a model limitation, recorded rather than papered over
+# with a hand-written heuristic — and the honest response to it is better
+# engines, which is the race's job, not this stage's.
+DROP_BELOW = float(os.environ.get("AEL_RERANK_DROP", "0.35"))
 
 # Never return an empty page. See the starvation note above — this is a floor of
 # one, not a target: a short page is fine, an empty one is not.