Upload folder using huggingface_hub

app/pipeline.py

@@ -70,8 +70,11 @@ def filter_phrases(filters: dict) -> list[str]:
 
 # The registry sweep can fill the first several dozen ranks with trials, so the screen has to
 # reach well past the displayed page or papers are cut before anyone judges them.
-SCREEN_LIMIT = 160
+SCREEN_LIMIT = 500
 SCREEN_BATCH = 40
+# A full screen is 13 calls; an unbounded burst risks a rate limit, which fails open to
+# unscreened results.
+SCREEN_CONCURRENCY = 8
 
 
 def recount(studies: list[dict], sesoi: float, effect_type: str) -> dict:
@@ -355,10 +358,16 @@ class SearchPipeline:
             }
             for d in head
         ]
+        slots = asyncio.Semaphore(SCREEN_CONCURRENCY)
+
+        async def screen_batch(batch: list[dict]) -> set[str]:
+            async with slots:
+                return await self.llm.screen(question, batch, usage)
+
         try:
             batches = await asyncio.gather(
                 *(
-                    self.llm.screen(question, rows[start : start + SCREEN_BATCH], usage)
+                    screen_batch(rows[start : start + SCREEN_BATCH])
                     for start in range(0, len(rows), SCREEN_BATCH)
                 )
             )

app/repository.py

@@ -519,7 +519,12 @@ def authority_query(query: dict) -> dict:
     }
 
 
-def rrf_fuse(*rankings: list[dict], limit: int = 200) -> list[dict]:
+# How many ranked records one search carries forward. It has to cover the relevance screen
+# (pipeline.SCREEN_LIMIT): a shorter list would silently cap what can be screened.
+RETRIEVE_LIMIT = 500
+
+
+def rrf_fuse(*rankings: list[dict], limit: int = RETRIEVE_LIMIT) -> list[dict]:
     scores: dict[str, float] = {}
     docs = {}
     for ranking in rankings:
@@ -862,7 +867,7 @@ class ElasticRepository:
         return {"documents": self.hits(result), "corpus": total["count"]}
 
     async def retrieve(self, query: dict, vector: list[float] | None) -> tuple[list[dict], str]:
-        base = {"index": self.index, "size": 200, "source_excludes": ["embedding"]}
+        base = {"index": self.index, "size": RETRIEVE_LIMIT, "source_excludes": ["embedding"]}
         # Third ranking signal: citation authority over the same match set.
         authority = authority_query(query)
         if vector is None:
@@ -874,8 +879,8 @@ class ElasticRepository:
         knn = {
             "field": "embedding",
             "query_vector": vector,
-            "k": 200,
-            "num_candidates": 1000,
+            "k": RETRIEVE_LIMIT,
+            "num_candidates": 2 * RETRIEVE_LIMIT,
             "filter": query,
         }
         try:
@@ -883,7 +888,7 @@ class ElasticRepository:
                 **base,
                 retriever={
                     "rrf": {
-                        "rank_window_size": 200,
+                        "rank_window_size": RETRIEVE_LIMIT,
                         "rank_constant": 60,
                         "retrievers": [
                             {"standard": {"query": query}},