Fix reranker/QA message order, add temperature=0, full QA prompt

data/filter_llama3.3-70B-Instruct.json

@@ -0,0 +1,84 @@
+{
+  "prog": {
+    "lm": null,
+    "traces": [],
+    "train": [],
+    "demos": [
+      {
+        "augmented": true,
+        "question": "Are Imperial River (Florida) and Amaradia (Dolj) both located in the same country?",
+        "fact_before_filter": "{\"fact\": [[\"imperial river\", \"is located in\", \"florida\"], [\"imperial river\", \"is a river in\", \"united states\"], [\"imperial river\", \"may refer to\", \"south america\"], [\"amaradia\", \"flows through\", \"ro ia de amaradia\"], [\"imperial river\", \"may refer to\", \"united states\"]]}",
+        "fact_after_filter": "{\"fact\":[[\"imperial river\",\"is located in\",\"florida\"],[\"imperial river\",\"is a river in\",\"united states\"],[\"amaradia\",\"flows through\",\"ro ia de amaradia\"]]}"
+      },
+      {
+        "augmented": true,
+        "question": "When is the director of film The Ancestor 's birthday?",
+        "fact_before_filter": "{\"fact\": [[\"jean jacques annaud\", \"born on\", \"1 october 1943\"], [\"tsui hark\", \"born on\", \"15 february 1950\"], [\"pablo trapero\", \"born on\", \"4 october 1971\"], [\"the ancestor\", \"directed by\", \"guido brignone\"], [\"benh zeitlin\", \"born on\", \"october 14  1982\"]]}",
+        "fact_after_filter": "{\"fact\":[[\"the ancestor\",\"directed by\",\"guido brignone\"]]}"
+      },
+      {
+        "augmented": true,
+        "question": "In what geographic region is the country where Teafuone is located?",
+        "fact_before_filter": "{\"fact\": [[\"teafuaniua\", \"is on the\", \"east\"], [\"motuloa\", \"lies between\", \"teafuaniua\"], [\"motuloa\", \"lies between\", \"teafuanonu\"], [\"teafuone\", \"is\", \"islet\"], [\"teafuone\", \"located in\", \"nukufetau\"]]}",
+        "fact_after_filter": "{\"fact\":[[\"teafuone\",\"is\",\"islet\"],[\"teafuone\",\"located in\",\"nukufetau\"]]}"
+      },
+      {
+        "augmented": true,
+        "question": "When did the director of film S.O.B. (Film) die?",
+        "fact_before_filter": "{\"fact\": [[\"allan dwan\", \"died on\", \"28 december 1981\"], [\"s o b\", \"written and directed by\", \"blake edwards\"], [\"robert aldrich\", \"died on\", \"december 5  1983\"], [\"robert siodmak\", \"died on\", \"10 march 1973\"], [\"bernardo bertolucci\", \"died on\", \"26 november 2018\"]]}",
+        "fact_after_filter": "{\"fact\":[[\"s o b\",\"written and directed by\",\"blake edwards\"]]}"
+      },
+      {
+        "augmented": true,
+        "question": "Do both films: Gloria (1980 Film) and A New Life (Film) have the directors from the same country?",
+        "fact_before_filter": "{\"fact\": [[\"sebasti n lelio watt\", \"received acclaim for directing\", \"gloria\"], [\"gloria\", \"is\", \"1980 american thriller crime drama film\"], [\"a brand new life\", \"is directed by\", \"ounie lecomte\"], [\"gloria\", \"written and directed by\", \"john cassavetes\"], [\"a new life\", \"directed by\", \"alan alda\"]]}",
+        "fact_after_filter": "{\"fact\":[[\"gloria\",\"is\",\"1980 american thriller crime drama film\"],[\"gloria\",\"written and directed by\",\"john cassavetes\"],[\"a new life\",\"directed by\",\"alan alda\"]]}"
+      },
+      {
+        "augmented": true,
+        "question": "What is the date of death of the director of film The Old Guard (1960 Film)?",
+        "fact_before_filter": "{\"fact\": [[\"the old guard\", \"is\", \"1960 french comedy film\"], [\"gilles grangier\", \"directed\", \"the old guard\"], [\"the old guard\", \"directed by\", \"gilles grangier\"], [\"the old fritz\", \"directed by\", \"gerhard lamprecht\"], [\"oswald albert mitchell\", \"directed\", \"old mother riley series of films\"]]}",
+        "fact_after_filter": "{\"fact\":[[\"the old guard\",\"is\",\"1960 french comedy film\"],[\"gilles grangier\",\"directed\",\"the old guard\"],[\"the old guard\",\"directed by\",\"gilles grangier\"]]}"
+      },
+      {
+        "augmented": true,
+        "question": "When is the composer of film Aulad (1968 Film) 's birthday?",
+        "fact_before_filter": "{\"fact\": [[\"aulad\", \"has music composed by\", \"chitragupta shrivastava\"], [\"aadmi sadak ka\", \"has music by\", \"ravi\"], [\"ravi shankar sharma\", \"composed music for\", \"hindi films\"], [\"gulzar\", \"was born on\", \"18 august 1934\"], [\"aulad\", \"is a\", \"1968 hindi language drama film\"]]}",
+        "fact_after_filter": "{\"fact\":[[\"aulad\",\"has music composed by\",\"chitragupta shrivastava\"],[\"aulad\",\"is a\",\"1968 hindi language drama film\"]]}"
+      },
+      {
+        "question": "How many households were in the city where Angelical Tears located?",
+        "fact_before_filter": "{\"fact\": [[\"dow city\", \"had\", \"219 households\"], [\"tucson\", \"had\", \"229 762 households\"], [\"atlantic city\", \"has\", \"15 504 households\"], [\"angelical tears\", \"located in\", \"oklahoma city\"], [\"atlantic city\", \"had\", \"15 848 households\"]]}",
+        "fact_after_filter": "{\"fact\": [[\"angelical tears\", \"located in\", \"oklahoma city\"]]}"
+      },
+      {
+        "question": "Did the movies In The Pope'S Eye and Virgin Mountain, originate from the same country?",
+        "fact_before_filter": "{\"fact\": [[\"virgin mountain\", \"released in\", \"icelandic cinemas\"], [\"virgin mountain\", \"directed by\", \"dagur k ri\"], [\"virgin mountain\", \"icelandic title is\", \"f si\"], [\"virgin mountain\", \"won\", \"2015 nordic council film prize\"], [\"virgin mountain\", \"is a\", \"2015 icelandic drama film\"]]}",
+        "fact_after_filter": "{\"fact\": [[\"virgin mountain\", \"released in\", \"icelandic cinemas\"], [\"virgin mountain\", \"directed by\", \"dagur k ri\"], [\"virgin mountain\", \"icelandic title is\", \"f si\"], [\"virgin mountain\", \"won\", \"2015 nordic council film prize\"], [\"virgin mountain\", \"is a\", \"2015 icelandic drama film\"]]}"
+      },
+      {
+        "question": "Which film has the director who died earlier, The Virtuous Model or Bulldog Drummond'S Peril?",
+        "fact_before_filter": "{\"fact\": [[\"the virtuous model\", \"is\", \"1919 american silent drama film\"], [\"bulldog drummond s peril\", \"directed by\", \"james p  hogan\"], [\"the virtuous model\", \"directed by\", \"albert capellani\"], [\"bulldog drummond s revenge\", \"directed by\", \"louis king\"], [\"bulldog drummond s peril\", \"is\", \"american film\"]]}",
+        "fact_after_filter": "{\"fact\": [[\"the virtuous model\", \"is\", \"1919 american silent drama film\"], [\"bulldog drummond s peril\", \"directed by\", \"james p  hogan\"], [\"the virtuous model\", \"directed by\", \"albert capellani\"], [\"bulldog drummond s peril\", \"is\", \"american film\"]]}"
+      }
+    ],
+    "signature": {
+      "instructions": "You are a critical component of a high-stakes question-answering system used by top researchers and decision-makers worldwide. Your task is to filter facts based on their relevance to a given query, ensuring that the most crucial information is presented to these stakeholders. The query requires careful analysis and possibly multi-hop reasoning to connect different pieces of information. You must select up to 4 relevant facts from the provided candidate list that have a strong connection to the query, aiding in reasoning and providing an accurate answer. The output should be in JSON format, e.g., {\"fact\": [[\"s1\", \"p1\", \"o1\"], [\"s2\", \"p2\", \"o2\"]]}, and if no facts are relevant, return an empty list, {\"fact\": []}. The accuracy of your response is paramount, as it will directly impact the decisions made by these high-level stakeholders. You must only use facts from the candidate list and not generate new facts. The future of critical decision-making relies on your ability to accurately filter and present relevant information.",
+      "fields": [
+        {
+          "prefix": "Question:",
+          "description": "Query for retrieval"
+        },
+        {
+          "prefix": "Fact Before Filter:",
+          "description": "Candidate facts to be filtered"
+        },
+        {
+          "prefix": "Fact After Filter:",
+          "description": "Filtered facts in JSON format"
+        }
+      ]
+    },
+    "system": "Your input fields are:\n1. `question` (str): Query for retrieval\n2. `fact_before_filter` (str): Candidate facts to be filtered\n\nYour output fields are:\n1. `fact_after_filter` (Fact): Filtered facts in JSON format\n\nAll interactions will be structured in the following way, with the appropriate values filled in.\n\n[[ ## question ## ]]\n{question}\n\n[[ ## fact_before_filter ## ]]\n{fact_before_filter}\n\n[[ ## fact_after_filter ## ]]\n{fact_after_filter}        # note: the value you produce must be pareseable according to the following JSON schema: {\"type\": \"object\", \"properties\": {\"fact\": {\"type\": \"array\", \"description\": \"A list of facts, each fact is a list of 3 strings: [subject, predicate, object]\", \"items\": {\"type\": \"array\", \"items\": {\"type\": \"string\"}}, \"title\": \"Fact\"}}, \"required\": [\"fact\"], \"title\": \"Fact\"}\n\n[[ ## completed ## ]]\n\nIn adhering to this structure, your objective is: \n        You are a critical component of a high-stakes question-answering system used by top researchers and decision-makers worldwide. Your task is to filter facts based on their relevance to a given query, ensuring that the most crucial information is presented to these stakeholders. The query requires careful analysis and possibly multi-hop reasoning to connect different pieces of information. You must select up to 4 relevant facts from the provided candidate list that have a strong connection to the query, aiding in reasoning and providing an accurate answer. The output should be in JSON format, e.g., {\"fact\": [[\"s1\", \"p1\", \"o1\"], [\"s2\", \"p2\", \"o2\"]]}, and if no facts are relevant, return an empty list, {\"fact\": []}. The accuracy of your response is paramount, as it will directly impact the decisions made by these high-level stakeholders. You must only use facts from the candidate list and not generate new facts. The future of critical decision-making relies on your ability to accurately filter and present relevant information."
+  }
+}

src/passage_entity/benchmark_runner.py

@@ -108,6 +108,8 @@ async def _openai_complete(model, prompt, system_prompt=None, history_messages=[
     client = _get_client(base_url, api_key)
     kwargs.pop("hashing_kv", None)
     kwargs.pop("keyword_extraction", None)
+    kwargs.setdefault("temperature", 0.0)
+    kwargs.setdefault("seed", 0)
     messages = []
     if system_prompt:
         messages.append({"role": "system", "content": system_prompt})
@@ -254,27 +256,15 @@ def run_qa(
     for qs in queries:
         passages = qs.docs[:qa_top_k]
         msgs = make_qa_messages(passages, qs.question)
-        # Convert messages to single call
-        system_prompt = None
-        history = []
-        user_prompt = ""
-        for msg in msgs:
-            if msg["role"] == "system":
-                system_prompt = msg["content"]
-            elif msg["role"] == "assistant":
-                history.append(msg)
-            elif msg["role"] == "user":
-                if user_prompt:
-                    history.append({"role": "user", "content": user_prompt})
-                user_prompt = msg["content"]
-
+        # Pass messages directly — preserves correct user/assistant order.
+        # The old decompose/recompose loop put assistant before user in demos.
         try:
             response = _run_sync(
                 llm_func(
-                    prompt=user_prompt,
-                    system_prompt=system_prompt,
-                    history_messages=history,
-                    max_tokens=512,
+                    prompt=msgs[-1]["content"],
+                    system_prompt=msgs[0]["content"] if msgs[0]["role"] == "system" else None,
+                    history_messages=msgs[1:-1],
+                    # No max_tokens cap — let the model use its full context
                 )
             )
             # Extract answer

src/passage_entity/prompts.py

@@ -231,24 +231,26 @@ RAG_QA_SYSTEM = (
     "additional elaborations."
 )
 
+# One-shot demo — 5 passages (including 3 distractors) with full-length text.
+# 5 passages (including 3 distractors), full-length text, \n separators.
 _RAG_QA_ONE_SHOT_DOCS = (
     "Wikipedia Title: The Last Horse\n"
-    "The Last Horse (Spanish:El último caballo) is a 1950 Spanish comedy film directed "
-    "by Edgar Neville starring Fernando Fernán Gómez.\n\n"
+    "The Last Horse (Spanish:El último caballo) is a 1950 Spanish comedy film directed by Edgar Neville starring Fernando Fernán Gómez.\n"
     "Wikipedia Title: Southampton\n"
-    "The University of Southampton, which was founded in 1862 and received its Royal "
-    "Charter as a university in 1952, has over 22,000 students. The university is ranked "
-    "in the top 100 research universities in the world in the Academic Ranking of World "
-    "Universities 2010.\n\n"
+    "The University of Southampton, which was founded in 1862 and received its Royal Charter as a university in 1952, has over 22,000 students. The university is ranked in the top 100 research universities in the world in the Academic Ranking of World Universities 2010. In 2010, the THES - QS World University Rankings positioned the University of Southampton in the top 80 universities in the world. The university considers itself one of the top 5 research universities in the UK. The university has a global reputation for research into engineering sciences, oceanography, chemistry, cancer sciences, sound and vibration research, computer science and electronics, optoelectronics and textile conservation at the Textile Conservation Centre (which is due to close in October 2009.) It is also home to the National Oceanography Centre, Southampton (NOCS), the focus of Natural Environment Research Council-funded marine research.\n"
+    "Wikipedia Title: Stanton Township, Champaign County, Illinois\n"
+    "Stanton Township is a township in Champaign County, Illinois, USA. As of the 2010 census, its population was 505 and it contained 202 housing units.\n"
     "Wikipedia Title: Neville A. Stanton\n"
-    "Neville A. Stanton is a British Professor of Human Factors and Ergonomics at the "
-    "University of Southampton. Prof Stanton is a Chartered Engineer (C.Eng), Chartered "
-    "Psychologist (C.Psychol) and Chartered Ergonomist (C.ErgHF).\n"
+    "Neville A. Stanton is a British Professor of Human Factors and Ergonomics at the University of Southampton. Prof Stanton is a Chartered Engineer (C.Eng), Chartered Psychologist (C.Psychol) and Chartered Ergonomist (C.ErgHF). He has written and edited over a forty books and over three hundered peer-reviewed journal papers on applications of the subject. Stanton is a Fellow of the British Psychological Society, a Fellow of The Institute of Ergonomics and Human Factors and a member of the Institution of Engineering and Technology. He has been published in academic journals including \"Nature\". He has also helped organisations design new human-machine interfaces, such as the Adaptive Cruise Control system for Jaguar Cars.\n"
+    "Wikipedia Title: Finding Nemo\n"
+    "Finding Nemo Theatrical release poster Directed by Andrew Stanton Produced by Graham Walters Screenplay by Andrew Stanton Bob Peterson David Reynolds Story by Andrew Stanton Starring Albert Brooks Ellen DeGeneres Alexander Gould Willem Dafoe Music by Thomas Newman Cinematography Sharon Calahan Jeremy Lasky Edited by David Ian Salter Production company Walt Disney Pictures Pixar Animation Studios Distributed by Buena Vista Pictures Distribution Release date May 30, 2003 (2003 - 05 - 30) Running time 100 minutes Country United States Language English Budget $$94 million Box office $$940.3 million"
 )
 
 RAG_QA_ONE_SHOT_INPUT = (
-    f"{_RAG_QA_ONE_SHOT_DOCS}\n\n"
-    "Question: When was Neville A. Stanton's employer founded?\nThought: "
+    f"{_RAG_QA_ONE_SHOT_DOCS}"
+    "\n\nQuestion: "
+    "When was Neville A. Stanton's employer founded?"
+    "\nThought: "
 )
 
 RAG_QA_ONE_SHOT_OUTPUT = (

src/passage_entity/reranker.py

@@ -117,24 +117,13 @@ class FactReranker:
 
     # ------------------------------------------------------------------
     def _call_llm(self, messages: List[Dict[str, str]]) -> str:
-        system_prompt = None
-        history = []
-        user_prompt = ""
-        for msg in messages:
-            if msg["role"] == "system":
-                system_prompt = msg["content"]
-            elif msg["role"] == "assistant":
-                history.append(msg)
-            elif msg["role"] == "user":
-                if user_prompt:
-                    history.append({"role": "user", "content": user_prompt})
-                user_prompt = msg["content"]
-
+        # Pass messages directly to OpenAI — the old decompose/recompose
+        # loop scrambled demo order (assistant before user in each pair).
         return _run_sync(
             self.llm_func(
-                prompt=user_prompt,
-                system_prompt=system_prompt,
-                history_messages=history,
+                prompt=messages[-1]["content"],
+                system_prompt=messages[0]["content"] if messages[0]["role"] == "system" else None,
+                history_messages=messages[1:-1],
                 max_tokens=512,
             )
         )