compare/records/151.json

{"qid": "151", "question": "What are the four major ethnic groups in Burma?", "gold_answers": ["Bamar", "Shan", "Karen", "Rakhine"], "configs_order": ["closed-book", "with-docs", "with-structures", "structure per q", "structure per ds", "DCI", "e2e v0", "e2e v2", "e2e v3"], "configs": {"closed-book": {"answer": "Explanation: Interpreting “four major ethnic groups” as the four largest ethnic communities traditionally listed for Burma/Myanmar, the standard set is Burman (Bamar), Shan, Karen (Kayin), and Rakhine (Arakanese). Some sources use slightly different classifications, but this is the most common four-group answer.\nExact Answer: Burman (Bamar); Shan; Karen (Kayin); Rakhine (Arakanese)\nConfidence: 78%", "pred_items": ["Burman (Bamar)", "Shan", "Karen (Kayin)", "Rakhine (Arakanese)"], "metrics": {"judge_score": 1.0, "correct": 1, "precision": 1.0, "recall": 1.0, "extracted_final_answer": null}, "model": "prod-gpt-54-reasoning", "mode": "closed-book", "system_prompt_file": "prompts/answer_list_aware_long_monaco_semicolon.md", "max_completion_tokens": 8192, "latency_ms": 34301.21, "judged_missing": false, "finish_reason": "stop", "usage": {"prompt_tokens": 420, "completion_tokens": 1718, "reasoning_tokens": 1612, "total_tokens": 2138}}, "with-docs": {"answer": "Explanation: The major groups listed by population are Bamar, Shan, Kayin/Karen, and Rakhine.\nExact Answer: Bamar; Shan; Karen; Rakhine\nConfidence: 90%", "pred_items": ["Bamar", "Shan", "Karen", "Rakhine"], "metrics": {"judge_score": 1.0, "correct": 1, "precision": 1.0, "recall": 1.0, "extracted_final_answer": null}, "model": "prod-gpt-54-reasoning", "mode": "with-docs", "system_prompt_file": "prompts/answer_list_aware_long_monaco_semicolon.md", "max_completion_tokens": 8192, "latency_ms": 7844.82, "judged_missing": false, "finish_reason": "stop", "usage": {"prompt_tokens": 6409, "completion_tokens": 571, "reasoning_tokens": 518, "total_tokens": 6980}}, "with-structures": {"answer": "Explanation: The four largest/major ethnic groups in Burma (Myanmar) are generally listed as Bamar, Shan, Kayin/Karen, and Rakhine.\nExact Answer: Bamar; Shan; Kayin; Rakhine\nConfidence: 88%", "pred_items": ["Bamar", "Shan", "Kayin", "Rakhine"], "metrics": {"judge_score": 1.0, "correct": 1, "precision": 1.0, "recall": 1.0, "extracted_final_answer": null}, "model": "prod-gpt-54-reasoning", "mode": "with-docs", "system_prompt_file": "prompts/answer_list_aware_long_monaco_semicolon.md", "max_completion_tokens": 8192, "latency_ms": 8239.01, "judged_missing": false, "finish_reason": "stop", "usage": {"prompt_tokens": 3895, "completion_tokens": 575, "reasoning_tokens": 512, "total_tokens": 4470}}, "structure per q": {"answer": "Explanation: Grounded in demographic percentages and alias files.\nExact Answer: Bamar (Burman); Shan; Kayin (Karen); Rakhine\nConfidence: 90%", "pred_items": ["Bamar (Burman)", "Shan", "Kayin (Karen)", "Rakhine"], "metrics": {"judge_score": 1.0, "correct": 1, "precision": 1.0, "recall": 1.0, "extracted_final_answer": null}, "model": "prod-gpt-54-reasoning", "mode": "agentic", "system_prompt_file": "aml_qa_jobs/agentic_answer/system_prompt.md", "max_completion_tokens": 8192, "latency_ms": 327727.68, "judged_missing": false, "stop_reason": "end_turn", "n_turns": 9, "max_turns": 50, "tool_call_counts": {"bash": 5, "read_file": 11}, "tokens": {"prompt_tokens": 56216, "completion_tokens": 2267, "total_tokens": 58483, "reasoning_tokens": 1512}, "finish_reasons": ["tool_calls", "tool_calls", "tool_calls", "tool_calls", "tool_calls", "tool_calls", "tool_calls", "tool_calls", "stop"], "timeout_retries": 0}, "structure per ds": {"answer": "Explanation: Grounded in the Myanmar ethnic group population and official-recognition files.\nExact Answer: Bamar; Kayin (Karen); Rakhine; Shan\nConfidence: 85%", "pred_items": ["Bamar", "Kayin (Karen)", "Rakhine", "Shan"], "metrics": {"judge_score": 1.0, "correct": 1, "precision": 1.0, "recall": 1.0, "extracted_final_answer": null}, "model": "prod-gpt-54-reasoning", "mode": "agentic", "system_prompt_file": "aml_qa_jobs/agentic_answer/system_prompt_corpus.md", "max_completion_tokens": 8192, "latency_ms": 273300.12, "judged_missing": false, "stop_reason": "end_turn", "n_turns": 7, "max_turns": 50, "tool_call_counts": {"bash": 4, "read_file": 6}, "tokens": {"prompt_tokens": 32352, "completion_tokens": 1458, "total_tokens": 33810, "reasoning_tokens": 792}, "finish_reasons": ["tool_calls", "tool_calls", "tool_calls", "tool_calls", "tool_calls", "tool_calls", "stop"], "timeout_retries": 0}, "DCI": {"answer": "Explanation: doc_002166.txt lists Myanmar/Burma’s main ethnic groups and population shares.\nExact Answer: Bamar; Shan; Kayin; Rakhine\nConfidence: 95%", "pred_items": ["Bamar", "Shan", "Kayin", "Rakhine"], "metrics": {"judge_score": 1.0, "correct": 1, "precision": 1.0, "recall": 1.0, "extracted_final_answer": null}, "model": "prod-gpt-54-reasoning", "mode": "agentic", "system_prompt_file": "aml_qa_jobs/agentic_answer/system_prompt_rawtext_v2_monaco.md", "max_completion_tokens": 8192, "latency_ms": 745183.47, "judged_missing": false, "stop_reason": "end_turn", "n_turns": 8, "max_turns": 50, "tool_call_counts": {"bash": 7}, "tokens": {"prompt_tokens": 91282, "completion_tokens": 1633, "total_tokens": 92915, "reasoning_tokens": 1144}, "finish_reasons": ["tool_calls", "tool_calls", "tool_calls", "tool_calls", "tool_calls", "tool_calls", "tool_calls", "stop"], "timeout_retries": 0}, "e2e v0": {"answer": "Explanation: Grounded in Myanmar ethnic group percentage tables showing the largest groups by population share.\nExact Answer: Bamar; Shan; Kayin; Rakhine\nConfidence: 90%", "pred_items": ["Bamar", "Shan", "Kayin", "Rakhine"], "metrics": {"judge_score": 1.0, "correct": 1, "precision": 1.0, "recall": 1.0, "extracted_final_answer": null}, "model": "prod-gpt-54-reasoning", "mode": "agentic", "system_prompt_file": "aml_qa_jobs/agentic_answer/system_prompt_e2e.md", "max_completion_tokens": 8192, "latency_ms": 1144244.49, "judged_missing": false, "stop_reason": "end_turn", "n_turns": 11, "max_turns": 50, "tool_call_counts": {"bash": 20, "read_file": 8}, "tokens": {"prompt_tokens": 182931, "completion_tokens": 4794, "total_tokens": 187725, "reasoning_tokens": 2881}, "finish_reasons": ["tool_calls", "tool_calls", "tool_calls", "tool_calls", "tool_calls", "tool_calls", "tool_calls", "tool_calls", "tool_calls", "tool_calls", "stop"], "timeout_retries": 0}, "e2e v2": {"answer": "Explanation: Grounded in the Myanmar ethnic group population shares CSV, which lists the largest groups by percentage.\nExact Answer: Bamar; Shan; Kayin; Rakhine\nConfidence: 95%", "pred_items": ["Bamar", "Shan", "Kayin", "Rakhine"], "metrics": {"judge_score": 1.0, "correct": 1, "precision": 1.0, "recall": 1.0, "extracted_final_answer": null}, "model": "prod-gpt-54-reasoning", "mode": "agentic", "system_prompt_file": "aml_qa_jobs/agentic_answer/system_prompt_e2e.md", "max_completion_tokens": 8192, "latency_ms": 2173167.03, "judged_missing": false, "stop_reason": "end_turn", "n_turns": 11, "max_turns": 50, "tool_call_counts": {"bash": 8, "read_file": 4}, "tokens": {"prompt_tokens": 187512, "completion_tokens": 2639, "total_tokens": 190151, "reasoning_tokens": 1386}, "finish_reasons": ["tool_calls", "tool_calls", "tool_calls", "tool_calls", "tool_calls", "tool_calls", "tool_calls", "tool_calls", "tool_calls", "tool_calls", "stop"], "timeout_retries": 1}, "e2e v3": {"answer": "Explanation: The Myanmar ethnic population-share tables list the top groups as Bamar/Burman, Shan, Kayin/Karen, and Rakhine.\nExact Answer: Burman; Shan; Karen; Rakhine\nConfidence: 90%", "pred_items": ["Burman", "Shan", "Karen", "Rakhine"], "metrics": {"judge_score": 1.0, "correct": 1, "precision": 1.0, "recall": 1.0, "extracted_final_answer": null}, "model": "prod-gpt-54-reasoning", "mode": "agentic", "system_prompt_file": "aml_qa_jobs/agentic_answer/system_prompt_e2e.md", "max_completion_tokens": 8192, "latency_ms": 812153.06, "judged_missing": false, "stop_reason": "end_turn", "n_turns": 7, "max_turns": 50, "tool_call_counts": {"bash": 7, "read_file": 7}, "tokens": {"prompt_tokens": 84688, "completion_tokens": 3010, "total_tokens": 87698, "reasoning_tokens": 1645}, "finish_reasons": ["tool_calls", "tool_calls", "tool_calls", "tool_calls", "tool_calls", "tool_calls", "stop"], "timeout_retries": 0}}}