Update app.py

app.py

@@ -112,6 +112,7 @@ def add_new_eval(
 
     with open(f"scored/{organization}_{model_name}.jsonl", "w") as scored_file:
         with open(file_path, 'r') as f:
+            submitted_ids = set()
             for ix, line in enumerate(f):
                 try:
                     task = json.loads(line)
@@ -141,12 +142,17 @@ def add_new_eval(
                 )
 
                 all_scores.append({"score": score, "has_ans": has_ans, "model_answer": answer, 'id': task_id})
-
+                submitted_ids.add(task["id"])
                 scores += score
                 num_questions += 1
                 difficulty_scores[difficulty] += score
                 difficulty_counts[difficulty] += 1
 
+    # Check if all gold answer IDs are present in the submission
+    missing_ids = set(gold_answers["test"].keys()) - submitted_ids
+    if missing_ids:
+        return format_error(f"Submission is missing the following IDs: {', '.join(missing_ids)}")
+
     accuracy_easy = difficulty_scores["Easy"] / difficulty_counts["Easy"] if difficulty_counts["Easy"] > 0 else 0
     accuracy_medium = difficulty_scores["Medium"] / difficulty_counts["Medium"] if difficulty_counts["Medium"] > 0 else 0
     accuracy_hard = difficulty_scores["Hard"] / difficulty_counts["Hard"] if difficulty_counts["Hard"] > 0 else 0