utilities/compare_question_set.py

import json

with open("gaia_questions.json") as f:
    initial_questions = json.load(f)

with open("new_gaia_questions.json") as f:
    new_questions = json.load(f)

# Compare question IDs
initial_ids = {q["task_id"] for q in initial_questions}
new_ids = {q["task_id"] for q in new_questions}

added_questions = new_ids - initial_ids
removed_questions = initial_ids - new_ids

print(f"Added Questions: {added_questions}")
print(f"Removed Questions: {removed_questions}")

if not added_questions and not removed_questions:
    print("✅ The question set has remained the same.")
else:
    print("⚠️ The question set has changed.")