dataset/data-normalization/check_duplicate_id.py

import json
from collections import Counter

with open("knowledge_base.json", "r", encoding="utf-8") as f:
    data = json.load(f)

ids = [item.get("id") for item in data]

counter = Counter(ids)
duplicates = {k: v for k, v in counter.items() if v > 1}

if duplicates:
    print("❌ Found duplicate IDs:")
    for k, v in duplicates.items():
        print(f"  ID '{k}' appears {v} times")
else:
    print("✅ No duplicate IDs found")