clearvoice-api / scripts /sample_scifact.py
manan77709's picture
add SciFact benchmark sampling, claims file and docs
8838306
Raw History Blame Contribute Delete
5.25 kB
import json
import os
import random
from typing import Optional
def map_claim_labels(labels: set[str]) -> Optional[tuple[str, str]]:
cleaned = {str(lbl).strip().upper() for lbl in labels if str(lbl).strip()}
if "SUPPORT" in cleaned and "CONTRADICT" in cleaned:
return None
if "SUPPORT" in cleaned:
return ("TRUE", "true")
if "CONTRADICT" in cleaned:
return ("FALSE", "false")
if len(cleaned) == 0:
return ("INSUFFICIENT EVIDENCE", "insufficient")
return None
def filter_scifact_records(records: list[dict]) -> tuple[dict[str, list[dict]], dict[str, int]]:
claims_by_id: dict[object, dict] = {}
for r in records:
cid = r.get("id")
claim_text = str(r.get("claim") or "").strip()
ev_label = str(r.get("evidence_label") or "").strip().upper()
if cid not in claims_by_id:
claims_by_id[cid] = {
"id": cid,
"claim": claim_text,
"labels": set(),
"cited_doc_ids": r.get("cited_doc_ids"),
}
if ev_label:
claims_by_id[cid]["labels"].add(ev_label)
by_class: dict[str, list[dict]] = {
"true": [],
"false": [],
"insufficient": [],
}
stats = {
"total_claims": len(claims_by_id),
"kept": 0,
"dropped_len_ge_200": 0,
"dropped_empty": 0,
"dropped_conflicting": 0,
"dropped_duplicate": 0,
}
seen_claims: set[str] = set()
for item in claims_by_id.values():
claim = item["claim"]
if not claim:
stats["dropped_empty"] += 1
continue
if len(claim) >= 200:
stats["dropped_len_ge_200"] += 1
continue
claim_key = claim.lower()
if claim_key in seen_claims:
stats["dropped_duplicate"] += 1
continue
seen_claims.add(claim_key)
mapped = map_claim_labels(item["labels"])
if mapped is None:
stats["dropped_conflicting"] += 1
continue
expected, category = mapped
by_class[category].append({
"claim": claim,
"expected": expected,
"category": category,
"source": "SciFact",
"scifact_id": item["id"],
})
stats["kept"] += 1
return by_class, stats
def sample_stratified_scifact(
by_class: dict[str, list[dict]],
n_per_class: int = 20,
seed: int = 42,
) -> tuple[list[dict], dict[str, int]]:
rng = random.Random(seed)
sampled: list[dict] = []
class_counts: dict[str, int] = {}
for category in ["true", "false", "insufficient"]:
pool = by_class.get(category, [])
count = min(len(pool), n_per_class)
class_counts[category] = count
if count < n_per_class:
print(f"Warning: category '{category}' only has {count} claims (requested {n_per_class})")
chosen = rng.sample(pool, count) if count < len(pool) else list(pool)
sampled.extend(chosen)
rng.shuffle(sampled)
for idx, item in enumerate(sampled, start=1):
item["id"] = idx
return sampled, class_counts
def load_scifact_validation_records() -> tuple[list[dict], str]:
try:
import datasets
ds = datasets.load_dataset("allenai/scifact", "claims", split="validation")
return [dict(r) for r in ds], "datasets library (script)"
except Exception as e_script:
print(f"Direct script load did not work ({e_script}). Trying parquet conversion branch...")
try:
import pandas as pd
parquet_url = (
"https://huggingface.co/datasets/allenai/scifact/resolve/refs%2Fconvert%2Fparquet/claims/validation/0000.parquet"
)
df = pd.read_parquet(parquet_url)
route_desc = "parquet conversion branch (refs/convert/parquet/claims/validation/0000.parquet)"
return df.to_dict(orient="records"), route_desc
except Exception as e_parquet:
print(f"Parquet conversion branch failed: {e_parquet}")
raise RuntimeError("Failed to load SciFact validation split via both script and parquet routes.")
def main() -> None:
records, route = load_scifact_validation_records()
print(f"SciFact loaded successfully via: {route}")
print(f"Loaded {len(records)} raw rows.")
by_class, stats = filter_scifact_records(records)
print("\nDropped claim statistics:")
for k, v in stats.items():
print(f" {k}: {v}")
print("\nEligible unique claims per class before sampling:")
for cat, items in by_class.items():
print(f" {cat}: {len(items)}")
sampled, sampled_counts = sample_stratified_scifact(by_class, n_per_class=20, seed=42)
print(f"\nSampled claims per class (total {len(sampled)}):")
for cat, cnt in sampled_counts.items():
print(f" {cat}: {cnt}")
root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
output_path = os.path.join(root_dir, "benchmark_scifact.json")
with open(output_path, "w", encoding="utf-8") as f:
json.dump(sampled, f, indent=2, ensure_ascii=False)
print(f"\nWrote {len(sampled)} claims to {output_path}")
if __name__ == "__main__":
main()