File size: 5,254 Bytes
8838306
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
import json
import os
import random
from typing import Optional


def map_claim_labels(labels: set[str]) -> Optional[tuple[str, str]]:
    cleaned = {str(lbl).strip().upper() for lbl in labels if str(lbl).strip()}
    if "SUPPORT" in cleaned and "CONTRADICT" in cleaned:
        return None
    if "SUPPORT" in cleaned:
        return ("TRUE", "true")
    if "CONTRADICT" in cleaned:
        return ("FALSE", "false")
    if len(cleaned) == 0:
        return ("INSUFFICIENT EVIDENCE", "insufficient")
    return None


def filter_scifact_records(records: list[dict]) -> tuple[dict[str, list[dict]], dict[str, int]]:
    claims_by_id: dict[object, dict] = {}
    for r in records:
        cid = r.get("id")
        claim_text = str(r.get("claim") or "").strip()
        ev_label = str(r.get("evidence_label") or "").strip().upper()

        if cid not in claims_by_id:
            claims_by_id[cid] = {
                "id": cid,
                "claim": claim_text,
                "labels": set(),
                "cited_doc_ids": r.get("cited_doc_ids"),
            }
        if ev_label:
            claims_by_id[cid]["labels"].add(ev_label)

    by_class: dict[str, list[dict]] = {
        "true": [],
        "false": [],
        "insufficient": [],
    }
    stats = {
        "total_claims": len(claims_by_id),
        "kept": 0,
        "dropped_len_ge_200": 0,
        "dropped_empty": 0,
        "dropped_conflicting": 0,
        "dropped_duplicate": 0,
    }
    seen_claims: set[str] = set()

    for item in claims_by_id.values():
        claim = item["claim"]
        if not claim:
            stats["dropped_empty"] += 1
            continue

        if len(claim) >= 200:
            stats["dropped_len_ge_200"] += 1
            continue

        claim_key = claim.lower()
        if claim_key in seen_claims:
            stats["dropped_duplicate"] += 1
            continue
        seen_claims.add(claim_key)

        mapped = map_claim_labels(item["labels"])
        if mapped is None:
            stats["dropped_conflicting"] += 1
            continue

        expected, category = mapped
        by_class[category].append({
            "claim": claim,
            "expected": expected,
            "category": category,
            "source": "SciFact",
            "scifact_id": item["id"],
        })
        stats["kept"] += 1

    return by_class, stats


def sample_stratified_scifact(
    by_class: dict[str, list[dict]],
    n_per_class: int = 20,
    seed: int = 42,
) -> tuple[list[dict], dict[str, int]]:
    rng = random.Random(seed)
    sampled: list[dict] = []
    class_counts: dict[str, int] = {}

    for category in ["true", "false", "insufficient"]:
        pool = by_class.get(category, [])
        count = min(len(pool), n_per_class)
        class_counts[category] = count
        if count < n_per_class:
            print(f"Warning: category '{category}' only has {count} claims (requested {n_per_class})")

        chosen = rng.sample(pool, count) if count < len(pool) else list(pool)
        sampled.extend(chosen)

    rng.shuffle(sampled)
    for idx, item in enumerate(sampled, start=1):
        item["id"] = idx

    return sampled, class_counts


def load_scifact_validation_records() -> tuple[list[dict], str]:
    try:
        import datasets

        ds = datasets.load_dataset("allenai/scifact", "claims", split="validation")
        return [dict(r) for r in ds], "datasets library (script)"
    except Exception as e_script:
        print(f"Direct script load did not work ({e_script}). Trying parquet conversion branch...")

    try:
        import pandas as pd

        parquet_url = (
            "https://huggingface.co/datasets/allenai/scifact/resolve/refs%2Fconvert%2Fparquet/claims/validation/0000.parquet"
        )
        df = pd.read_parquet(parquet_url)
        route_desc = "parquet conversion branch (refs/convert/parquet/claims/validation/0000.parquet)"
        return df.to_dict(orient="records"), route_desc
    except Exception as e_parquet:
        print(f"Parquet conversion branch failed: {e_parquet}")

    raise RuntimeError("Failed to load SciFact validation split via both script and parquet routes.")


def main() -> None:
    records, route = load_scifact_validation_records()
    print(f"SciFact loaded successfully via: {route}")
    print(f"Loaded {len(records)} raw rows.")

    by_class, stats = filter_scifact_records(records)
    print("\nDropped claim statistics:")
    for k, v in stats.items():
        print(f"  {k}: {v}")

    print("\nEligible unique claims per class before sampling:")
    for cat, items in by_class.items():
        print(f"  {cat}: {len(items)}")

    sampled, sampled_counts = sample_stratified_scifact(by_class, n_per_class=20, seed=42)
    print(f"\nSampled claims per class (total {len(sampled)}):")
    for cat, cnt in sampled_counts.items():
        print(f"  {cat}: {cnt}")

    root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
    output_path = os.path.join(root_dir, "benchmark_scifact.json")
    with open(output_path, "w", encoding="utf-8") as f:
        json.dump(sampled, f, indent=2, ensure_ascii=False)

    print(f"\nWrote {len(sampled)} claims to {output_path}")


if __name__ == "__main__":
    main()