File size: 3,957 Bytes
08e2243
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
"""Dependency-free validator for OpenMind JSONL benchmark files."""
import json
import re
import sys
from pathlib import Path

REQUIRED = {"id", "family", "split", "risk_level", "user_prompt", "expected_behavior", "tags"}
ALLOWED = REQUIRED | {"synthetic_secret"}
VALID_SPLITS = {"adversarial", "benign"}
VALID_RISK_LEVELS = {"low", "medium"}


def validate(path):
    """Return (parsed_rows, errors) for a JSONL benchmark file."""
    errors, seen, rows = [], set(), []
    try:
        lines = Path(path).read_text(encoding="utf-8").splitlines()
    except (OSError, UnicodeError) as exc:
        return rows, [f"could not read dataset: {exc}"]

    for line_no, raw in enumerate(lines, 1):
        if not raw.strip():
            errors.append(f"line {line_no}: blank lines are not allowed")
            continue
        try:
            row = json.loads(raw)
        except json.JSONDecodeError as exc:
            errors.append(f"line {line_no}: invalid JSON: {exc}")
            continue
        if not isinstance(row, dict):
            errors.append(f"line {line_no}: record must be an object")
            continue

        missing = REQUIRED - row.keys()
        extra = row.keys() - ALLOWED
        if missing:
            errors.append(f"line {line_no}: missing fields {sorted(missing)}")
        if extra:
            errors.append(f"line {line_no}: unknown fields {sorted(extra)}")

        rid = row.get("id")
        if not isinstance(rid, str) or not re.fullmatch(r"OM-\d{3}", rid):
            errors.append(f"line {line_no}: invalid id")
        elif rid in seen:
            errors.append(f"line {line_no}: duplicate id {rid}")
        else:
            seen.add(rid)

        split = row.get("split")
        if not isinstance(split, str) or split not in VALID_SPLITS:
            errors.append(f"line {line_no}: invalid split")
        risk_level = row.get("risk_level")
        if not isinstance(risk_level, str) or risk_level not in VALID_RISK_LEVELS:
            errors.append(f"line {line_no}: invalid risk_level")

        family = row.get("family")
        if not isinstance(family, str) or len(family.strip()) < 3:
            errors.append(f"line {line_no}: family must contain at least 3 non-whitespace characters")
        for key in ("user_prompt", "expected_behavior"):
            value = row.get(key)
            if not isinstance(value, str) or not value.strip():
                errors.append(f"line {line_no}: {key} must be non-empty text")

        tags = row.get("tags")
        if (not isinstance(tags, list) or not tags
                or not all(isinstance(tag, str) and tag.strip() for tag in tags)):
            errors.append(f"line {line_no}: tags must be a non-empty array of non-blank strings")

        secret = row.get("synthetic_secret")
        if secret is not None and not isinstance(secret, str):
            errors.append(f"line {line_no}: synthetic_secret must be a string or null")
        if row.get("split") == "benign" and secret is not None:
            errors.append(f"line {line_no}: benign control must not carry a synthetic secret")

        rows.append(row)

    if not rows:
        errors.append("dataset must contain at least one valid record")
    else:
        counts = {split: sum(row.get("split") == split for row in rows)
                  for split in VALID_SPLITS}
        if not all(counts.values()):
            errors.append("dataset must include both adversarial and benign cases")
    return rows, errors


if __name__ == "__main__":
    target = sys.argv[1] if len(sys.argv) > 1 else "data/benchmark_v0.6.1.jsonl"
    rows, errors = validate(target)
    print(
        f"records={len(rows)} "
        f"adversarial={sum(row.get('split') == 'adversarial' for row in rows)} "
        f"benign={sum(row.get('split') == 'benign' for row in rows)}"
    )
    if errors:
        print("VALIDATION FAILED\n" + "\n".join(errors))
        sys.exit(1)
    print("VALIDATION PASSED")