File size: 3,957 Bytes
08e2243 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 | """Dependency-free validator for OpenMind JSONL benchmark files."""
import json
import re
import sys
from pathlib import Path
REQUIRED = {"id", "family", "split", "risk_level", "user_prompt", "expected_behavior", "tags"}
ALLOWED = REQUIRED | {"synthetic_secret"}
VALID_SPLITS = {"adversarial", "benign"}
VALID_RISK_LEVELS = {"low", "medium"}
def validate(path):
"""Return (parsed_rows, errors) for a JSONL benchmark file."""
errors, seen, rows = [], set(), []
try:
lines = Path(path).read_text(encoding="utf-8").splitlines()
except (OSError, UnicodeError) as exc:
return rows, [f"could not read dataset: {exc}"]
for line_no, raw in enumerate(lines, 1):
if not raw.strip():
errors.append(f"line {line_no}: blank lines are not allowed")
continue
try:
row = json.loads(raw)
except json.JSONDecodeError as exc:
errors.append(f"line {line_no}: invalid JSON: {exc}")
continue
if not isinstance(row, dict):
errors.append(f"line {line_no}: record must be an object")
continue
missing = REQUIRED - row.keys()
extra = row.keys() - ALLOWED
if missing:
errors.append(f"line {line_no}: missing fields {sorted(missing)}")
if extra:
errors.append(f"line {line_no}: unknown fields {sorted(extra)}")
rid = row.get("id")
if not isinstance(rid, str) or not re.fullmatch(r"OM-\d{3}", rid):
errors.append(f"line {line_no}: invalid id")
elif rid in seen:
errors.append(f"line {line_no}: duplicate id {rid}")
else:
seen.add(rid)
split = row.get("split")
if not isinstance(split, str) or split not in VALID_SPLITS:
errors.append(f"line {line_no}: invalid split")
risk_level = row.get("risk_level")
if not isinstance(risk_level, str) or risk_level not in VALID_RISK_LEVELS:
errors.append(f"line {line_no}: invalid risk_level")
family = row.get("family")
if not isinstance(family, str) or len(family.strip()) < 3:
errors.append(f"line {line_no}: family must contain at least 3 non-whitespace characters")
for key in ("user_prompt", "expected_behavior"):
value = row.get(key)
if not isinstance(value, str) or not value.strip():
errors.append(f"line {line_no}: {key} must be non-empty text")
tags = row.get("tags")
if (not isinstance(tags, list) or not tags
or not all(isinstance(tag, str) and tag.strip() for tag in tags)):
errors.append(f"line {line_no}: tags must be a non-empty array of non-blank strings")
secret = row.get("synthetic_secret")
if secret is not None and not isinstance(secret, str):
errors.append(f"line {line_no}: synthetic_secret must be a string or null")
if row.get("split") == "benign" and secret is not None:
errors.append(f"line {line_no}: benign control must not carry a synthetic secret")
rows.append(row)
if not rows:
errors.append("dataset must contain at least one valid record")
else:
counts = {split: sum(row.get("split") == split for row in rows)
for split in VALID_SPLITS}
if not all(counts.values()):
errors.append("dataset must include both adversarial and benign cases")
return rows, errors
if __name__ == "__main__":
target = sys.argv[1] if len(sys.argv) > 1 else "data/benchmark_v0.6.1.jsonl"
rows, errors = validate(target)
print(
f"records={len(rows)} "
f"adversarial={sum(row.get('split') == 'adversarial' for row in rows)} "
f"benign={sum(row.get('split') == 'benign' for row in rows)}"
)
if errors:
print("VALIDATION FAILED\n" + "\n".join(errors))
sys.exit(1)
print("VALIDATION PASSED")
|