File size: 3,488 Bytes
8f46582 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 | #!/usr/bin/env python3
"""Audit 2-arm star datasets for structural bugs / leakage / distribution shifts.
For each depth: validate every train+valid sample's graph structure, check
train<->valid leakage at the graph level AND the hop-1-pattern level, and
report stats that could differ between working (L6/L10) and failing (L12/L14)
depths.
"""
import json
import sys
from collections import Counter
def load(path):
return json.load(open(path))
def audit_depth(L):
print(f"\n================ L{L} ================")
tr = load(f"data/star_2arm_L{L}_train_fo_bfs.json")
va = load(f"data/star_2arm_L{L}_valid_fo_bfs.json")
n_nodes = 2 + 4 * L
bad = Counter()
def check(s):
edges = [tuple(e) for e in s["edges"]]
if len(edges) != 4 * L:
bad["edge_count"] += 1
root, neg_root = s["root"], s["neg_root"]
# adjacency (directed src->dst as generated)
out = {}
for a, b in edges:
out.setdefault(a, []).append(b)
# root must have exactly 2 outgoing edges (2 arms)
if sorted(out.get(root, [])) != sorted(s["neighbor_k"]["1"]):
bad["hop1_mismatch"] += 1
if len(out.get(root, [])) != 2:
bad["root_degree"] += 1
# BFS frontier sets: neighbor_k[k] must be reachable at exactly k hops
frontier = {root}
for k in range(1, L + 1):
nxt = set()
for v in frontier:
nxt.update(out.get(v, []))
if set(s["neighbor_k"][str(k)]) != nxt:
bad[f"frontier_k"] += 1
break
frontier = nxt
# target must be a leaf at hop L; neg_target unreachable from root
reach = {root}
stack = [root]
while stack:
v = stack.pop()
for w in out.get(v, []):
if w not in reach:
reach.add(w)
stack.append(w)
if s["target"] not in reach:
bad["target_unreachable"] += 1
if s["neg_target"] in reach:
bad["neg_target_reachable"] += 1
if len(reach) != 1 + 2 * L:
bad["component_size"] += 1
for s in tr + va:
check(s)
# graph-level leakage
def key(s):
return (s["root"], s["target"], s["neg_target"],
frozenset(tuple(e) for e in s["edges"]))
tr_keys = {key(s) for s in tr}
va_keys = {key(s) for s in va}
leak = len(tr_keys & va_keys)
# hop-1 pattern coverage: (root, frozenset(hop1 neighbors))
tr_h1 = {(s["root"], frozenset(s["neighbor_k"]["1"])) for s in tr}
va_h1 = [(s["root"], frozenset(s["neighbor_k"]["1"])) for s in va]
seen_h1 = sum(1 for h in va_h1 if h in tr_h1)
ids = Counter()
for s in tr:
for x in s["idx_to_symbol"]:
ids[int(x)] += 1
pool = max(ids) + 1
print(f"train={len(tr)} valid={len(va)} nodes/sample={n_nodes} pool={pool}")
print(f"structural violations: {dict(bad) if bad else 'NONE'}")
print(f"graph-level train/val overlap: {leak}")
print(f"unique hop-1 patterns in train: {len(tr_h1)}")
print(f"val hop-1 patterns also present in train: {seen_h1}/{len(va)} "
f"({seen_h1/len(va):.1%})")
mn, mx = min(ids.values()), max(ids.values())
print(f"id usage min/max across pool: {mn}/{mx} (ratio {mn/mx:.2f})")
if __name__ == "__main__":
for L in (int(x) for x in (sys.argv[1:] or ["6", "10", "12", "14"])):
audit_depth(L)
|