MIT LICENSE, README license fix, make_dataset.py, 019 re-evaluation, technical report and CITATION.cff
86415de verified | #!/usr/bin/env python3 | |
| """Regenerate the Cuerdas 014-018 synthetic dataset (expanded.json) from its documented protocol. | |
| The original file was produced during iteration 014 together with ChatGPT and is not | |
| redistributed. This script rebuilds an equivalent dataset deterministically, with only the | |
| Python standard library: | |
| * vocabulary: 12 actions x 12 targets x 16 statuses x 10 windows (23,040 combinations); | |
| * every combination inside the 7x7x8x5 universe of iteration 013 (index < 7,7,8,5 per field) | |
| is excluded (1,960 combinations); | |
| * held-out relations: action-target pairs (a, (a+d) mod 12) for d in {2, 5} (24 pairs) and | |
| status-window pairs (s, (m*s+b) mod 10) for (m, b) in {(2,1), (3,2)} (31 distinct pairs); | |
| * a combination is "ood" if it contains a held-out pair, otherwise "covered"; | |
| * both pools are shuffled with random.Random(seed) (default 140014) and cut into | |
| fit 1200 / validation 240 / calibration_id 200 / id_test 400 (covered) and | |
| calibration_ood 200 / ood_test 800 (ood). | |
| Usage: | |
| python make_dataset.py --out data/expanded.json # regenerate | |
| python make_dataset.py --out data/expanded.json --check OLD # also compare tuples/splits with an existing file | |
| The output keeps the keys read by run015-run018.py: vocabulary, rows[*].values, splits. | |
| """ | |
| import argparse, itertools, json, random, sys, hashlib | |
| FIELDS = ['action', 'target', 'status', 'window'] | |
| VOCAB = [ | |
| ['query', 'create', 'update', 'delete', 'notify', 'execute', 'report', 'read', 'write', 'check', 'start', 'stop'], | |
| ['invoice', 'email', 'file', 'meeting', 'inventory', 'machine', 'job', 'order', 'customer', 'pump', 'valve', 'motor'], | |
| ['pending', 'paid', 'failed', 'complete', 'open', 'low', 'overload', 'none', 'active', 'idle', 'closed', 'ready', | |
| 'high', 'normal', 'blocked', 'running'], | |
| ['now', 'day', 'week', 'month', 'none', 'hour', 'minute', 'year', 'today', 'tomorrow'], | |
| ] | |
| SIZES_013 = (7, 7, 8, 5) | |
| def build(seed=140014): | |
| hold_at = sorted({(a, (a + d) % 12) for a in range(12) for d in (2, 5)}) | |
| hold_sw = sorted({(s, (m * s + b) % 10) for s in range(16) for m, b in ((2, 1), (3, 2))}) | |
| hat, hsw = set(hold_at), set(hold_sw) | |
| covered, ood = [], [] | |
| for v in itertools.product(*(range(len(x)) for x in VOCAB)): | |
| if all(v[j] < SIZES_013[j] for j in range(4)): | |
| continue # exclude the whole 013 universe | |
| (ood if (v[:2] in hat or v[2:] in hsw) else covered).append(v) | |
| n_cov, n_ood = len(covered), len(ood) | |
| rng = random.Random(seed) | |
| rng.shuffle(covered); rng.shuffle(ood) | |
| rows, splits = [], {} | |
| # rows are numbered in this split order (the order used by the original file) | |
| plan = [('fit', 'c', 1200), ('validation', 'c', 240), ('calibration_id', 'c', 200), | |
| ('calibration_ood', 'o', 200), ('id_test', 'c', 400), ('ood_test', 'o', 800)] | |
| pos = {'c': 0, 'o': 0}; pools = {'c': covered, 'o': ood} | |
| for name, src, n in plan: | |
| splits[name] = [] | |
| for v in pools[src][pos[src]:pos[src] + n]: | |
| splits[name].append(len(rows)) | |
| rows.append({'id': len(rows), 'split': name, 'values': list(v), | |
| 'event': {'schema': 'cuerdas014.v1', 'kind': 'DATA', | |
| 'values': {f: VOCAB[j][v[j]] for j, f in enumerate(FIELDS)}}}) | |
| pos[src] += n | |
| used_c, used_o = pos['c'], pos['o'] | |
| splits['calibration'] = splits['calibration_id'] + splits['calibration_ood'] | |
| # --- invariants --- | |
| assert len({tuple(r['values']) for r in rows}) == len(rows) | |
| fit = [rows[i]['values'] for i in splits['fit']] | |
| seen_at = {tuple(v[:2]) for v in fit}; seen_sw = {tuple(v[2:]) for v in fit} | |
| for q in range(4): | |
| assert {v[q] for v in fit} == set(range(len(VOCAB[q]))), 'every value must appear in fit' | |
| for s in ('validation', 'calibration_id', 'id_test'): | |
| assert all(tuple(rows[i]['values'][:2]) in seen_at and tuple(rows[i]['values'][2:]) in seen_sw for i in splits[s]) | |
| for s in ('calibration_ood', 'ood_test'): | |
| assert all(tuple(rows[i]['values'][:2]) not in seen_at or tuple(rows[i]['values'][2:]) not in seen_sw for i in splits[s]) | |
| return {'experiment': 'CUERDAS-014 (regenerated by make_dataset.py)', 'seed': seed, 'vocabulary': VOCAB, | |
| 'rows': rows, 'splits': splits, 'universe_size': 23040, 'excluded_entire_013_universe': 1960, | |
| 'unused_covered': n_cov - used_c, 'unused_ood': n_ood - used_o, | |
| 'holdout_action_target': [list(p) for p in hold_at], 'holdout_status_window': [list(p) for p in hold_sw], | |
| 'calibration_distribution': '50% new covered combinations, 50% unseen training relations'} | |
| def fingerprint(meta): | |
| """Hash of what the experiments actually read: vocabulary, ordered tuples and split indices.""" | |
| core = {'vocabulary': meta['vocabulary'], 'values': [r['values'] for r in meta['rows']], | |
| 'splits': {k: meta['splits'][k] for k in sorted(meta['splits'])}} | |
| return hashlib.sha256(json.dumps(core, sort_keys=True).encode()).hexdigest() | |
| if __name__ == '__main__': | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument('--out', default='data/expanded.json'); ap.add_argument('--seed', type=int, default=140014) | |
| ap.add_argument('--check', help='existing expanded.json to compare against') | |
| a = ap.parse_args() | |
| meta = build(a.seed) | |
| import os; os.makedirs(os.path.dirname(a.out) or '.', exist_ok=True) | |
| json.dump(meta, open(a.out, 'w')) | |
| print(json.dumps({'out': a.out, 'rows': len(meta['rows']), 'splits': {k: len(v) for k, v in meta['splits'].items()}, | |
| 'held_out_at': len(meta['holdout_action_target']), 'held_out_sw': len(meta['holdout_status_window']), | |
| 'fingerprint': fingerprint(meta)})) | |
| if a.check: | |
| old = json.load(open(a.check)) | |
| same = fingerprint(old) == fingerprint(meta) | |
| print(json.dumps({'check_against': a.check, 'identical_tuples_and_splits': same})) | |
| sys.exit(0 if same else 1) | |