"""Fetch the pinned public development suite and append authored Russian probes."""
import argparse
import hashlib
import json
import urllib.request
from pathlib import Path

REVISION = '2074cc44256200bb37540103fa4572901faa10f3'
ORIGINAL_SHA256 = 'ff374c49c6c9f15f8a56fb274b4a4857d20497eb8dd1ac07ce01560e682a5f2e'
COMPARISON_SHA256 = '9266d6ea1ce83c9755a60cf48c7469b0fa3264c229e96f4142d683bada8fef91'


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('--out', type=Path, required=True)
    args = parser.parse_args()
    if args.out.exists():
        raise ValueError('Output must be a new directory')
    url = f'https://raw.githubusercontent.com/jaredpalmer/kev/{REVISION}/evals/v4/transfer-v4/development.jsonl'
    with urllib.request.urlopen(url, timeout=60) as response:
        data = response.read()
    if hashlib.sha256(data).hexdigest() != ORIGINAL_SHA256:
        raise ValueError('Pinned upstream dataset hash mismatch')
    converted = []
    for i, line in enumerate(data.decode().splitlines()):
        row = json.loads(line)
        converted.append({'id': f'transfer-v4-{i:04}', 'source': row['_meta']['source'],
            'state': row['state'], 'questions': {k: {n:v for n,v in q.items() if n != 'src'}
                                               for k,q in row['questions'].items()}})
    text = ''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in converted)
    russian = Path(__file__).resolve().parents[1]/'evaluation/benchmarks/russian-typed.jsonl'
    combined = (text+russian.read_text()).encode()
    if len(converted) != 764 or hashlib.sha256(combined).hexdigest() != COMPARISON_SHA256:
        raise ValueError('Converted comparison hash mismatch')
    args.out.mkdir(parents=True)
    (args.out/'transfer-v4.jsonl').write_text(text)
    (args.out/'comparison.jsonl').write_bytes(combined)
    (args.out/'manifest.json').write_text(json.dumps({'url':url, 'upstreamSha256':ORIGINAL_SHA256,
        'comparisonSha256':COMPARISON_SHA256, 'records':812, 'questions':908,
        'usage':'Development comparison, not training or fresh customer gold',
        'rights':'Upstream data retains its original source licenses; review Kev transfer-v4 source manifest'}, indent=2)+'\n')
    print(COMPARISON_SHA256)


if __name__ == '__main__':
    main()