"""Build lightweight portal statistics from the published pinned task CSV.

Usage: python build_dataset_summary.py /path/to/1000_new.csv
Download that CSV from the dataset revision recorded below, not the moving main.
"""
import argparse
from collections import Counter
import csv
import hashlib
import io
import json
from pathlib import Path

REVISION = '59f9512a38e594528807744214a60ee00367434e'
MAPPING = {'OLS': 'OLS', 'panel OLS': 'OLS', 'DID': 'DID', 'IV': 'IV', 'IV-2SLS': 'IV', 'RDD': 'RDD', 'PROBIT': 'Other', 'Probit': 'Other', 'LOGIT': 'Other', 'Logit': 'Other', 'Negative binomial': 'Other', 'LMM': 'Other', 'PSM': 'Other', 'Cox proportional hazards': 'Other'}
p = argparse.ArgumentParser(description=__doc__)
p.add_argument('task_list', type=Path)
p.add_argument('--output', type=Path, default=Path('dataset-summary.json'))
a = p.parse_args()
raw = a.task_list.read_bytes()
assert hashlib.sha256(raw).hexdigest() == '5338c8af548b35dab0e92a63b0acfebb0bc29c7901da4088851025f5709d6bb3', 'Input does not match the pinned task list'
rows = list(csv.DictReader(io.StringIO(raw.decode('utf-8-sig'))))
assert len(rows) == 1000 and len({r['sequence_id'] for r in rows}) == 1000
journals = Counter(r['journal'] for r in rows)
methods = Counter(MAPPING[r['method']] for r in rows)
result = {
 'dataset': 'CamoAiLab/InferenceNet', 'revision': REVISION,
 'task_list': 'Selected_1000/1000_new.csv', 'task_list_sha256': hashlib.sha256(raw).hexdigest(),
 'tasks': len(rows), 'journal_count': len(journals), 'distinct_article_titles': len({r['article'] for r in rows}),
 'raw_method_labels': len({r['method'] for r in rows}), 'languages': dict(Counter(r['language'] for r in rows)),
 'methods': [{'label': k, 'count': v} for k,v in methods.most_common()],
 'journals': [{'label': k, 'count': v} for k,v in journals.most_common()], 'method_mapping': MAPPING,
}
a.output.write_text(json.dumps(result, indent=2, ensure_ascii=False) + '\n')
print(f"Wrote {a.output}: {len(rows)} tasks, {len(journals)} journals")