| """Build lightweight portal statistics from the published pinned task CSV. |
| |
| Usage: python build_dataset_summary.py /path/to/1000_new.csv |
| Download that CSV from the dataset revision recorded below, not the moving main. |
| """ |
| import argparse |
| from collections import Counter |
| import csv |
| import hashlib |
| import io |
| import json |
| from pathlib import Path |
|
|
| REVISION = '59f9512a38e594528807744214a60ee00367434e' |
| MAPPING = {'OLS': 'OLS', 'panel OLS': 'OLS', 'DID': 'DID', 'IV': 'IV', 'IV-2SLS': 'IV', 'RDD': 'RDD', 'PROBIT': 'Other', 'Probit': 'Other', 'LOGIT': 'Other', 'Logit': 'Other', 'Negative binomial': 'Other', 'LMM': 'Other', 'PSM': 'Other', 'Cox proportional hazards': 'Other'} |
| p = argparse.ArgumentParser(description=__doc__) |
| p.add_argument('task_list', type=Path) |
| p.add_argument('--output', type=Path, default=Path('dataset-summary.json')) |
| a = p.parse_args() |
| raw = a.task_list.read_bytes() |
| assert hashlib.sha256(raw).hexdigest() == '5338c8af548b35dab0e92a63b0acfebb0bc29c7901da4088851025f5709d6bb3', 'Input does not match the pinned task list' |
| rows = list(csv.DictReader(io.StringIO(raw.decode('utf-8-sig')))) |
| assert len(rows) == 1000 and len({r['sequence_id'] for r in rows}) == 1000 |
| journals = Counter(r['journal'] for r in rows) |
| methods = Counter(MAPPING[r['method']] for r in rows) |
| result = { |
| 'dataset': 'CamoAiLab/InferenceNet', 'revision': REVISION, |
| 'task_list': 'Selected_1000/1000_new.csv', 'task_list_sha256': hashlib.sha256(raw).hexdigest(), |
| 'tasks': len(rows), 'journal_count': len(journals), 'distinct_article_titles': len({r['article'] for r in rows}), |
| 'raw_method_labels': len({r['method'] for r in rows}), 'languages': dict(Counter(r['language'] for r in rows)), |
| 'methods': [{'label': k, 'count': v} for k,v in methods.most_common()], |
| 'journals': [{'label': k, 'count': v} for k,v in journals.most_common()], 'method_mapping': MAPPING, |
| } |
| a.output.write_text(json.dumps(result, indent=2, ensure_ascii=False) + '\n') |
| print(f"Wrote {a.output}: {len(rows)} tasks, {len(journals)} journals") |
|
|
