| """Auditable local text preparation. Fetching a dataset is deliberately a notebook/user action.""" |
| import argparse, hashlib, json, re |
| from pathlib import Path |
| CONTROL=re.compile(r'[\x00-\x08\x0b\x0c\x0e-\x1f]') |
| |
| EMAIL=re.compile(r'\b[\w.+-]+@[\w.-]+\.[A-Za-z]{2,}\b') |
| PHONE=re.compile(r'\b(?:\+?1[-. ]?)?(?:\(?\d{3}\)?[-. ]?){2}\d{4}\b') |
| def clean(text): |
| text=CONTROL.sub('',text).replace('\r\n','\n').replace('\r','\n') |
| text=EMAIL.sub('[EMAIL_REDACTED]',text);text=PHONE.sub('[PHONE_REDACTED]',text) |
| return '\n'.join(x.rstrip() for x in text.splitlines()).strip() |
| def prepare(inp,out,minimum=80): |
| seen=set(); stats={'read':0,'kept':0,'duplicates':0,'too_short':0};Path(out).parent.mkdir(parents=True,exist_ok=True) |
| with open(out,'w',encoding='utf8') as dst: |
| for line in Path(inp).open(encoding='utf8'): |
| stats['read']+=1 |
| try: raw=json.loads(line);text=clean(str(raw.get('text',''))) |
| except json.JSONDecodeError: continue |
| key=hashlib.sha256(text.encode()).hexdigest() |
| if len(text)<minimum:stats['too_short']+=1;continue |
| if key in seen:stats['duplicates']+=1;continue |
| seen.add(key);dst.write(json.dumps({'text':text,'sha256':key},ensure_ascii=False)+'\n');stats['kept']+=1 |
| return stats |
| def main(): |
| p=argparse.ArgumentParser();p.add_argument('--input',required=True);p.add_argument('--out',required=True);p.add_argument('--audit',default='data/audit.json');p.add_argument('--approved',action='store_true',help='Required confirmation that data licence/provenance was reviewed.');a=p.parse_args() |
| if not a.approved:raise SystemExit('Refusing: review the manifest/dataset card, then pass --approved.') |
| stats=prepare(a.input,a.out);Path(a.audit).parent.mkdir(parents=True,exist_ok=True);Path(a.audit).write_text(json.dumps(stats,indent=2));print(json.dumps(stats)) |
| if __name__=='__main__':main() |
|
|
