"""Auditable local text preparation. Fetching a dataset is deliberately a notebook/user action."""
import argparse, hashlib, json, re
from pathlib import Path
CONTROL=re.compile(r'[\x00-\x08\x0b\x0c\x0e-\x1f]')
# Baseline filters; they are not a legal/privacy guarantee.
EMAIL=re.compile(r'\b[\w.+-]+@[\w.-]+\.[A-Za-z]{2,}\b')
PHONE=re.compile(r'\b(?:\+?1[-. ]?)?(?:\(?\d{3}\)?[-. ]?){2}\d{4}\b')
def clean(text):
 text=CONTROL.sub('',text).replace('\r\n','\n').replace('\r','\n')
 text=EMAIL.sub('[EMAIL_REDACTED]',text);text=PHONE.sub('[PHONE_REDACTED]',text)
 return '\n'.join(x.rstrip() for x in text.splitlines()).strip()
def prepare(inp,out,minimum=80):
 seen=set(); stats={'read':0,'kept':0,'duplicates':0,'too_short':0};Path(out).parent.mkdir(parents=True,exist_ok=True)
 with open(out,'w',encoding='utf8') as dst:
  for line in Path(inp).open(encoding='utf8'):
   stats['read']+=1
   try: raw=json.loads(line);text=clean(str(raw.get('text','')))
   except json.JSONDecodeError: continue
   key=hashlib.sha256(text.encode()).hexdigest()
   if len(text)<minimum:stats['too_short']+=1;continue
   if key in seen:stats['duplicates']+=1;continue
   seen.add(key);dst.write(json.dumps({'text':text,'sha256':key},ensure_ascii=False)+'\n');stats['kept']+=1
 return stats
def main():
 p=argparse.ArgumentParser();p.add_argument('--input',required=True);p.add_argument('--out',required=True);p.add_argument('--audit',default='data/audit.json');p.add_argument('--approved',action='store_true',help='Required confirmation that data licence/provenance was reviewed.');a=p.parse_args()
 if not a.approved:raise SystemExit('Refusing: review the manifest/dataset card, then pass --approved.')
 stats=prepare(a.input,a.out);Path(a.audit).parent.mkdir(parents=True,exist_ok=True);Path(a.audit).write_text(json.dumps(stats,indent=2));print(json.dumps(stats))
if __name__=='__main__':main()