""" Batch Processing Module — NEW feature Process multiple documents in one run. Returns aggregated entity list + per-document breakdown. """ import os from modules.doc_reader import read_document, chunk_text from modules.ner_engine import run_ner from modules.analytics import label_counts def process_single(path: str) -> dict: """Process one document. Returns dict with text, entities, meta.""" text, meta = read_document(path) chunks = chunk_text(text, max_chars=400) all_entities = [] char_offset = 0 for chunk in chunks: ents = run_ner(chunk) # Adjust character offsets to be relative to full document for e in ents: e["start"] += char_offset e["end"] += char_offset all_entities.extend(ents) char_offset += len(chunk) + 1 # +1 for the space between chunks return { "path": path, "meta": meta, "text": text, "entities": all_entities, "counts": label_counts(all_entities), } def process_batch(paths: list[str]) -> list[dict]: """Process a list of file paths. Returns list of results.""" results = [] for path in paths: if not os.path.exists(path): continue try: result = process_single(path) results.append(result) except Exception as e: results.append({ "path": path, "error": str(e), }) return results def aggregate_entities(batch_results: list[dict]) -> list[dict]: """Combine entities from all documents in a batch.""" combined = [] for res in batch_results: if "entities" in res: combined.extend(res["entities"]) return combined