Spaces:
Sleeping
Sleeping
File size: 1,782 Bytes
fbd78fc | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 | """
Batch Processing Module — NEW feature
Process multiple documents in one run.
Returns aggregated entity list + per-document breakdown.
"""
import os
from modules.doc_reader import read_document, chunk_text
from modules.ner_engine import run_ner
from modules.analytics import label_counts
def process_single(path: str) -> dict:
"""Process one document. Returns dict with text, entities, meta."""
text, meta = read_document(path)
chunks = chunk_text(text, max_chars=400)
all_entities = []
char_offset = 0
for chunk in chunks:
ents = run_ner(chunk)
# Adjust character offsets to be relative to full document
for e in ents:
e["start"] += char_offset
e["end"] += char_offset
all_entities.extend(ents)
char_offset += len(chunk) + 1 # +1 for the space between chunks
return {
"path": path,
"meta": meta,
"text": text,
"entities": all_entities,
"counts": label_counts(all_entities),
}
def process_batch(paths: list[str]) -> list[dict]:
"""Process a list of file paths. Returns list of results."""
results = []
for path in paths:
if not os.path.exists(path):
continue
try:
result = process_single(path)
results.append(result)
except Exception as e:
results.append({
"path": path,
"error": str(e),
})
return results
def aggregate_entities(batch_results: list[dict]) -> list[dict]:
"""Combine entities from all documents in a batch."""
combined = []
for res in batch_results:
if "entities" in res:
combined.extend(res["entities"])
return combined
|