Spaces:
Runtime error
Runtime error
| """Clean page text and split it into sentence-aware, overlapping chunks.""" | |
| import re | |
| _SENT_SPLIT = re.compile(r"(?<=[.!?\u0964])\s+") # \u0964 = Hindi danda | |
| def paragraphs(text: str) -> list[str]: | |
| text = text.replace("\r", "") | |
| text = re.sub(r"-\n(?=[a-z])", "", text) # join words hyphenated across lines | |
| text = re.sub(r"[ \t\u00a0]+", " ", text) | |
| paras = re.split(r"\n\s*\n", text) | |
| paras = [re.sub(r"\s*\n\s*", " ", p).strip() for p in paras] | |
| return [p for p in paras if p] | |
| def _split_units(par: str, size: int) -> list[str]: | |
| units: list[str] = [] | |
| for sent in _SENT_SPLIT.split(par): | |
| sent = sent.strip() | |
| while len(sent) > size: # one huge "sentence" (tables, code): cut at a space | |
| cut = sent.rfind(" ", 0, size) | |
| cut = cut if cut > size * 0.5 else size | |
| units.append(sent[:cut].strip()) | |
| sent = sent[cut:].strip() | |
| if sent: | |
| units.append(sent) | |
| return units | |
| def chunk_text(text: str, size: int, overlap: int) -> list[str]: | |
| units: list[str] = [] | |
| for par in paragraphs(text): | |
| units.extend(_split_units(par, size)) | |
| chunks: list[str] = [] | |
| cur: list[str] = [] | |
| cur_len = 0 | |
| for u in units: | |
| add = len(u) + (1 if cur else 0) | |
| if cur and cur_len + add > size: | |
| chunks.append(" ".join(cur)) | |
| keep: list[str] = [] # carry the last sentences over as overlap | |
| keep_len = 0 | |
| for prev in reversed(cur): | |
| if keep_len + len(prev) + 1 > overlap: | |
| break | |
| keep.insert(0, prev) | |
| keep_len += len(prev) + 1 | |
| cur = keep | |
| cur_len = sum(len(x) for x in cur) + max(0, len(cur) - 1) | |
| add = len(u) + (1 if cur else 0) | |
| cur.append(u) | |
| cur_len += add | |
| if cur: | |
| chunks.append(" ".join(cur)) | |
| return [c for c in chunks if c.strip()] | |