Spaces:
Building
Building
File size: 1,800 Bytes
08b3daf | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 | import re
def count_tokens_approx(text: str) -> int:
"""
Ước lượng token đơn giản.
Với tiếng Việt, tạm dùng số word/punctuation.
Có thể thay bằng tokenizer chính xác hơn như tiktoken nếu cần.
"""
if not text:
return 0
tokens = re.findall(r"\w+|[^\w\s]", text, flags=re.UNICODE)
return len(tokens)
def split_text_by_paragraph(
text: str,
max_tokens: int,
overlap_tokens: int = 0,
) -> list[str]:
"""
Split theo paragraph trước.
Nếu paragraph quá dài thì split tiếp theo câu.
"""
paragraphs = [p.strip() for p in text.split("\n") if p.strip()]
chunks = []
current = []
for paragraph in paragraphs:
candidate = "\n".join(current + [paragraph])
if count_tokens_approx(candidate) <= max_tokens:
current.append(paragraph)
else:
if current:
chunks.append("\n".join(current))
if count_tokens_approx(paragraph) > max_tokens:
chunks.extend(split_text_by_sentence(paragraph, max_tokens))
current = []
else:
current = [paragraph]
if current:
chunks.append("\n".join(current))
return chunks
def split_text_by_sentence(text: str, max_tokens: int) -> list[str]:
sentences = re.split(r"(?<=[.!?。])\s+", text)
chunks = []
current = []
for sentence in sentences:
candidate = " ".join(current + [sentence])
if count_tokens_approx(candidate) <= max_tokens:
current.append(sentence)
else:
if current:
chunks.append(" ".join(current))
current = [sentence]
if current:
chunks.append(" ".join(current))
return chunks
|