File size: 1,473 Bytes
c4bf3b5 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 | # -*- coding: utf-8 -*-
"""
Created on Fri Jul 3 15:41:48 2026
@author: ALBERT
"""
from nltk.tokenize import sent_tokenize
import re
def clean_text(text: str) -> str:
"""Nettoyage basique du texte"""
text = text.strip()
# supprime répétitions type "aaaaaa"
if re.search(r"(.)\1{8,}", text):
return ""
# supprime bruit évident
if len(text) < 30:
return ""
return text
def chunk_text(text, max_sentences=3, overlap=1):
sentences = sent_tokenize(text)
# nettoyage sentences
sentences = [s.strip() for s in sentences if len(s.strip()) > 0]
chunks = []
step = max_sentences - overlap
for i in range(0, len(sentences), step):
chunk_sentences = sentences[i:i + max_sentences]
chunk = " ".join(chunk_sentences)
chunk = clean_text(chunk)
if chunk: # garde uniquement chunks valides
chunks.append(chunk)
return chunks
def create_chunks_metadata(documents):
all_chunks = []
metadata = []
for doc in documents:
chunks = chunk_text(doc["text"])
for i, chunk in enumerate(chunks):
all_chunks.append(chunk)
metadata.append({
"text": chunk,
"filename": doc.get("filename", "unknown"),
"chunk_id": i,
"length": len(chunk),
"source": doc.get("filename", "unknown")
})
return all_chunks, metadata |