src/data_processing/txt_chunker.py

# -*- coding: utf-8 -*-
"""
Created on Fri Jul  3 15:41:48 2026

@author: ALBERT
"""

from nltk.tokenize import sent_tokenize
import re


def clean_text(text: str) -> str:
    """Nettoyage basique du texte"""

    text = text.strip()

    # supprime répétitions type "aaaaaa"
    if re.search(r"(.)\1{8,}", text):
        return ""

    # supprime bruit évident
    if len(text) < 30:
        return ""

    return text


def chunk_text(text, max_sentences=3, overlap=1):

    sentences = sent_tokenize(text)

    # nettoyage sentences
    sentences = [s.strip() for s in sentences if len(s.strip()) > 0]

    chunks = []

    step = max_sentences - overlap

    for i in range(0, len(sentences), step):

        chunk_sentences = sentences[i:i + max_sentences]
        chunk = " ".join(chunk_sentences)

        chunk = clean_text(chunk)

        if chunk:  # garde uniquement chunks valides
            chunks.append(chunk)

    return chunks

def create_chunks_metadata(documents):

    all_chunks = []
    metadata = []

    for doc in documents:

        chunks = chunk_text(doc["text"])

        for i, chunk in enumerate(chunks):

            all_chunks.append(chunk)

            metadata.append({
                "text": chunk,
                "filename": doc.get("filename", "unknown"),
                "chunk_id": i,
                "length": len(chunk),
                "source": doc.get("filename", "unknown")
            })

    return all_chunks, metadata