backend/chunking/chunk_manager.py

from pathlib import Path

from backend.chunking.fallback_chunker import DOC_LANGUAGES, fallback_chunk_file
from backend.chunking.language_detector import detect_language
from backend.chunking.semantic_chunker import extract_semantic_chunks
from backend.chunking.tree_sitter_parser import tree_sitter_parser
from backend.core.constants import TREE_SITTER_LANGUAGES
from backend.database.schemas import Chunk


def chunk_file(path: Path, repo_root: Path, repo_id: str) -> list[Chunk]:
    path = path.resolve()
    repo_root = repo_root.resolve()
    language = detect_language(path)
    text = path.read_text(encoding="utf-8", errors="ignore")

    if language in DOC_LANGUAGES:
        return fallback_chunk_file(path, repo_root, repo_id, language)

    if language in TREE_SITTER_LANGUAGES:
        parsed = tree_sitter_parser.parse(text, language)
        if parsed is not None:
            chunks = extract_semantic_chunks(parsed, path, repo_root, repo_id, language)
            if chunks:
                return chunks

    return fallback_chunk_file(path, repo_root, repo_id, language)