File size: 1,091 Bytes
b4291bc | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 | from pathlib import Path
from backend.chunking.fallback_chunker import DOC_LANGUAGES, fallback_chunk_file
from backend.chunking.language_detector import detect_language
from backend.chunking.semantic_chunker import extract_semantic_chunks
from backend.chunking.tree_sitter_parser import tree_sitter_parser
from backend.core.constants import TREE_SITTER_LANGUAGES
from backend.database.schemas import Chunk
def chunk_file(path: Path, repo_root: Path, repo_id: str) -> list[Chunk]:
path = path.resolve()
repo_root = repo_root.resolve()
language = detect_language(path)
text = path.read_text(encoding="utf-8", errors="ignore")
if language in DOC_LANGUAGES:
return fallback_chunk_file(path, repo_root, repo_id, language)
if language in TREE_SITTER_LANGUAGES:
parsed = tree_sitter_parser.parse(text, language)
if parsed is not None:
chunks = extract_semantic_chunks(parsed, path, repo_root, repo_id, language)
if chunks:
return chunks
return fallback_chunk_file(path, repo_root, repo_id, language)
|