| import re |
| from pathlib import Path |
|
|
| from backend.database.schemas import Chunk |
|
|
| SYMBOL_RE = re.compile( |
| r"^\s*(export\s+default\s+|export\s+)?" |
| r"(async\s+)?(function\s+(?P<fn>\w+)|class\s+(?P<class>\w+)|" |
| r"const\s+(?P<const>\w+)\s*=\s*(async\s*)?(\([^)]*\)|\w+)\s*=>)", |
| ) |
|
|
|
|
| def _language(path: Path) -> str: |
| return { |
| ".js": "javascript", |
| ".jsx": "javascript", |
| ".ts": "typescript", |
| ".tsx": "typescript", |
| }.get(path.suffix.lower(), "text") |
|
|
|
|
| def chunk_js_file(path: Path, repo_root: Path, repo_id: str) -> list[Chunk]: |
| text = path.read_text(encoding="utf-8", errors="ignore") |
| lines = text.splitlines() |
| rel = path.relative_to(repo_root).as_posix() |
| starts: list[tuple[int, str, str]] = [] |
|
|
| import_lines = [i + 1 for i, line in enumerate(lines) if line.strip().startswith("import ")] |
| if import_lines: |
| starts.append((min(import_lines), "imports", "imports")) |
|
|
| for i, line in enumerate(lines, start=1): |
| match = SYMBOL_RE.match(line) |
| if match: |
| symbol = match.group("fn") or match.group("class") or match.group("const") or "anonymous" |
| kind = "class" if match.group("class") else "function" |
| starts.append((i, symbol, kind)) |
|
|
| if not starts: |
| return [ |
| Chunk( |
| id=f"{repo_id}:{rel}:1", |
| repo_id=repo_id, |
| path=rel, |
| language=_language(path), |
| start_line=1, |
| end_line=max(1, len(lines)), |
| content=text, |
| ) |
| ] |
|
|
| starts = sorted(set(starts), key=lambda item: item[0]) |
| chunks: list[Chunk] = [] |
| for idx, (start, symbol, kind) in enumerate(starts): |
| end = starts[idx + 1][0] - 1 if idx + 1 < len(starts) else len(lines) |
| chunks.append( |
| Chunk( |
| id=f"{repo_id}:{rel}:{start}:{symbol}", |
| repo_id=repo_id, |
| path=rel, |
| language=_language(path), |
| symbol=symbol, |
| kind=kind, |
| start_line=start, |
| end_line=max(start, end), |
| content="\n".join(lines[start - 1 : end]).strip(), |
| ) |
| ) |
| return chunks |
|
|
