Spaces:
Running
Running
File size: 2,110 Bytes
73702d3 501b0f3 52c1352 73702d3 52c1352 73702d3 52c1352 73702d3 52c1352 73702d3 c343217 4ddb694 501b0f3 52c1352 501b0f3 73702d3 52c1352 73702d3 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 | """Unit tests for the classes under chunker.py.
These are minimal happy-path tests to ensure that the chunkers don't crash.
Dependencies:
pip install pytest
pip install pytest-mock
"""
import os
from pytest import mark, param
import sage.chunker
def test_text_chunker_happy_path():
"""Tests the happy path for the TextFileChunker."""
chunker = sage.chunker.TextFileChunker(max_tokens=100)
file_path = os.path.join(os.path.dirname(__file__), "../README.md")
with open(file_path, "r") as file:
content = file.read()
metadata = {"file_path": file_path}
chunks = chunker.chunk(content, metadata)
assert len(chunks) >= 1
def test_code_chunker_happy_path():
"""Tests the happy path for the CodeFileChunker."""
chunker = sage.chunker.CodeFileChunker(max_tokens=100)
file_path = os.path.join(os.path.dirname(__file__), "../sage/chunker.py")
with open(file_path, "r") as file:
content = file.read()
metadata = {"file_path": file_path}
chunks = chunker.chunk(content, metadata)
assert len(chunks) >= 1
@mark.parametrize("filename", [param("assets/sample-script.ts"), param("assets/sample-script.tsx")])
def test_code_chunker_typescript_happy_path(filename):
"""Tests the happy path for the CodeFileChunker on .ts and .tsx files."""
file_path = os.path.join(os.path.dirname(__file__), filename)
with open(file_path, "r") as file:
content = file.read()
metadata = {"file_path": file_path}
chunker = sage.chunker.CodeFileChunker(max_tokens=100)
chunks = chunker.chunk(content, metadata)
assert len(chunks) >= 1
def test_ipynb_chunker_happy_path():
"""Tests the happy path for the IPynbChunker."""
code_chunker = sage.chunker.CodeFileChunker(max_tokens=100)
chunker = sage.chunker.IpynbFileChunker(code_chunker)
file_path = os.path.join(os.path.dirname(__file__), "assets/sample-notebook.ipynb")
with open(file_path, "r") as file:
content = file.read()
metadata = {"file_path": file_path}
chunks = chunker.chunk(content, metadata)
assert len(chunks) >= 1
|