File size: 3,427 Bytes
f8b5d42 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 |
const { TextSplitter } = require("../../../utils/TextSplitter");
const _ = require("lodash");
describe("TextSplitter", () => {
test("should split long text into n sized chunks", async () => {
const text = "This is a test text to be split into chunks".repeat(2);
const textSplitter = new TextSplitter({
chunkSize: 20,
chunkOverlap: 0,
});
const chunks = await textSplitter.splitText(text);
expect(chunks.length).toEqual(5);
});
test("applies chunk overlap of 20 characters on invalid chunkOverlap", async () => {
const text = "This is a test text to be split into chunks".repeat(2);
const textSplitter = new TextSplitter({
chunkSize: 30,
});
const chunks = await textSplitter.splitText(text);
expect(chunks.length).toEqual(6);
});
test("does not allow chunkOverlap to be greater than chunkSize", async () => {
expect(() => {
new TextSplitter({
chunkSize: 20,
chunkOverlap: 21,
});
}).toThrow();
});
test("applies specific metadata to stringifyHeader to each chunk", async () => {
const metadata = {
id: "123e4567-e89b-12d3-a456-426614174000",
url: "https://example.com",
title: "Example",
docAuthor: "John Doe",
published: "2021-01-01",
chunkSource: "link://https://example.com",
description: "This is a test text to be split into chunks",
};
const chunkHeaderMeta = TextSplitter.buildHeaderMeta(metadata);
expect(chunkHeaderMeta).toEqual({
sourceDocument: metadata.title,
source: metadata.url,
published: metadata.published,
});
});
test("applies a valid chunkPrefix to each chunk", async () => {
const text = "This is a test text to be split into chunks".repeat(2);
let textSplitter = new TextSplitter({
chunkSize: 20,
chunkOverlap: 0,
chunkPrefix: "testing: ",
});
let chunks = await textSplitter.splitText(text);
expect(chunks.length).toEqual(5);
expect(chunks.every(chunk => chunk.startsWith("testing: "))).toBe(true);
textSplitter = new TextSplitter({
chunkSize: 20,
chunkOverlap: 0,
chunkPrefix: "testing2: ",
});
chunks = await textSplitter.splitText(text);
expect(chunks.length).toEqual(5);
expect(chunks.every(chunk => chunk.startsWith("testing2: "))).toBe(true);
textSplitter = new TextSplitter({
chunkSize: 20,
chunkOverlap: 0,
chunkPrefix: undefined,
});
chunks = await textSplitter.splitText(text);
expect(chunks.length).toEqual(5);
expect(chunks.every(chunk => !chunk.startsWith(": "))).toBe(true);
textSplitter = new TextSplitter({
chunkSize: 20,
chunkOverlap: 0,
chunkPrefix: "",
});
chunks = await textSplitter.splitText(text);
expect(chunks.length).toEqual(5);
expect(chunks.every(chunk => !chunk.startsWith(": "))).toBe(true);
// Applied chunkPrefix with chunkHeaderMeta
textSplitter = new TextSplitter({
chunkSize: 20,
chunkOverlap: 0,
chunkHeaderMeta: TextSplitter.buildHeaderMeta({
title: "Example",
url: "https://example.com",
published: "2021-01-01",
}),
chunkPrefix: "testing3: ",
});
chunks = await textSplitter.splitText(text);
expect(chunks.length).toEqual(5);
expect(chunks.every(chunk => chunk.startsWith("testing3: <document_metadata>"))).toBe(true);
});
});
|