Spaces:
Paused
Paused
File size: 2,307 Bytes
0e759d2 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 |
// crawler.test.ts
import { WebCrawler } from "../crawler";
import axios from "axios";
import robotsParser from "robots-parser";
jest.mock("axios");
jest.mock("robots-parser");
describe("WebCrawler", () => {
let crawler: WebCrawler;
const mockAxios = axios as jest.Mocked<typeof axios>;
const mockRobotsParser = robotsParser as jest.MockedFunction<
typeof robotsParser
>;
let maxCrawledDepth: number;
beforeEach(() => {
// Setup default mocks
mockAxios.get.mockImplementation((url) => {
if (url.includes("robots.txt")) {
return Promise.resolve({ data: "User-agent: *\nAllow: /" });
} else if (url.includes("sitemap.xml")) {
return Promise.resolve({ data: "sitemap content" }); // You would normally parse this to URLs
}
return Promise.resolve({ data: "<html></html>" });
});
mockRobotsParser.mockReturnValue({
isAllowed: jest.fn().mockReturnValue(true),
isDisallowed: jest.fn().mockReturnValue(false),
getMatchingLineNumber: jest.fn().mockReturnValue(0),
getCrawlDelay: jest.fn().mockReturnValue(0),
getSitemaps: jest.fn().mockReturnValue([]),
getPreferredHost: jest.fn().mockReturnValue("example.com"),
});
});
it("should respect the limit parameter by not returning more links than specified", async () => {
const initialUrl = "http://example.com";
const limit = 2; // Set a limit for the number of links
crawler = new WebCrawler({
jobId: "TEST",
initialUrl: initialUrl,
includes: [],
excludes: [],
limit: limit, // Apply the limit
maxCrawledDepth: 10,
});
// Mock sitemap fetching function to return more links than the limit
crawler["tryFetchSitemapLinks"] = jest
.fn()
.mockResolvedValue([
initialUrl,
initialUrl + "/page1",
initialUrl + "/page2",
initialUrl + "/page3",
]);
const filteredLinks = crawler["filterLinks"](
[
initialUrl,
initialUrl + "/page1",
initialUrl + "/page2",
initialUrl + "/page3",
],
limit,
10,
);
expect(filteredLinks.length).toBe(limit); // Check if the number of results respects the limit
expect(filteredLinks).toEqual([initialUrl, initialUrl + "/page1"]);
});
});
|