Spaces:
Sleeping
Sleeping
malteos
Add content_languages filter; vendor index SQL to drop cdx_toolkit dependency
8eaca44 unverified | from src import buckets, config | |
| def test_parse_targets(): | |
| assert buckets.parse_targets("a.com, b.org\nc.net d.io") == ["a.com", "b.org", "c.net", "d.io"] | |
| assert buckets.parse_targets("") == [] | |
| assert buckets.parse_targets(" Example.COM ") == ["example.com"] | |
| def test_validate_targets(): | |
| assert buckets.validate_targets([], [])[0] is False | |
| assert buckets.validate_targets([], ["example.com"])[0] is True | |
| assert buckets.validate_targets(["bad host"], [])[0] is False # space | |
| assert buckets.validate_targets([], ["nodot"])[0] is False # no dot | |
| def test_validate_languages(): | |
| assert buckets.validate_languages([])[0] is True # optional filter | |
| assert buckets.validate_languages(["deu", "eng"])[0] is True | |
| ok, msg = buckets.validate_languages(["eng", "xxx"]) | |
| assert not ok and "xxx" in msg | |
| def test_language_choices(): | |
| choices = config.language_choices() | |
| assert ("German (deu)", "deu") in choices | |
| assert ("English (eng)", "eng") in choices | |
| codes = [c for _, c in choices] | |
| assert len(codes) == len(set(codes)) # no duplicate codes | |
| assert set(codes) == set(config.CONTENT_LANGUAGE_CODES) | |
| assert all(len(c) == 3 and c.isalpha() and c.islower() for c in codes) | |
| def test_validate_crawls(): | |
| assert buckets.validate_crawls([])[0] is False | |
| ok, msg = buckets.validate_crawls(["CC-MAIN-2026-25"]) | |
| assert ok and msg == "" | |
| ok, msg = buckets.validate_crawls([f"c{i}" for i in range(config.COST_GUARD_MAX_CRAWLS + 1)]) | |
| assert ok and "⚠️" in msg # allowed but warned | |
| def test_validate_target_bucket_format_errors_no_network(): | |
| # These return before any network call. | |
| assert buckets.validate_target_bucket("justname", "p")[0] is False | |
| assert buckets.validate_target_bucket("a/b/c", "p")[0] is False | |
| assert buckets.validate_target_bucket("me/bucket", "../escape")[0] is False | |
| assert buckets.validate_target_bucket("me/bucket", "/abs")[0] is False | |