| from astroparse_api.parse import remove_repeating_lines, repair_hyphenation, strip_page_artifacts | |
| def test_removes_running_headers(): | |
| pages = [["The Astrophysical Journal 1", "real content A"], | |
| ["The Astrophysical Journal 2", "real content B"], | |
| ["The Astrophysical Journal 3", "real content C"], | |
| ["unique line", "real content D"]] | |
| cleaned = remove_repeating_lines(pages) | |
| flat = [l for p in cleaned for l in p] | |
| assert "real content A" in flat and "unique line" in flat | |
| assert not any("Astrophysical Journal" in l for l in flat) | |
| def test_repairs_hyphenated_breaks(): | |
| assert repair_hyphenation("measure-\nments of the") == "measurements of the" | |
| assert repair_hyphenation("non-\nparametric") == "nonparametric" | |
| def test_strips_page_numbers_and_stamps(): | |
| lines = ["12", "arXiv:2402.08696v2 [astro-ph.GA] 9 Dec 2024", "Real sentence here."] | |
| assert strip_page_artifacts(lines) == ["Real sentence here."] | |
| def test_strip_page_artifacts_preserves_blank_lines(): | |
| lines = ["First para.", "", "12", "Second para."] | |
| assert strip_page_artifacts(lines) == ["First para.", "", "Second para."] | |
| def test_demarkdown_negative_exponent(): | |
| from astroparse_api.parse import demarkdown | |
| assert "10⁻⁵" in demarkdown("a density of 10[-5] cm^-3") | |
| assert "10⁻¹²" in demarkdown("flux of 10[-12] W/m²") | |
| def test_demarkdown_cid_stripped(): | |
| from astroparse_api.parse import demarkdown | |
| result = demarkdown("some (cid:45) text here") | |
| assert "(cid:" not in result | |
| assert "some" in result and "text here" in result | |