"""What each parsing pattern matches, using lines from the books themselves. uv run pytest """ import pytest from ingest.parse_books import ( CALLOUT, FENCE, HEADING, HIDDEN_LINE, LEGACY_ANCHOR, LINK_DEFINITION, LISTING_OPEN, QUOTE, RULE_MARKER, strip_emphasis, strip_links, ) from rag.config import SOURCES_DIR def test_quote(): assert QUOTE.match("> NOTE: There's a `transparent_unions` nightly feature") # nomicon/other-reprs.md assert QUOTE.match(">> nested") assert not QUOTE.match("a > b") def test_fence(): assert FENCE.match("```rust") # nomicon/other-reprs.md assert FENCE.match("```rust,ignore") # nomicon/aliasing.md assert FENCE.match("```rust,compile_fail") # nomicon/dropck.md assert FENCE.match("```") tagged = FENCE.match("```rust,ignore") assert tagged and tagged.group("info") == "rust,ignore" assert not FENCE.match("```rust inline``` text") def test_heading(): assert HEADING.match("# Working With Uninitialized Memory") # nomicon/uninitialized.md assert HEADING.match("## repr(C)") # nomicon/other-reprs.md subsection = HEADING.match("## repr(C)") assert subsection and subsection.group("hashes") == "##" assert not HEADING.match("#no space") def test_hidden_line_keeps_rust_attributes(): """Only ever applied inside a Rust fence, since it also matches an h1.""" assert HIDDEN_LINE.match("# use std::mem::size_of;") # nomicon/other-reprs.md assert HIDDEN_LINE.match("# use libc::{c_int, size_t};") # nomicon/ffi.md assert HIDDEN_LINE.match("#") assert not HIDDEN_LINE.match("#[derive(Debug)]") assert not HIDDEN_LINE.match("#![allow(dead_code)]") assert HIDDEN_LINE.match("# Alternative representations") def test_rule_marker(): assert RULE_MARKER.match("r[macro.proc]") # reference/procedural-macros.md assert RULE_MARKER.match("r[macro.proc.intro]") # reference/procedural-macros.md assert not RULE_MARKER.match("r[macro.proc] and then prose") def test_link_definition(): assert LINK_DEFINITION.match("[drop flags]: drop-flags.html") # nomicon/other-reprs.md assert LINK_DEFINITION.match("[ub loads]: https://github.com/rust-lang/rust/issues/27060") assert not LINK_DEFINITION.match("[^1]: a footnote") assert not LINK_DEFINITION.match("see [drop flags]: inline") def test_legacy_anchor(): assert LEGACY_ANCHOR.match('') # book/ch10-03 assert not LEGACY_ANCHOR.match('text') def test_callout_is_given_a_blockquote_stripped_line(): """The books write these as `> [!NOTE]`, and the parser strips the prefix first.""" assert CALLOUT.match(QUOTE.sub("", "> [!NOTE]")) # reference/procedural-macros.md assert CALLOUT.match(QUOTE.sub("", "> [!EXAMPLE]")) assert not CALLOUT.match("[!NOTE] with trailing prose") def test_listing_open(): assert LISTING_OPEN.search('') # book/ch07-02 assert LISTING_OPEN.search('') # book/ch07-02 assert not LISTING_OPEN.search("") def test_emphasis_leaves_snake_case_alone(): """Two identifiers must not pair their underscores into one emphasis span.""" for line in ("map_err and filter_map", "c_int, size_t", "to_string and from_str"): assert strip_emphasis(line) == line def test_emphasis_still_strips_real_emphasis(): assert strip_emphasis("the _outlives_ relationship") == "the outlives relationship" # nomicon/dropck.md assert strip_emphasis("**bold** and *italic*") == "bold and italic" def test_strip_links_keeps_the_text(): assert strip_links("[Concurrency Chapter](concurrency.html).") == "Concurrency Chapter." # nomicon/aliasing.md assert strip_links("See [the RFC][really-tagged] for details.") == "See the RFC for details." # nomicon/other-reprs.md def test_strip_links_spans_a_wrapped_line(): """These books wrap at 80 columns, so a link often wraps over two lines.""" assert strip_links("see [the tracking\nissue][tracking] now") == "see the tracking\nissue now" CORPUS = sorted(SOURCES_DIR.rglob("*.md")) if SOURCES_DIR.exists() else [] @pytest.mark.skipif(not CORPUS, reason="run ingest.fetch_sources first") def test_every_pattern_still_matches_the_books(): """A pattern that matches nothing means the books changed their markup.""" checks = { "QUOTE": QUOTE.match, "FENCE": FENCE.match, "HEADING": HEADING.match, "HIDDEN_LINE": HIDDEN_LINE.match, "RULE_MARKER": RULE_MARKER.match, "LINK_DEFINITION": LINK_DEFINITION.match, "LEGACY_ANCHOR": LEGACY_ANCHOR.match, "LISTING_OPEN": LISTING_OPEN.search, "CALLOUT": lambda line: CALLOUT.match(QUOTE.sub("", line)), } unmatched = set(checks) for page in CORPUS: for raw in page.read_text(encoding="utf-8").split("\n"): line = raw.strip() if line: unmatched -= {name for name in unmatched if checks[name](line)} if not unmatched: return assert not unmatched, f"no longer found in the books: {sorted(unmatched)}"