ingest/test_parse_books.py

"""What each parsing pattern matches, using lines from the books themselves.

    uv run pytest
"""

import pytest

from ingest.parse_books import (
    CALLOUT,
    FENCE,
    HEADING,
    HIDDEN_LINE,
    LEGACY_ANCHOR,
    LINK_DEFINITION,
    LISTING_OPEN,
    QUOTE,
    RULE_MARKER,
    strip_emphasis,
    strip_links,
)
from rag.config import SOURCES_DIR


def test_quote():
    assert QUOTE.match("> NOTE: There's a `transparent_unions` nightly feature")  # nomicon/other-reprs.md
    assert QUOTE.match(">> nested")
    assert not QUOTE.match("a > b")


def test_fence():
    assert FENCE.match("```rust")  # nomicon/other-reprs.md
    assert FENCE.match("```rust,ignore")  # nomicon/aliasing.md
    assert FENCE.match("```rust,compile_fail")  # nomicon/dropck.md
    assert FENCE.match("```")

    tagged = FENCE.match("```rust,ignore")
    assert tagged and tagged.group("info") == "rust,ignore"
    assert not FENCE.match("```rust inline``` text")


def test_heading():
    assert HEADING.match("# Working With Uninitialized Memory")  # nomicon/uninitialized.md
    assert HEADING.match("## repr(C)")  # nomicon/other-reprs.md

    subsection = HEADING.match("## repr(C)")
    assert subsection and subsection.group("hashes") == "##"
    assert not HEADING.match("#no space")


def test_hidden_line_keeps_rust_attributes():
    """Only ever applied inside a Rust fence, since it also matches an h1."""
    assert HIDDEN_LINE.match("# use std::mem::size_of;")  # nomicon/other-reprs.md
    assert HIDDEN_LINE.match("# use libc::{c_int, size_t};")  # nomicon/ffi.md
    assert HIDDEN_LINE.match("#")
    assert not HIDDEN_LINE.match("#[derive(Debug)]")
    assert not HIDDEN_LINE.match("#![allow(dead_code)]")
    assert HIDDEN_LINE.match("# Alternative representations")


def test_rule_marker():
    assert RULE_MARKER.match("r[macro.proc]")  # reference/procedural-macros.md
    assert RULE_MARKER.match("r[macro.proc.intro]")  # reference/procedural-macros.md
    assert not RULE_MARKER.match("r[macro.proc] and then prose")


def test_link_definition():
    assert LINK_DEFINITION.match("[drop flags]: drop-flags.html")  # nomicon/other-reprs.md
    assert LINK_DEFINITION.match("[ub loads]: https://github.com/rust-lang/rust/issues/27060")
    assert not LINK_DEFINITION.match("[^1]: a footnote")
    assert not LINK_DEFINITION.match("see [drop flags]: inline")


def test_legacy_anchor():
    assert LEGACY_ANCHOR.match('<a id="lifetime-annotations-in-function-signatures"></a>')  # book/ch10-03
    assert not LEGACY_ANCHOR.match('<a href="page.html">text</a>')


def test_callout_is_given_a_blockquote_stripped_line():
    """The books write these as `> [!NOTE]`, and the parser strips the prefix first."""
    assert CALLOUT.match(QUOTE.sub("", "> [!NOTE]"))  # reference/procedural-macros.md
    assert CALLOUT.match(QUOTE.sub("", "> [!EXAMPLE]"))
    assert not CALLOUT.match("[!NOTE] with trailing prose")


def test_listing_open():
    assert LISTING_OPEN.search('<Listing file-name="src/main.rs">')  # book/ch07-02
    assert LISTING_OPEN.search('<Listing number="7-1" caption="A module">')  # book/ch07-02
    assert not LISTING_OPEN.search("</Listing>")


def test_emphasis_leaves_snake_case_alone():
    """Two identifiers must not pair their underscores into one emphasis span."""
    for line in ("map_err and filter_map", "c_int, size_t", "to_string and from_str"):
        assert strip_emphasis(line) == line


def test_emphasis_still_strips_real_emphasis():
    assert strip_emphasis("the _outlives_ relationship") == "the outlives relationship"  # nomicon/dropck.md
    assert strip_emphasis("**bold** and *italic*") == "bold and italic"


def test_strip_links_keeps_the_text():
    assert strip_links("[Concurrency Chapter](concurrency.html).") == "Concurrency Chapter."  # nomicon/aliasing.md
    assert strip_links("See [the RFC][really-tagged] for details.") == "See the RFC for details."  # nomicon/other-reprs.md


def test_strip_links_spans_a_wrapped_line():
    """These books wrap at 80 columns, so a link often wraps over two lines."""
    assert strip_links("see [the tracking\nissue][tracking] now") == "see the tracking\nissue now"


CORPUS = sorted(SOURCES_DIR.rglob("*.md")) if SOURCES_DIR.exists() else []


@pytest.mark.skipif(not CORPUS, reason="run ingest.fetch_sources first")
def test_every_pattern_still_matches_the_books():
    """A pattern that matches nothing means the books changed their markup."""
    checks = {
        "QUOTE": QUOTE.match,
        "FENCE": FENCE.match,
        "HEADING": HEADING.match,
        "HIDDEN_LINE": HIDDEN_LINE.match,
        "RULE_MARKER": RULE_MARKER.match,
        "LINK_DEFINITION": LINK_DEFINITION.match,
        "LEGACY_ANCHOR": LEGACY_ANCHOR.match,
        "LISTING_OPEN": LISTING_OPEN.search,
        "CALLOUT": lambda line: CALLOUT.match(QUOTE.sub("", line)),
    }
    unmatched = set(checks)
    for page in CORPUS:
        for raw in page.read_text(encoding="utf-8").split("\n"):
            line = raw.strip()
            if line:
                unmatched -= {name for name in unmatched if checks[name](line)}
        if not unmatched:
            return
    assert not unmatched, f"no longer found in the books: {sorted(unmatched)}"