tests/test_enf.py

"""Unit tests for the ENF manual ingester (canlex/enf.py). Offline only.

The ingester rebuilds enf.json from a fresh probe of the canada.ca dam
directory, so a renamed or unreachable chapter silently disappears from the
corpus -- the failure mode that emptied dmemos.json on 2026-07-27. These cover
the identity preserve_failed keys on and the failed-set it derives; the write
guard itself is covered by tests/test_common.py.
"""
import subprocess
import unittest
import urllib.error

from canlex import enf


def _pages(n, headings=7):
    """A stand-in for pypdf's per-page text: a cover page plus numbered
    headings, enough of them to take the heading-segmentation path."""
    body = "Body text for this section. " * 8
    return [f"ENF {n}\nLast updated: 2024-03-12\n",
            "\n".join(f"{i}. Heading number {i}\n{body}"
                      for i in range(1, headings + 1))]


class ChapterIdentityTests(unittest.TestCase):
    """preserve_failed keys on act_code, so the generator must keep emitting
    exactly the identity the failed-set is built from."""

    def test_every_chunk_of_a_chapter_shares_one_act_code(self):
        chunks = enf._chunks_for(5, _pages(5), "https://x/enf05-eng.pdf")
        self.assertTrue(chunks)
        self.assertEqual({c["act_code"] for c in chunks}, {"ENF-5"})

    def test_failed_set_matches_the_generated_act_code(self):
        stored = enf._chunks_for(5, _pages(5), "https://x/enf05-eng.pdf")
        chunks, preserved = enf.preserve_failed([], [(5, "fetch failed")],
                                                stored)
        self.assertEqual(preserved, ["ENF-5"])
        self.assertEqual(len(chunks), len(stored))


class PreserveFailedTests(unittest.TestCase):
    STORED = [{"act_code": "ENF-14", "text": "rehabilitation"},
              {"act_code": "ENF-14", "text": "part two"},
              {"act_code": "ENF-9", "text": "judicial review"}]

    def test_unfetchable_chapter_keeps_its_last_good_chunks(self):
        chunks, preserved = enf.preserve_failed(
            [], [(14, "fetch failed: HTTPError: 404")], self.STORED)
        self.assertEqual(preserved, ["ENF-14"])
        self.assertEqual([c["text"] for c in chunks],
                         ["rehabilitation", "part two"])

    def test_chapter_dropped_on_purpose_is_not_resurrected(self):
        # ENF 9 was not probed this run (removed from _CHAPTERS), so it never
        # failed and must not come back.
        chunks, preserved = enf.preserve_failed(
            [], [(14, "no text layer and no enf14-ocr.txt")], self.STORED)
        self.assertNotIn("ENF-9", preserved)
        self.assertNotIn("judicial review", [c["text"] for c in chunks])

    def test_freshly_parsed_chunks_win_over_stored(self):
        fresh = [{"act_code": "ENF-14", "text": "new text"}]
        chunks, preserved = enf.preserve_failed(
            fresh, [(14, "parse failed: PdfReadError: x")], self.STORED)
        self.assertEqual((chunks, preserved), (fresh, []))

    def test_chapter_we_never_had_cannot_be_preserved(self):
        chunks, preserved = enf.preserve_failed([], [(31, "fetch failed")],
                                                self.STORED)
        self.assertEqual((chunks, preserved), ([], []))

    def test_a_clean_run_preserves_nothing(self):
        fresh = [{"act_code": "ENF-14", "text": "x"}]
        chunks, preserved = enf.preserve_failed(fresh, [], self.STORED)
        self.assertEqual((chunks, preserved), (fresh, []))

    def test_every_failing_chapter_is_preserved_independently(self):
        stored = self.STORED + [{"act_code": "ENF-3", "text": "hearings"}]
        _chunks, preserved = enf.preserve_failed(
            [], [(3, "fetch failed"), (14, "parse failed")], stored)
        self.assertEqual(preserved, ["ENF-14", "ENF-3"])  # sorted by identity


class MissingUpstreamTests(unittest.TestCase):
    """A 404 is this module's retirement signal, not a failure to preserve
    through -- _CHAPTERS is a blind range(1, 41) probe, so most numbers 404 on
    a healthy run and a chapter IRCC withdraws starts 404ing too."""

    def _http_error(self, code):
        # HTTPError is file-like; close it or the suite emits ResourceWarnings.
        exc = urllib.error.HTTPError("http://x", code, "boom", {}, None)
        self.addCleanup(exc.close)
        return exc

    def test_http_404_is_a_retirement(self):
        self.assertTrue(enf.is_missing_upstream(self._http_error(404)))

    def test_other_http_statuses_are_real_failures(self):
        for code in (403, 500, 502, 503):
            self.assertFalse(enf.is_missing_upstream(self._http_error(code)))

    def test_network_errors_are_real_failures(self):
        self.assertFalse(enf.is_missing_upstream(urllib.error.URLError("dns")))
        self.assertFalse(enf.is_missing_upstream(TimeoutError("timed out")))

    def test_powershell_404_is_read_off_stderr(self):
        # The canada.ca path shells out, so the status only appears in prose.
        exc = subprocess.CalledProcessError(1, "powershell")
        exc.stderr = ("Invoke-WebRequest : The remote server returned an "
                      "error: (404) Not Found.")
        self.assertTrue(enf.is_missing_upstream(exc))

    def test_powershell_bytes_stderr_is_decoded(self):
        exc = subprocess.CalledProcessError(1, "powershell")
        exc.stderr = b"error: (404) Not Found."
        self.assertTrue(enf.is_missing_upstream(exc))

    def test_powershell_other_failure_is_preserved_through(self):
        exc = subprocess.CalledProcessError(1, "powershell")
        exc.stderr = "The underlying connection was closed: TLS failure."
        self.assertFalse(enf.is_missing_upstream(exc))

    def test_no_stderr_at_all_counts_as_a_failure(self):
        # Unknown cause -> keep the stored chapter; losing in-force guidance
        # is the worse error.
        self.assertFalse(enf.is_missing_upstream(RuntimeError("who knows")))


if __name__ == "__main__":
    unittest.main()