tests/test_agreement.py

"""Unit tests for the collective-agreement ingester (canlex/agreement.py).

Offline only. Ingestion is a full rebuild, so the failure that cost the D-memo
corpus on 2026-07-27 -- an upstream page that changes shape, scrapes to nothing
and is written straight over the good chunks -- applies here too: canada.ca is
one redesign away from parsing to zero Articles.
"""
import unittest

from canlex import agreement


class ParseAgreementTests(unittest.TestCase):
    def test_redesigned_page_parses_to_nothing(self):
        # What a layout change looks like: no <main>, so no Articles. ingest()
        # records this as a failure rather than writing the empty result.
        self.assertEqual(agreement.parse_agreement("<div>hello</div>", "FB"), [])

    def test_article_heading_becomes_a_numbered_chunk(self):
        html = ("<main><h3>Article 17: discipline</h3>"
                "<p>An employee may be disciplined.</p></main>")
        chunk, = agreement.parse_agreement(html, "FB")
        self.assertEqual(chunk["section"], "17")
        self.assertEqual(chunk["marginal_note"], "discipline")
        self.assertEqual(chunk["act_code"], "FB")

    def test_every_chunk_carries_its_agreement_code(self):
        # preserve_failed keys on act_code, so it has to be on every chunk.
        html = ("<main><h3>Article 1: purpose</h3><p>one</p>"
                "<h3>Appendix A</h3><p>two</p></main>")
        chunks = agreement.parse_agreement(html, "FB")
        self.assertEqual({c["act_code"] for c in chunks}, {"FB"})


class PreserveFailedTests(unittest.TestCase):
    STORED = [{"act_code": "FB", "text": "hours of work"},
              {"act_code": "FB", "text": "overtime"},
              {"act_code": "PA", "text": "leave"}]

    def test_failed_agreement_keeps_its_last_good_chunks(self):
        failures = [("FB", "no content parsed")]
        chunks, preserved = agreement.preserve_failed([], failures, self.STORED)
        self.assertEqual(preserved, ["FB"])
        self.assertEqual([c["text"] for c in chunks],
                         ["hours of work", "overtime"])

    def test_fetch_error_counts_as_a_failure_too(self):
        failures = [("FB", "URLError: timed out")]
        _chunks, preserved = agreement.preserve_failed([], failures, self.STORED)
        self.assertEqual(preserved, ["FB"])

    def test_agreement_dropped_on_purpose_is_not_resurrected(self):
        # PA is stored but no longer in AGREEMENTS, so it never failed: a
        # retired agreement must not come back through the preserve path.
        chunks, preserved = agreement.preserve_failed(
            [], [("FB", "no content parsed")], self.STORED)
        self.assertNotIn("PA", preserved)
        self.assertNotIn("leave", [c["text"] for c in chunks])

    def test_freshly_parsed_chunks_win_over_stored(self):
        fresh = [{"act_code": "FB", "text": "new text"}]
        chunks, preserved = agreement.preserve_failed(
            fresh, [("FB", "no content parsed")], self.STORED)
        self.assertEqual((chunks, preserved), (fresh, []))

    def test_agreement_we_never_stored_cannot_be_preserved(self):
        chunks, preserved = agreement.preserve_failed(
            [], [("SV", "URLError: timed out")], self.STORED)
        self.assertEqual((chunks, preserved), ([], []))

    def test_nothing_happens_without_failures(self):
        fresh = [{"act_code": "FB", "text": "x"}]
        chunks, preserved = agreement.preserve_failed(fresh, [], self.STORED)
        self.assertEqual((chunks, preserved), (fresh, []))


if __name__ == "__main__":
    unittest.main()