"""Deadline ingest — robust parse, mapper, post-import backfill.""" from __future__ import annotations import json from pathlib import Path import pytest from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker from sqlalchemy.pool import StaticPool # ── Unit: parser formats ───────────────────────────────────────────────────── @pytest.mark.parametrize( "raw,expected", [ ("2026-12-31", "2026-12-31"), ("31.12.2026", "2026-12-31"), ("31-12-2026", "2026-12-31"), ("18/06/2026", "2026-06-18"), ("15 maja 2026", "2026-05-15"), ("7 stycznia 2025", "2025-01-07"), ("29 czerwca 2026", "2026-06-29"), ("do 30.06.2026", "2026-06-30"), ("1.03.2026 - 30.06.2026", "2026-06-30"), ("Q2 2026", "2026-06-15"), ("ciągły", None), ("", None), ("19.06.202", None), # truncated garbage ("12.2026", "2026-12-31"), # month.year → end of month ("2026-03", "2026-03-31"), # year-month → end of month ("deadline 2026-07-01T23:59:59", "2026-07-01"), # ISO datetime, not 2001-07-26 ("Call closes 15 March 2026", "2026-03-15"), ], ) def test_parse_deadline_formats(raw, expected): from core.search.deadline_utils import parse_deadline_robust got = parse_deadline_robust(raw, "") if expected is None: assert got is None else: assert got is not None assert got.isoformat() == expected def test_iso_datetime_not_misparsed_as_eu_fragment(): """Regression: '2026-07-01T23:59:59' must not become 2001-07-26.""" from core.search.deadline_utils import parse_deadline_robust, deadline_to_iso assert parse_deadline_robust("2026-07-01T23:59:59", "").isoformat() == "2026-07-01" assert deadline_to_iso("deadline 2026-07-01T00:00:00Z") == "2026-07-01" # Past deadline must not be re-labeled active by parser alone assert parse_deadline_robust("2020-01-15", "").isoformat() == "2020-01-15" def test_extract_from_text_phrase(): from core.search.deadline_utils import extract_deadline_from_text hit = extract_deadline_from_text( "Nabór wniosków trwa do 15 września 2026 r. Więcej na stronie operatora." ) assert hit is not None assert hit.iso == "2026-09-15" assert hit.confidence >= 0.7 def test_extract_grant_deadline_primary_field(): from core.search.deadline_utils import extract_grant_deadline hit = extract_grant_deadline( { "nazwa": "Program X", "termin_do": "25 maja 2026", "opis_krotki": "bez daty", } ) assert hit is not None assert hit.iso == "2026-05-25" assert hit.source == "termin_do" def test_extract_grant_deadline_from_description_when_empty_field(): from core.search.deadline_utils import extract_grant_deadline hit = extract_grant_deadline( { "nazwa": "Program Y", "termin_do": "", "opis_krotki": "Składanie wniosków do 12.11.2026 w systemie LSI.", "status": "otwarty", } ) assert hit is not None assert hit.iso == "2026-11-12" def test_continuous_status_no_forced_deadline(): from core.search.deadline_utils import apply_deadline_to_grant_dict g = apply_deadline_to_grant_dict( { "name": "Program ciągły", "status": "ciągły", "description": "Nabór całoroczny 2026/2027", "termin_do": "", } ) # continuous without explicit termin_do stays empty assert g.get("deadline") in ("", None) # ── Mapper ─────────────────────────────────────────────────────────────────── def test_mapper_parses_polish_and_slash_dates(): from core.grants.wyszukiwarka_mapper import map_wyszukiwarka_entry e1 = map_wyszukiwarka_entry( { "id": "d1", "nazwa": "Test PL month", "operator": "PARP", "zrodlo": "PARP", "termin_do": "15 maja 2026", "status": "otwarty", "link_strona": "https://www.parp.gov.pl/x", "poziom_weryfikacji": "katalog", } ) assert e1 is not None assert e1["deadline"] == "2026-05-15" assert e1["confidence_on_dates"] >= 0.8 e2 = map_wyszukiwarka_entry( { "id": "d2", "nazwa": "Test slash", "operator": "NCBR", "zrodlo": "NCBR", "termin_do": "18/06/2026", "status": "otwarty", "link_strona": "https://www.ncbr.gov.pl/x", "poziom_weryfikacji": "katalog", } ) assert e2["deadline"] == "2026-06-18" def test_mapper_backfills_from_opis(): from core.grants.wyszukiwarka_mapper import map_wyszukiwarka_entry e = map_wyszukiwarka_entry( { "id": "d3", "nazwa": "Program bez termin_do", "operator": "BGK", "zrodlo": "BGK", "termin_do": "", "status": "otwarty", "opis_krotki": "Termin naboru: do 31.08.2026. Wnioski przez portal.", "link_strona": "https://www.bgk.pl/x", "poziom_weryfikacji": "katalog", } ) assert e is not None assert e["deadline"] == "2026-08-31" assert e.get("deadline_source", "").startswith("text") or e.get("deadline_source") # ── Dump coverage improvement ──────────────────────────────────────────────── def test_dump_deadline_coverage_improves(): """On local wyszukiwarka dump: robust extract fills more than raw termin_do.""" path = Path(__file__).resolve().parents[1] / "data" / "wyszukiwarka" / "dotacje-latest.json" if not path.exists() or path.stat().st_size < 5000: pytest.skip("no dump") data = json.loads(path.read_text(encoding="utf-8")) grants = data.get("grants") or [] if len(grants) < 50: pytest.skip("tiny dump") from core.search.deadline_utils import backfill_deadlines_report report = backfill_deadlines_report(grants) # Parseable coverage must not drop; expect lift vs raw-parseable baseline assert report["with_deadline_after"] >= report["with_deadline_before_parseable"] assert report["pct_after"] >= report["pct_before_parseable"] # Real dump: extract beats parseable-only baseline; text fills additional rows if report["total"] > 500: assert report["pct_after"] >= 29.0 assert report["newly_filled"] >= 15 assert report["with_deadline_after"] > report["with_deadline_before_parseable"] # ── Post-import DB backfill ────────────────────────────────────────────────── def test_post_import_fills_row_deadline(): from core.subscription.db import Base import core.grants.models # noqa: F401 from core.grants.models import Grant from core.grants.post_import_pipeline import normalize_catalog_defaults engine = create_engine( "sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool, ) Base.metadata.create_all(bind=engine) Session = sessionmaker(bind=engine) db = Session() try: row = Grant( id="g-dl-1", source_id="g-dl-1", name="Program bez deadline w kolumnie", program="PARP", status="active", source="wyszukiwarka:PARP", deadline="", description="Składanie wniosków do 20.10.2026 w generatorze wniosków.", raw_data={ "termin_do": "20 października 2026", "opis_krotki": "x", }, ) db.add(row) db.commit() stats = normalize_catalog_defaults(db) db.refresh(row) assert row.deadline == "2026-10-20" assert stats.get("deadlines_filled", 0) >= 1 or stats.get("rows_updated", 0) >= 1 finally: db.close() Base.metadata.drop_all(bind=engine) def test_post_import_normalizes_slash_deadline(): from core.subscription.db import Base import core.grants.models # noqa: F401 from core.grants.models import Grant from core.grants.post_import_pipeline import normalize_catalog_defaults engine = create_engine( "sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool, ) Base.metadata.create_all(bind=engine) Session = sessionmaker(bind=engine) db = Session() try: row = Grant( id="g-dl-2", source_id="g-dl-2", name="Slash deadline", program="NCBR", status="active", source="wyszukiwarka:NCBR", deadline="18/06/2026", description="", raw_data={"termin_do": "18/06/2026"}, ) db.add(row) db.commit() normalize_catalog_defaults(db) db.refresh(row) assert row.deadline == "2026-06-18" finally: db.close() Base.metadata.drop_all(bind=engine)