backend/tests/test_deadline_ingest.py

"""Deadline ingest — robust parse, mapper, post-import backfill."""
from __future__ import annotations

import json
from pathlib import Path

import pytest
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from sqlalchemy.pool import StaticPool


# ── Unit: parser formats ─────────────────────────────────────────────────────


@pytest.mark.parametrize(
    "raw,expected",
    [
        ("2026-12-31", "2026-12-31"),
        ("31.12.2026", "2026-12-31"),
        ("31-12-2026", "2026-12-31"),
        ("18/06/2026", "2026-06-18"),
        ("15 maja 2026", "2026-05-15"),
        ("7 stycznia 2025", "2025-01-07"),
        ("29 czerwca 2026", "2026-06-29"),
        ("do 30.06.2026", "2026-06-30"),
        ("1.03.2026 - 30.06.2026", "2026-06-30"),
        ("Q2 2026", "2026-06-15"),
        ("ciągły", None),
        ("", None),
        ("19.06.202", None),  # truncated garbage
        ("12.2026", "2026-12-31"),  # month.year → end of month
        ("2026-03", "2026-03-31"),  # year-month → end of month
        ("deadline 2026-07-01T23:59:59", "2026-07-01"),  # ISO datetime, not 2001-07-26
        ("Call closes 15 March 2026", "2026-03-15"),
    ],
)
def test_parse_deadline_formats(raw, expected):
    from core.search.deadline_utils import parse_deadline_robust

    got = parse_deadline_robust(raw, "")
    if expected is None:
        assert got is None
    else:
        assert got is not None
        assert got.isoformat() == expected


def test_iso_datetime_not_misparsed_as_eu_fragment():
    """Regression: '2026-07-01T23:59:59' must not become 2001-07-26."""
    from core.search.deadline_utils import parse_deadline_robust, deadline_to_iso

    assert parse_deadline_robust("2026-07-01T23:59:59", "").isoformat() == "2026-07-01"
    assert deadline_to_iso("deadline 2026-07-01T00:00:00Z") == "2026-07-01"
    # Past deadline must not be re-labeled active by parser alone
    assert parse_deadline_robust("2020-01-15", "").isoformat() == "2020-01-15"


def test_extract_from_text_phrase():
    from core.search.deadline_utils import extract_deadline_from_text

    hit = extract_deadline_from_text(
        "Nabór wniosków trwa do 15 września 2026 r. Więcej na stronie operatora."
    )
    assert hit is not None
    assert hit.iso == "2026-09-15"
    assert hit.confidence >= 0.7


def test_extract_grant_deadline_primary_field():
    from core.search.deadline_utils import extract_grant_deadline

    hit = extract_grant_deadline(
        {
            "nazwa": "Program X",
            "termin_do": "25 maja 2026",
            "opis_krotki": "bez daty",
        }
    )
    assert hit is not None
    assert hit.iso == "2026-05-25"
    assert hit.source == "termin_do"


def test_extract_grant_deadline_from_description_when_empty_field():
    from core.search.deadline_utils import extract_grant_deadline

    hit = extract_grant_deadline(
        {
            "nazwa": "Program Y",
            "termin_do": "",
            "opis_krotki": "Składanie wniosków do 12.11.2026 w systemie LSI.",
            "status": "otwarty",
        }
    )
    assert hit is not None
    assert hit.iso == "2026-11-12"


def test_continuous_status_no_forced_deadline():
    from core.search.deadline_utils import apply_deadline_to_grant_dict

    g = apply_deadline_to_grant_dict(
        {
            "name": "Program ciągły",
            "status": "ciągły",
            "description": "Nabór całoroczny 2026/2027",
            "termin_do": "",
        }
    )
    # continuous without explicit termin_do stays empty
    assert g.get("deadline") in ("", None)


# ── Mapper ───────────────────────────────────────────────────────────────────


def test_mapper_parses_polish_and_slash_dates():
    from core.grants.wyszukiwarka_mapper import map_wyszukiwarka_entry

    e1 = map_wyszukiwarka_entry(
        {
            "id": "d1",
            "nazwa": "Test PL month",
            "operator": "PARP",
            "zrodlo": "PARP",
            "termin_do": "15 maja 2026",
            "status": "otwarty",
            "link_strona": "https://www.parp.gov.pl/x",
            "poziom_weryfikacji": "katalog",
        }
    )
    assert e1 is not None
    assert e1["deadline"] == "2026-05-15"
    assert e1["confidence_on_dates"] >= 0.8

    e2 = map_wyszukiwarka_entry(
        {
            "id": "d2",
            "nazwa": "Test slash",
            "operator": "NCBR",
            "zrodlo": "NCBR",
            "termin_do": "18/06/2026",
            "status": "otwarty",
            "link_strona": "https://www.ncbr.gov.pl/x",
            "poziom_weryfikacji": "katalog",
        }
    )
    assert e2["deadline"] == "2026-06-18"


def test_mapper_backfills_from_opis():
    from core.grants.wyszukiwarka_mapper import map_wyszukiwarka_entry

    e = map_wyszukiwarka_entry(
        {
            "id": "d3",
            "nazwa": "Program bez termin_do",
            "operator": "BGK",
            "zrodlo": "BGK",
            "termin_do": "",
            "status": "otwarty",
            "opis_krotki": "Termin naboru: do 31.08.2026. Wnioski przez portal.",
            "link_strona": "https://www.bgk.pl/x",
            "poziom_weryfikacji": "katalog",
        }
    )
    assert e is not None
    assert e["deadline"] == "2026-08-31"
    assert e.get("deadline_source", "").startswith("text") or e.get("deadline_source")


# ── Dump coverage improvement ────────────────────────────────────────────────


def test_dump_deadline_coverage_improves():
    """On local wyszukiwarka dump: robust extract fills more than raw termin_do."""
    path = Path(__file__).resolve().parents[1] / "data" / "wyszukiwarka" / "dotacje-latest.json"
    if not path.exists() or path.stat().st_size < 5000:
        pytest.skip("no dump")
    data = json.loads(path.read_text(encoding="utf-8"))
    grants = data.get("grants") or []
    if len(grants) < 50:
        pytest.skip("tiny dump")

    from core.search.deadline_utils import backfill_deadlines_report

    report = backfill_deadlines_report(grants)
    # Parseable coverage must not drop; expect lift vs raw-parseable baseline
    assert report["with_deadline_after"] >= report["with_deadline_before_parseable"]
    assert report["pct_after"] >= report["pct_before_parseable"]
    # Real dump: extract beats parseable-only baseline; text fills additional rows
    if report["total"] > 500:
        assert report["pct_after"] >= 29.0
        assert report["newly_filled"] >= 15
        assert report["with_deadline_after"] > report["with_deadline_before_parseable"]


# ── Post-import DB backfill ──────────────────────────────────────────────────


def test_post_import_fills_row_deadline():
    from core.subscription.db import Base
    import core.grants.models  # noqa: F401
    from core.grants.models import Grant
    from core.grants.post_import_pipeline import normalize_catalog_defaults

    engine = create_engine(
        "sqlite://",
        connect_args={"check_same_thread": False},
        poolclass=StaticPool,
    )
    Base.metadata.create_all(bind=engine)
    Session = sessionmaker(bind=engine)
    db = Session()
    try:
        row = Grant(
            id="g-dl-1",
            source_id="g-dl-1",
            name="Program bez deadline w kolumnie",
            program="PARP",
            status="active",
            source="wyszukiwarka:PARP",
            deadline="",
            description="Składanie wniosków do 20.10.2026 w generatorze wniosków.",
            raw_data={
                "termin_do": "20 października 2026",
                "opis_krotki": "x",
            },
        )
        db.add(row)
        db.commit()

        stats = normalize_catalog_defaults(db)
        db.refresh(row)
        assert row.deadline == "2026-10-20"
        assert stats.get("deadlines_filled", 0) >= 1 or stats.get("rows_updated", 0) >= 1
    finally:
        db.close()
        Base.metadata.drop_all(bind=engine)


def test_post_import_normalizes_slash_deadline():
    from core.subscription.db import Base
    import core.grants.models  # noqa: F401
    from core.grants.models import Grant
    from core.grants.post_import_pipeline import normalize_catalog_defaults

    engine = create_engine(
        "sqlite://",
        connect_args={"check_same_thread": False},
        poolclass=StaticPool,
    )
    Base.metadata.create_all(bind=engine)
    Session = sessionmaker(bind=engine)
    db = Session()
    try:
        row = Grant(
            id="g-dl-2",
            source_id="g-dl-2",
            name="Slash deadline",
            program="NCBR",
            status="active",
            source="wyszukiwarka:NCBR",
            deadline="18/06/2026",
            description="",
            raw_data={"termin_do": "18/06/2026"},
        )
        db.add(row)
        db.commit()
        normalize_catalog_defaults(db)
        db.refresh(row)
        assert row.deadline == "2026-06-18"
    finally:
        db.close()
        Base.metadata.drop_all(bind=engine)