tests/api/test_sync_catalog_bulk.py

"""`sync_catalog` rewritten from update_or_create()-per-item to bulk writes.

Against the remote Neon DB (us-east-1, no local replica), the old command did a
SELECT + INSERT/UPDATE round trip per item -- ~44 700 of them, all inside one
`@transaction.atomic` -- and after 70 minutes it still had not committed. These
tests pin the replacement contract: a single upfront SELECT to classify
create/update, batched bulk_create/bulk_update (one transaction per batch, not
one for the whole run), a cheap skip path for byte-identical rows so a routine
re-sync writes nothing, `--dry-run`, `--media-type`, and cache invalidation of
the `catalog_{type}` Redis key at the end (the per-process RAM cache in
CatalogService._cached_catalogs cannot be reached from here and is documented,
not silently ignored).
"""

import json
from io import StringIO

import pytest
from animetix.models import MediaItem
from django.core.cache import cache
from django.core.management import call_command


def _write_catalog(tmp_path, monkeypatch, filename, items):
    from animetix.management.commands import sync_catalog

    processed = tmp_path / "data" / "processed"
    processed.mkdir(parents=True, exist_ok=True)
    (processed / filename).write_text(json.dumps(items), encoding="utf-8")
    # Same trick the pre-existing sync_catalog tests use: abspath is mocked to
    # ignore its argument entirely, so both calls in the command (dirname(
    # abspath(__file__)) and abspath(join(...))) collapse onto tmp_path.
    monkeypatch.setattr(
        sync_catalog.os.path, "abspath", lambda _p: str(tmp_path), raising=False
    )


def _run(**kwargs):
    out = StringIO()
    call_command("sync_catalog", "--no-color", stdout=out, **kwargs)
    return out.getvalue()


@pytest.mark.django_db
def test_second_run_of_unchanged_data_writes_nothing(tmp_path, monkeypatch):
    data = [{"idMal": 1, "title": "Frieren", "popularity": 5000}]
    _write_catalog(tmp_path, monkeypatch, "clean_root_animes.json", data)

    _run()
    assert MediaItem.objects.count() == 1

    output = _run()

    assert "created=0" in output
    assert "updated=0" in output
    assert "skipped=1" in output
    assert MediaItem.objects.get(external_id="1").title == "Frieren"


@pytest.mark.django_db
def test_new_items_created_and_changed_items_updated(tmp_path, monkeypatch):
    data = [
        {"idMal": 1, "title": "Frieren", "popularity": 5000},
        {"idMal": 2, "title": "Old Title", "popularity": 10},
    ]
    _write_catalog(tmp_path, monkeypatch, "clean_root_animes.json", data)
    _run()

    updated_data = [
        {"idMal": 1, "title": "Frieren", "popularity": 5000},  # unchanged
        {"idMal": 2, "title": "New Title", "popularity": 999},  # changed
        {"idMal": 3, "title": "Brand New", "popularity": 42},  # new
    ]
    _write_catalog(tmp_path, monkeypatch, "clean_root_animes.json", updated_data)
    output = _run()

    assert "created=1" in output
    assert "updated=1" in output
    assert "skipped=1" in output

    assert MediaItem.objects.count() == 3
    changed = MediaItem.objects.get(external_id="2")
    assert changed.title == "New Title"
    assert changed.popularity == 999.0
    assert MediaItem.objects.get(external_id="3").title == "Brand New"


@pytest.mark.django_db
def test_dry_run_writes_nothing(tmp_path, monkeypatch):
    data = [{"idMal": 1, "title": "Frieren", "popularity": 5000}]
    _write_catalog(tmp_path, monkeypatch, "clean_root_animes.json", data)

    output = _run(dry_run=True)

    assert MediaItem.objects.count() == 0
    assert "created=1" in output


@pytest.mark.django_db
def test_dry_run_on_a_changed_row_reports_update_without_writing(tmp_path, monkeypatch):
    data = [{"idMal": 1, "title": "Frieren", "popularity": 5000}]
    _write_catalog(tmp_path, monkeypatch, "clean_root_animes.json", data)
    _run()

    changed = [{"idMal": 1, "title": "Frieren Season 2", "popularity": 5000}]
    _write_catalog(tmp_path, monkeypatch, "clean_root_animes.json", changed)
    output = _run(dry_run=True)

    assert "updated=1" in output
    # Dry run: the row in the DB must be untouched.
    assert MediaItem.objects.get(external_id="1").title == "Frieren"


@pytest.mark.django_db
def test_media_type_filter_syncs_only_that_type(tmp_path, monkeypatch):
    from animetix.management.commands import sync_catalog

    processed = tmp_path / "data" / "processed"
    processed.mkdir(parents=True, exist_ok=True)
    (processed / "clean_root_animes.json").write_text(
        json.dumps([{"idMal": 1, "title": "Frieren", "popularity": 1}]),
        encoding="utf-8",
    )
    (processed / "clean_root_mangas.json").write_text(
        json.dumps([{"idMal": 2, "title": "Berserk", "popularity": 1}]),
        encoding="utf-8",
    )
    monkeypatch.setattr(
        sync_catalog.os.path, "abspath", lambda _p: str(tmp_path), raising=False
    )

    _run(media_type="Anime")

    assert MediaItem.objects.filter(media_type="Anime").count() == 1
    assert MediaItem.objects.filter(media_type="Manga").count() == 0


@pytest.mark.django_db
def test_sync_invalidates_the_catalogue_cache(tmp_path, monkeypatch):
    data = [{"idMal": 1, "title": "Frieren", "popularity": 5000}]
    _write_catalog(tmp_path, monkeypatch, "clean_root_animes.json", data)

    cache.set("catalog_Anime", {"stale": True}, timeout=3600)
    _run()

    assert cache.get("catalog_Anime") is None


@pytest.mark.django_db
def test_dry_run_does_not_touch_the_cache(tmp_path, monkeypatch):
    data = [{"idMal": 1, "title": "Frieren", "popularity": 5000}]
    _write_catalog(tmp_path, monkeypatch, "clean_root_animes.json", data)

    cache.set("catalog_Anime", {"stale": True}, timeout=3600)
    _run(dry_run=True)

    assert cache.get("catalog_Anime") == {"stale": True}