Spaces:
Sleeping
Sleeping
File size: 5,780 Bytes
3aa66b0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 | """`sync_catalog` rewritten from update_or_create()-per-item to bulk writes.
Against the remote Neon DB (us-east-1, no local replica), the old command did a
SELECT + INSERT/UPDATE round trip per item -- ~44 700 of them, all inside one
`@transaction.atomic` -- and after 70 minutes it still had not committed. These
tests pin the replacement contract: a single upfront SELECT to classify
create/update, batched bulk_create/bulk_update (one transaction per batch, not
one for the whole run), a cheap skip path for byte-identical rows so a routine
re-sync writes nothing, `--dry-run`, `--media-type`, and cache invalidation of
the `catalog_{type}` Redis key at the end (the per-process RAM cache in
CatalogService._cached_catalogs cannot be reached from here and is documented,
not silently ignored).
"""
import json
from io import StringIO
import pytest
from animetix.models import MediaItem
from django.core.cache import cache
from django.core.management import call_command
def _write_catalog(tmp_path, monkeypatch, filename, items):
from animetix.management.commands import sync_catalog
processed = tmp_path / "data" / "processed"
processed.mkdir(parents=True, exist_ok=True)
(processed / filename).write_text(json.dumps(items), encoding="utf-8")
# Same trick the pre-existing sync_catalog tests use: abspath is mocked to
# ignore its argument entirely, so both calls in the command (dirname(
# abspath(__file__)) and abspath(join(...))) collapse onto tmp_path.
monkeypatch.setattr(
sync_catalog.os.path, "abspath", lambda _p: str(tmp_path), raising=False
)
def _run(**kwargs):
out = StringIO()
call_command("sync_catalog", "--no-color", stdout=out, **kwargs)
return out.getvalue()
@pytest.mark.django_db
def test_second_run_of_unchanged_data_writes_nothing(tmp_path, monkeypatch):
data = [{"idMal": 1, "title": "Frieren", "popularity": 5000}]
_write_catalog(tmp_path, monkeypatch, "clean_root_animes.json", data)
_run()
assert MediaItem.objects.count() == 1
output = _run()
assert "created=0" in output
assert "updated=0" in output
assert "skipped=1" in output
assert MediaItem.objects.get(external_id="1").title == "Frieren"
@pytest.mark.django_db
def test_new_items_created_and_changed_items_updated(tmp_path, monkeypatch):
data = [
{"idMal": 1, "title": "Frieren", "popularity": 5000},
{"idMal": 2, "title": "Old Title", "popularity": 10},
]
_write_catalog(tmp_path, monkeypatch, "clean_root_animes.json", data)
_run()
updated_data = [
{"idMal": 1, "title": "Frieren", "popularity": 5000}, # unchanged
{"idMal": 2, "title": "New Title", "popularity": 999}, # changed
{"idMal": 3, "title": "Brand New", "popularity": 42}, # new
]
_write_catalog(tmp_path, monkeypatch, "clean_root_animes.json", updated_data)
output = _run()
assert "created=1" in output
assert "updated=1" in output
assert "skipped=1" in output
assert MediaItem.objects.count() == 3
changed = MediaItem.objects.get(external_id="2")
assert changed.title == "New Title"
assert changed.popularity == 999.0
assert MediaItem.objects.get(external_id="3").title == "Brand New"
@pytest.mark.django_db
def test_dry_run_writes_nothing(tmp_path, monkeypatch):
data = [{"idMal": 1, "title": "Frieren", "popularity": 5000}]
_write_catalog(tmp_path, monkeypatch, "clean_root_animes.json", data)
output = _run(dry_run=True)
assert MediaItem.objects.count() == 0
assert "created=1" in output
@pytest.mark.django_db
def test_dry_run_on_a_changed_row_reports_update_without_writing(tmp_path, monkeypatch):
data = [{"idMal": 1, "title": "Frieren", "popularity": 5000}]
_write_catalog(tmp_path, monkeypatch, "clean_root_animes.json", data)
_run()
changed = [{"idMal": 1, "title": "Frieren Season 2", "popularity": 5000}]
_write_catalog(tmp_path, monkeypatch, "clean_root_animes.json", changed)
output = _run(dry_run=True)
assert "updated=1" in output
# Dry run: the row in the DB must be untouched.
assert MediaItem.objects.get(external_id="1").title == "Frieren"
@pytest.mark.django_db
def test_media_type_filter_syncs_only_that_type(tmp_path, monkeypatch):
from animetix.management.commands import sync_catalog
processed = tmp_path / "data" / "processed"
processed.mkdir(parents=True, exist_ok=True)
(processed / "clean_root_animes.json").write_text(
json.dumps([{"idMal": 1, "title": "Frieren", "popularity": 1}]),
encoding="utf-8",
)
(processed / "clean_root_mangas.json").write_text(
json.dumps([{"idMal": 2, "title": "Berserk", "popularity": 1}]),
encoding="utf-8",
)
monkeypatch.setattr(
sync_catalog.os.path, "abspath", lambda _p: str(tmp_path), raising=False
)
_run(media_type="Anime")
assert MediaItem.objects.filter(media_type="Anime").count() == 1
assert MediaItem.objects.filter(media_type="Manga").count() == 0
@pytest.mark.django_db
def test_sync_invalidates_the_catalogue_cache(tmp_path, monkeypatch):
data = [{"idMal": 1, "title": "Frieren", "popularity": 5000}]
_write_catalog(tmp_path, monkeypatch, "clean_root_animes.json", data)
cache.set("catalog_Anime", {"stale": True}, timeout=3600)
_run()
assert cache.get("catalog_Anime") is None
@pytest.mark.django_db
def test_dry_run_does_not_touch_the_cache(tmp_path, monkeypatch):
data = [{"idMal": 1, "title": "Frieren", "popularity": 5000}]
_write_catalog(tmp_path, monkeypatch, "clean_root_animes.json", data)
cache.set("catalog_Anime", {"stale": True}, timeout=3600)
_run(dry_run=True)
assert cache.get("catalog_Anime") == {"stale": True}
|