fix: prefer Wikimedia images over Pixabay stock photos on re-check

- After Pixabay returns an image, re-check English Wikipedia and
Wikidata tiers in case they were rate-limited on first pass
- Added multi-language Wikipedia tier (fr/de/es/it/ja) as Tier 2
- Fixed cache check to skip empty-string entries (re-try failed items)
- Cache: non-empty URLs only — empty cache entries no longer block retries

src/services/recommender.py

@@ -328,6 +328,64 @@ def _fetch_wiki_image(name: str, city: str = "") -> str:
     return ""
 
 
+_MULTILANG_WIKI = ["fr", "de", "es", "it", "ja"]
+
+
+def _fetch_wiki_image_multilang(name: str, city: str = "") -> str:
+    """Tier 1.5: Search non-English Wikipedias for an image.
+    When English Wikipedia has no thumbnail, try French, German, Spanish,
+    Italian, and Japanese editions in parallel — the next largest by article
+    count and rich in travel-related imagery.
+    """
+    clean = re.sub(r"\s*\(.*?\)\s*$", "", name).strip()
+    search_terms = [clean] if clean and clean != name else [clean, name]
+    if city:
+        search_terms.append(f"{clean}, {city}" if clean else f"{name}, {city}")
+
+    import concurrent.futures
+
+    def _try_lang(lang: str) -> str:
+        # Try just the cleaned name (most likely to match across languages)
+        for term in search_terms[:2]:  # try at most 2 terms
+            if not term:
+                continue
+            try:
+                url = f"https://{lang}.wikipedia.org/w/api.php?" + urllib.parse.urlencode({
+                    "action": "query",
+                    "generator": "search",
+                    "gsrsearch": term,
+                    "gsrlimit": 3,
+                    "prop": "pageimages",
+                    "pithumbsize": 500,
+                    "format": "json",
+                })
+                req = urllib.request.Request(url, headers={"User-Agent": "TravelPlanner/1.0"})
+                with urllib.request.urlopen(req, timeout=3) as resp:
+                    data = json.loads(resp.read().decode())
+                pages = data.get("query", {}).get("pages", {})
+                for page in pages.values():
+                    thumb = page.get("thumbnail", {}).get("source", "")
+                    if thumb:
+                        return thumb
+            except Exception:
+                continue
+        return ""
+
+    with concurrent.futures.ThreadPoolExecutor(max_workers=5) as pool:
+        futures = {pool.submit(_try_lang, lang): lang for lang in _MULTILANG_WIKI}
+        for f in concurrent.futures.as_completed(futures):
+            try:
+                result = f.result(timeout=5)
+                if result:
+                    # Cancel remaining futures — we found one
+                    for other in futures:
+                        other.cancel()
+                    return result
+            except Exception:
+                continue
+    return ""
+
+
 # Tourism-related keywords to disambiguate Wikidata results
 _TOURISM_KEYWORDS = {
     "church", "cathedral", "basilica", "monument", "museum", "palace",
@@ -512,6 +570,42 @@ def _fetch_local_name_image(name: str, city: str = "", country: str = "") -> str
     return ""
 
 
+def _fetch_pixabay_api_image(name: str, city: str = "", country: str = "") -> str:
+    """Tier 5: Search Pixabay for a high-quality landscape photo.
+    Free tier: 100 req/min, no monthly cap. Much better limits than Unsplash.
+    """
+    pixabay_key = os.environ.get("PIXABAY_API_KEY", "")
+    if not pixabay_key:
+        return ""
+
+    # Build search query: name + city for better relevance
+    clean = re.sub(r"\s*\(.*?\)\s*$", "", name).strip()
+    query = clean
+    if city:
+        query = f"{clean} {city}"
+    elif country:
+        query = f"{clean} {country}"
+
+    search_url = "https://pixabay.com/api/?" + urllib.parse.urlencode({
+        "key": pixabay_key,
+        "q": query,
+        "per_page": 3,
+        "orientation": "horizontal",
+        "safesearch": "true",
+        "image_type": "photo",
+    })
+    try:
+        with urllib.request.urlopen(search_url, timeout=8) as resp:
+            data = json.loads(resp.read().decode())
+        hits = data.get("hits", [])
+        if hits:
+            # webformatURL is ~640px wide — perfect for cards
+            return hits[0]["webformatURL"]
+    except Exception:
+        pass
+    return ""
+
+
 def _fetch_unsplash_api_image(name: str, city: str = "", country: str = "") -> str:
     """Tier 6: Search Unsplash for a high-quality landscape photo.
     Only called when all Wikimedia sources fail. Uses orientation=landscape
@@ -551,12 +645,14 @@ def _fetch_unsplash_api_image(name: str, city: str = "", country: str = "") -> s
 
 
 def _enrich_one_item(item: dict, city: str = "", country: str = "") -> None:
-    """Look up image for a single item using 5-tier fallback:
-    1. Wikipedia REST/pageimages API
-    2. Wikidata P18 image claim (with city/country context)
-    3. Wikimedia Commons search (with simplified name variants embedded)
-    4. Local name from parentheses (e.g. Koko-shima from Awaji Island)
-    5. Unsplash search (landscape orientation, last resort)
+    """Look up image for a single item using 7-tier fallback:
+    1. Wikipedia REST/pageimages API (English)
+    2. Wikipedia REST/pageimages API (French, German, Spanish, Italian, Japanese)
+    3. Wikidata P18 image claim (with city/country context)
+    4. Wikimedia Commons search (with simplified name variants embedded)
+    5. Local name from parentheses (e.g. Koko-shima from Awaji Island)
+    6. Pixabay search (free tier, no monthly cap)
+    7. Unsplash search (landscape orientation, last resort)
 
     Results are cached in _IMAGE_CACHE to avoid repeat API calls across searches.
     """
@@ -567,42 +663,66 @@ def _enrich_one_item(item: dict, city: str = "", country: str = "") -> None:
         item["image_url"] = ""
         return
 
-    # Check image cache first
+    # Check image cache first (only use cached if it's a real URL — empty strings
+    # mean the item was never successfully resolved, so re-try)
     cache_key = (name, city or "", country or "")
     cached_url = _IMAGE_CACHE.get(cache_key)
-    if cached_url is not None:
+    if cached_url:  # truthy check — non-empty URL only
         item["image_url"] = cached_url
         return
 
-    # Tier 1: Wikipedia
+    # Tier 1: Wikipedia (English)
     url = _fetch_wiki_image(name, city=city)
     if url:
         _IMAGE_CACHE[cache_key] = url
         item["image_url"] = url
         _save_image_cache()
         return
-    # Tier 2: Wikidata (with city/country for disambiguation)
+    # Tier 2: Wikipedia (multi-language — fr, de, es, it, ja)
+    url = _fetch_wiki_image_multilang(name, city=city)
+    if url:
+        _IMAGE_CACHE[cache_key] = url
+        item["image_url"] = url
+        _save_image_cache()
+        return
+    # Tier 3: Wikidata (with city/country for disambiguation)
     url = _fetch_wikidata_image(name, city=city, country=country)
     if url:
         _IMAGE_CACHE[cache_key] = url
         item["image_url"] = url
         _save_image_cache()
         return
-    # Tier 3: Wikimedia Commons (includes simplified/variant names)
+    # Tier 4: Wikimedia Commons (includes simplified/variant names)
     url = _fetch_commons_image(name, city=city, country=country)
     if url:
         _IMAGE_CACHE[cache_key] = url
         item["image_url"] = url
         _save_image_cache()
         return
-    # Tier 4: Local name from parentheses
+    # Tier 5: Local name from parentheses
     url = _fetch_local_name_image(name, city=city, country=country)
     if url:
         _IMAGE_CACHE[cache_key] = url
         item["image_url"] = url
         _save_image_cache()
         return
-    # Tier 5: Unsplash (landscape only, last resort)
+    # Tier 6: Pixabay (free tier, no monthly cap)
+    url = _fetch_pixabay_api_image(name, city=city, country=country)
+    if url:
+        # Second chance: re-check Wikipedia/Wikidata tiers in case they were
+        # rate-limited or unavailable on the first pass. Pixabay often returns
+        # generic stock photos (same image under different query URLs); prefer
+        # a specific Wikimedia image when one exists.
+        wiki_url = _fetch_wiki_image(name, city=city)
+        if not wiki_url:
+            wiki_url = _fetch_wikidata_image(name, city=city, country=country)
+        if wiki_url:
+            url = wiki_url
+        _IMAGE_CACHE[cache_key] = url
+        item["image_url"] = url
+        _save_image_cache()
+        return
+    # Tier 7: Unsplash (landscape only, last resort)
     url = _fetch_unsplash_api_image(name, city=city, country=country)
     if url:
         _IMAGE_CACHE[cache_key] = url

tests/test_recommender.py

@@ -7,6 +7,7 @@ import pytest
 
 # Import the units under test from the recommender module
 from src.services.recommender import (
+    _enrich_one_item,
     _fetch_wiki_image,
     _haversine_km,
     _is_media_entertainment_page,
@@ -273,3 +274,82 @@ class TestFetchWikiImage:
 
             result = _fetch_wiki_image("XyzzyNonexistentPlace")
             assert result == ""
+
+
+# ──────────────────────────────────────────────────────────────────────
+# 6. _enrich_one_item — Pixabay fallback re-check
+# ──────────────────────────────────────────────────────────────────────
+class TestEnrichOneItemPixabayRecheck:
+    """Tests for Pixabay fallback re-checking Wikipedia/Wikidata for specific images.
+
+    When Pixabay returns a generic stock photo, the function re-checks
+    Wikipedia/Wikidata tiers (which may have been rate-limited on first pass).
+    If a specific Wikimedia image is now available, it should be preferred.
+    """
+
+    def test_prefers_wikipedia_over_pixabay_on_recheck(self):
+        """After Pixabay returns a URL, re-check Wikipedia finds a specific
+        image -> use Wikipedia URL instead of Pixabay generic."""
+        item = {"name": "Adashino Nenbutsuji"}
+
+        with (
+            patch("src.services.recommender._IMAGE_CACHE", {}),
+            patch("src.services.recommender._fetch_wiki_image") as mock_wiki,
+            patch("src.services.recommender._fetch_wiki_image_multilang", return_value=""),
+            patch("src.services.recommender._fetch_wikidata_image", return_value=""),
+            patch("src.services.recommender._fetch_commons_image", return_value=""),
+            patch("src.services.recommender._fetch_local_name_image", return_value=""),
+            patch("src.services.recommender._fetch_pixabay_api_image", return_value="https://pixabay.com/generic.jpg"),
+            patch("src.services.recommender._fetch_unsplash_api_image", return_value=""),
+            patch("src.services.recommender._save_image_cache"),
+        ):
+            # First call (Tier 1): Wikipedia fails (rate-limited)
+            # Second call (second chance after Pixabay): Wikipedia succeeds
+            mock_wiki.side_effect = ["", "https://upload.wikimedia.org/specific.jpg"]
+
+            _enrich_one_item(item, city="Kyoto")
+
+            assert item["image_url"] == "https://upload.wikimedia.org/specific.jpg"
+
+    def test_prefers_wikidata_over_pixabay_on_recheck(self):
+        """After Pixabay returns a URL and Wikipedia still fails, re-check
+        Wikidata finds a specific image -> use Wikidata URL instead."""
+        item = {"name": "Some Temple"}
+
+        with (
+            patch("src.services.recommender._IMAGE_CACHE", {}),
+            patch("src.services.recommender._fetch_wiki_image", return_value=""),
+            patch("src.services.recommender._fetch_wiki_image_multilang", return_value=""),
+            patch("src.services.recommender._fetch_wikidata_image") as mock_wikidata,
+            patch("src.services.recommender._fetch_commons_image", return_value=""),
+            patch("src.services.recommender._fetch_local_name_image", return_value=""),
+            patch("src.services.recommender._fetch_pixabay_api_image", return_value="https://pixabay.com/generic.jpg"),
+            patch("src.services.recommender._fetch_unsplash_api_image", return_value=""),
+            patch("src.services.recommender._save_image_cache"),
+        ):
+            # First call (Tier 3): Wikidata fails (rate-limited)
+            # Second call (second chance after Pixabay): Wikidata succeeds
+            mock_wikidata.side_effect = ["", "https://upload.wikimedia.org/commons/specific.jpg"]
+
+            _enrich_one_item(item, city="Kyoto")
+
+            assert item["image_url"] == "https://upload.wikimedia.org/commons/specific.jpg"
+
+    def test_keeps_pixabay_when_both_wikipedia_and_wikidata_still_fail(self):
+        """When both re-checks still fail, Pixabay URL is used as-is."""
+        item = {"name": "Obscure Place"}
+
+        with (
+            patch("src.services.recommender._IMAGE_CACHE", {}),
+            patch("src.services.recommender._fetch_wiki_image", return_value=""),
+            patch("src.services.recommender._fetch_wiki_image_multilang", return_value=""),
+            patch("src.services.recommender._fetch_wikidata_image", return_value=""),
+            patch("src.services.recommender._fetch_commons_image", return_value=""),
+            patch("src.services.recommender._fetch_local_name_image", return_value=""),
+            patch("src.services.recommender._fetch_pixabay_api_image", return_value="https://pixabay.com/generic.jpg"),
+            patch("src.services.recommender._fetch_unsplash_api_image", return_value=""),
+            patch("src.services.recommender._save_image_cache"),
+        ):
+            _enrich_one_item(item, city="Kyoto")
+
+            assert item["image_url"] == "https://pixabay.com/generic.jpg"