Upload 22 files

app.py

@@ -899,18 +899,52 @@ def _resolve_target_category(requested_target: str, wardrobe_snapshot: dict[str,
     return "bottomwear"
 
 
-def _product_text_for_relevance(product: dict[str, Any]) -> str:
-    name = str(product.get("name") or "")
-    url = str(product.get("item_link") or "")
-    return _norm(f"{name} {url}")
-
-
-def _is_relevant_scraped_product(
-    product: dict[str, Any],
-    target_slot: str,
-    planned_category: str,
-    occasion_bucket: str,
-) -> bool:
+def _product_text_for_relevance(product: dict[str, Any]) -> str:
+    name = str(product.get("name") or "")
+    url = str(product.get("item_link") or "")
+    color = str(product.get("color") or "")
+    brand = str(product.get("brand") or "")
+    return _norm(f"{name} {color} {brand} {url}")
+
+
+SCRAPER_COLOR_KEYWORDS: dict[str, set[str]] = {
+    "black": {"black", "jet black"},
+    "white": {"white", "bright white", "off white", "off-white"},
+    "navy": {"navy", "dark blue", "dk blue", "dress blues", "moonlit ocean", "midnight blue"},
+    "blue": {"blue", "navy", "dark blue", "dk blue", "dress blues", "ice blue", "light blue", "skyway", "moonlit ocean"},
+    "grey": {"grey", "gray", "dark grey", "dark gray", "steel grey", "steel gray", "charcoal"},
+    "gray": {"grey", "gray", "dark grey", "dark gray", "steel grey", "steel gray", "charcoal"},
+    "beige": {"beige", "sand", "tan", "stone", "morel"},
+    "brown": {"brown", "tan", "morel"},
+    "olive": {"olive", "khaki"},
+    "green": {"green", "olive", "khaki"},
+    "red": {"red", "brick red", "winetasting", "wine"},
+    "maroon": {"maroon", "burgundy", "wine", "winetasting"},
+}
+
+
+def _color_keywords_for_relevance(color: str) -> set[str]:
+    normalized = extract_base_color(color or "") or _norm(color)
+    if not normalized or normalized == "unknown":
+        return set()
+    return SCRAPER_COLOR_KEYWORDS.get(normalized, {normalized})
+
+
+def _matches_planned_color(product: dict[str, Any], planned_color: str) -> bool:
+    keywords = _color_keywords_for_relevance(planned_color)
+    if not keywords:
+        return True
+    text = _product_text_for_relevance(product)
+    return any(keyword in text for keyword in keywords)
+
+
+def _is_relevant_scraped_product(
+    product: dict[str, Any],
+    target_slot: str,
+    planned_category: str,
+    planned_color: str,
+    occasion_bucket: str,
+) -> bool:
     text = _product_text_for_relevance(product)
     if not text:
         return False
@@ -918,10 +952,13 @@ def _is_relevant_scraped_product(
     if any(token in text for token in SCRAPER_RELEVANCE_EXCLUDE_TOKENS):
         return False
 
-    planned = _norm(planned_category)
-    planned_keywords = SCRAPER_CATEGORY_KEYWORDS.get(planned, {planned} if planned else set())
-    if planned_keywords and not any(keyword in text for keyword in planned_keywords):
-        return False
+    planned = _norm(planned_category)
+    planned_keywords = SCRAPER_CATEGORY_KEYWORDS.get(planned, {planned} if planned else set())
+    if planned_keywords and not any(keyword in text for keyword in planned_keywords):
+        return False
+
+    if not _matches_planned_color(product, planned_color):
+        return False
 
     if target_slot == "topwear":
         topwear_terms = {"shirt", "polo", "blazer", "jacket", "coat", "t-shirt", "tee", "hoodie"}
@@ -1501,12 +1538,13 @@ def _generate_scraper_plan_with_nemotron(
                     continue
                 seen_links.add(item_link)
 
-                if _is_relevant_scraped_product(
-                    product=product,
-                    target_slot=resolved_target,
-                    planned_category=category,
-                    occasion_bucket=occasion_bucket,
-                ):
+                if _is_relevant_scraped_product(
+                    product=product,
+                    target_slot=resolved_target,
+                    planned_category=category,
+                    planned_color=color,
+                    occasion_bucket=occasion_bucket,
+                ):
                     scraped_products.append(product)
                     new_products += 1
                 else:
@@ -1525,17 +1563,27 @@ def _generate_scraper_plan_with_nemotron(
                 }
             )
 
-        if not scraped_products and fallback_products:
-            scraped_products = fallback_products[:scrape_limit]
-            intermediate_steps.append(
-                {
+        if not scraped_products and fallback_products and not _color_keywords_for_relevance(color):
+            scraped_products = fallback_products[:scrape_limit]
+            intermediate_steps.append(
+                {
                     "step": "scrape_fallback",
                     "query": query,
                     "new_products": len(scraped_products),
                     "total_products": len(scraped_products),
-                    "message": "Used non-filtered scrape fallback because strict relevance filtering returned no products.",
-                }
-            )
+                    "message": "Used non-filtered scrape fallback because strict relevance filtering returned no products.",
+                }
+            )
+        elif not scraped_products and fallback_products:
+            intermediate_steps.append(
+                {
+                    "step": "scrape_filter",
+                    "query": query,
+                    "rejected_products": len(fallback_products),
+                    "total_products": 0,
+                    "message": "Rejected scraped products because none matched the planned color and category.",
+                }
+            )
 
     query_plan_payload = {
         "color": color,

zalando_scraper.py

@@ -5,7 +5,7 @@ import json
 import os
 import re
 from typing import Any, Callable, Optional
-from urllib.parse import urlencode, urlparse
+from urllib.parse import parse_qs, urlencode, urlparse
 
 import requests
 from bs4 import BeautifulSoup
@@ -60,7 +60,7 @@ CATEGORY_PATH_MAP = {
     "sportswear": {"women": "womens-sports", "men": "mens-sports", "unisex": "sports"},
 }
 
-_COLOR_TERMS = [
+_COLOR_TERMS = [
     "black",
     "white",
     "navy",
@@ -81,7 +81,32 @@ _COLOR_TERMS = [
     "purple",
     "yellow",
     "orange",
-]
+]
+
+_COLOR_QUERY_KEYWORDS: dict[str, set[str]] = {
+    "black": {"black"},
+    "white": {"white", "bright white", "off white", "off-white"},
+    "navy": {"navy", "dark blue", "dk blue", "dress blues", "moonlit ocean", "midnight blue"},
+    "blue": {"blue", "navy", "dark blue", "dk blue", "dress blues", "ice blue", "light blue", "skyway", "moonlit ocean"},
+    "grey": {"grey", "gray", "dark grey", "dark gray", "steel grey", "steel gray", "charcoal"},
+    "gray": {"grey", "gray", "dark grey", "dark gray", "steel grey", "steel gray", "charcoal"},
+    "beige": {"beige", "sand", "tan", "stone", "morel"},
+    "brown": {"brown", "tan", "morel"},
+    "olive": {"olive", "khaki"},
+    "green": {"green", "olive", "khaki"},
+    "red": {"red", "brick red", "winetasting", "wine"},
+    "maroon": {"maroon", "burgundy", "wine", "winetasting"},
+}
+
+_CATEGORY_QUERY_KEYWORDS: dict[str, set[str]] = {
+    "shirt": {"shirt", "formal shirt"},
+    "polo": {"polo"},
+    "jacket": {"jacket", "blazer", "coat"},
+    "trousers": {"trousers", "pants", "chinos"},
+    "pants": {"pants", "trousers", "chinos"},
+    "shorts": {"shorts"},
+    "jeans": {"jeans"},
+}
 
 
 ScrapePostprocessFn = Callable[[list[dict[str, str]]], list[dict[str, str]]]
@@ -89,8 +114,61 @@ WardrobeSummary = dict[str, Any]
 TextCompletionFn = Callable[[str, int], str]
 
 
-def _norm(value: Any) -> str:
-    return str(value or "").strip().lower()
+def _norm(value: Any) -> str:
+    return str(value or "").strip().lower()
+
+
+def _query_from_search_url(search_url: str) -> str:
+    parsed = urlparse(str(search_url or ""))
+    values = parse_qs(parsed.query).get("q") or []
+    return str(values[0] if values else "").strip()
+
+
+def _query_color_keywords(query: str) -> set[str]:
+    normalized = _norm(query)
+    for color in _COLOR_TERMS:
+        if color in normalized:
+            return _COLOR_QUERY_KEYWORDS.get(color, {color})
+    return set()
+
+
+def _query_category_keywords(query: str) -> set[str]:
+    normalized = _norm(query)
+    for category, keywords in _CATEGORY_QUERY_KEYWORDS.items():
+        if category in normalized:
+            return keywords
+    return set()
+
+
+def _product_match_text(product: dict[str, str]) -> str:
+    return _norm(
+        " ".join(
+            [
+                str(product.get("name") or ""),
+                str(product.get("color") or ""),
+                str(product.get("brand") or ""),
+                str(product.get("item_link") or ""),
+            ]
+        )
+    )
+
+
+def _filter_products_for_search_query(products: list[dict[str, str]], search_url: str) -> list[dict[str, str]]:
+    query = _query_from_search_url(search_url)
+    color_keywords = _query_color_keywords(query)
+    category_keywords = _query_category_keywords(query)
+    if not color_keywords and not category_keywords:
+        return products
+
+    filtered: list[dict[str, str]] = []
+    for product in products:
+        text = _product_match_text(product)
+        if color_keywords and not any(keyword in text for keyword in color_keywords):
+            continue
+        if category_keywords and not any(keyword in text for keyword in category_keywords):
+            continue
+        filtered.append(product)
+    return filtered
 
 
 def _normalize_target_category(value: Any) -> str:
@@ -557,17 +635,11 @@ def _apify_actor_id_from_endpoint(endpoint: str) -> str:
     return "vistics~zalando-scraper"
 
 
-def _build_apify_payload(search_url: str, max_results: int, start_urls_as_objects: bool) -> dict[str, Any]:
-    start_urls: list[Any]
-    if start_urls_as_objects:
-        start_urls = [{"url": search_url}]
-    else:
-        start_urls = [search_url]
-
-    return {
-        "startUrls": start_urls,
-        "maxResults": int(max_results),
-    }
+def _build_apify_payload(search_url: str, max_results: int) -> dict[str, Any]:
+    return {
+        "startUrls": [str(search_url or "").strip()],
+        "maxResults": int(max_results),
+    }
 
 
 def _http_error_detail(exc: requests.RequestException, limit: int = 800) -> str:
@@ -632,13 +704,10 @@ def _scrape_with_apify_run_dataset_fallback(
         wait_for_finish,
     )
 
-    variants = [
-        ("string", False),
-        ("object", True),
-    ]
-
-    for variant_name, use_object_start_urls in variants:
-        run_payload = _build_apify_payload(search_url, effective_limit, start_urls_as_objects=use_object_start_urls)
+    variants = ["string"]
+
+    for variant_name in variants:
+        run_payload = _build_apify_payload(search_url, effective_limit)
         run_id = ""
         run_status = ""
         dataset_id = ""
@@ -777,13 +846,18 @@ def _normalize_product(item: dict[str, Any]) -> dict[str, str]:
         )
     )
 
-    return {
-        "name": name or "N/A",
-        "price": price or "N/A",
-        "brand": brand,
-        "currency_symbol": currency_symbol,
-        "promotional_price": promotional_price,
-        "original_price": original_price,
+    color = str(item.get("color") or item.get("colorName") or item.get("colour") or "").strip()
+    if not color and " - " in name:
+        color = name.rsplit(" - ", 1)[-1].strip()
+
+    return {
+        "name": name or "N/A",
+        "price": price or "N/A",
+        "brand": brand,
+        "color": color,
+        "currency_symbol": currency_symbol,
+        "promotional_price": promotional_price,
+        "original_price": original_price,
         "discount_percent": discount_percent,
         "image_url": image_url,
         "item_link": url_value,
@@ -804,14 +878,11 @@ def _scrape_with_apify(search_url: str, max_products: int | None, timeout_second
         actor_id,
     )
 
-    variants = [
-        ("string", False),
-        ("object", True),
-    ]
-    variant_errors: list[str] = []
-    for variant_name, use_object_start_urls in variants:
-        try:
-            payload = _build_apify_payload(search_url, effective_limit, start_urls_as_objects=use_object_start_urls)
+    variants = ["string"]
+    variant_errors: list[str] = []
+    for variant_name in variants:
+        try:
+            payload = _build_apify_payload(search_url, effective_limit)
             response = requests.post(_apify_request_url(), json=payload, timeout=apify_timeout)
             response.raise_for_status()
 
@@ -987,15 +1058,17 @@ def extract_product_summaries(
             errors.append(f"html: {exc}")
             logger.warning("zalando crawl failed source=html search_url=%s error=%s", search_url, exc)
 
-    if postprocess and _requires_postprocess(products):
-        try:
-            products = postprocess(products)
-        except Exception:
-            # Never fail scraping because post-processing failed.
-            pass
-
-    if not products and errors:
-        logger.warning("zalando crawl completed with no results search_url=%s errors=%s", search_url, "; ".join(errors))
+    if postprocess and _requires_postprocess(products):
+        try:
+            products = postprocess(products)
+        except Exception:
+            # Never fail scraping because post-processing failed.
+            pass
+
+    products = _filter_products_for_search_query(products, search_url)
+
+    if not products and errors:
+        logger.warning("zalando crawl completed with no results search_url=%s errors=%s", search_url, "; ".join(errors))
         raise requests.RequestException("; ".join(errors))
 
     logger.info("zalando crawl completed search_url=%s crawled=%s items=%s", search_url, bool(products), len(products))