src/aspect_dict.py

"""Aspect keyword dictionary for clothing domain.

Seed words for each aspect. Can be extended using metadata's `features`/`categories` text.
"""
import json
import re
from collections import Counter
from typing import Dict, List, Iterable

from .config import ASPECT_DICT_PATH


# Seed dictionary - manually curated for clothing domain
SEED_ASPECT_DICT: Dict[str, List[str]] = {
    "SIZE": [
        "alterations",
        "baggy",
        "big",
        "boxy fit",
        "fit",
        "fit perfectly",
        "fits",
        "fitting",
        "great fit",
        "huge",
        "large",
        "length",
        "long",
        "loose",
        "loose fit",
        "narrow",
        "oversized",
        "perfect fit",
        "petite",
        "poor fit",
        "right size",
        "roomy",
        "runs big",
        "runs large",
        "runs small",
        "runs tight",
        "short",
        "shorter than expected",
        "size",
        "size down",
        "size up",
        "sizing",
        "small",
        "snug",
        "stretched out",
        "tight",
        "tiny",
        "too big",
        "too large",
        "too long",
        "too loose",
        "too short",
        "too small",
        "too tight",
        "true to size",
        "tts",
        "waist",
        "wide",
        "wrong size",
    ],
    "MATERIAL": [
        "blend",
        "breathable",
        "bulky",
        "cashmere",
        "cheap fabric",
        "cheap material",
        "comfortable fabric",
        "cotton",
        "denim",
        "elastic",
        "fabric",
        "fabric feels",
        "feel",
        "feeling",
        "feels",
        "fleece",
        "heavy",
        "heavier",
        "heavier than expected",
        "heavy fabric",
        "itchy",
        "leather",
        "lightweight",
        "linen",
        "lycra",
        "material",
        "nylon",
        "polyester",
        "rayon",
        "rough",
        "scratchy",
        "see through",
        "see-through",
        "sheer",
        "silk",
        "soft",
        "spandex",
        "stiff",
        "stretch",
        "stretchy",
        "suede",
        "synthetic",
        "texture",
        "thick",
        "thin",
        "too heavy",
        "transparent",
        "uncomfortable fabric",
        "velvet",
        "warm",
        "weight",
        "wool",
    ],
    "QUALITY": [
        "after wash",
        "after washing",
        "broke",
        "broken",
        "button",
        "buttons",
        "cheaply made",
        "construction",
        "craftsmanship",
        "defect",
        "defective",
        "durability",
        "durable",
        "faded",
        "faded after wash",
        "fading",
        "falling apart",
        "fell apart",
        "first wash",
        "flimsy",
        "fragile",
        "hem",
        "holes",
        "lasted",
        "lasts",
        "loose thread",
        "loose threads",
        "made well",
        "pilled",
        "pilling",
        "poor quality",
        "poorly made",
        "quality",
        "rip",
        "ripped",
        "seam",
        "seams",
        "shedding",
        "shrink",
        "shrunk",
        "shrunk after wash",
        "shrunk after washing",
        "stitch",
        "stitching",
        "sturdy",
        "tear",
        "threads",
        "tore",
        "torn",
        "well constructed",
        "well made",
        "well-made",
        "zipper",
    ],
    "APPEARANCE": [
        "as advertised",
        "as described",
        "as pictured",
        "as shown",
        "beautiful",
        "bright",
        "color",
        "color is off",
        "color off",
        "colors",
        "colour",
        "darker",
        "design",
        "different color",
        "different from picture",
        "different from photo",
        "dull",
        "faded color",
        "graphic",
        "gorgeous",
        "lighter",
        "logo",
        "looked like",
        "looks like",
        "matches the picture",
        "misleading photo",
        "off color",
        "pattern",
        "photo",
        "picture",
        "print",
        "shade",
        "stunning",
        "tone",
        "true color",
        "true to color",
        "ugly",
        "vibrant",
    ],
    "STYLE": [
        "boho",
        "casual",
        "chic",
        "classic",
        "collar",
        "compliment",
        "complimented",
        "compliments",
        "cut",
        "cute",
        "elegant",
        "fashion",
        "fashionable",
        "flattering",
        "formal",
        "modern",
        "neckline",
        "outfit",
        "preppy",
        "professional",
        "received compliments",
        "shape",
        "shapeless",
        "silhouette",
        "sleeves",
        "style",
        "stylish",
        "trendy",
        "unflattering",
        "vintage",
    ],
    "VALUE": [
        "affordable",
        "bargain",
        "cheap price",
        "cost",
        "discount",
        "exchanged",
        "expensive",
        "for the price",
        "for this price",
        "good deal",
        "great deal",
        "money",
        "money back",
        "money well spent",
        "not worth",
        "not worth it",
        "overpriced",
        "price",
        "pricey",
        "refund",
        "return",
        "returning",
        "value",
        "value for money",
        "waste of money",
        "wasted money",
        "worth",
        "worth it",
    ],
}
def get_aspect_dict() -> Dict[str, List[str]]:
    """Return seed dict (callers can extend it)."""
    return {k: list(v) for k, v in SEED_ASPECT_DICT.items()}


# Tokens / phrases we consider noise when mining metadata text
_STOPWORDS = set("""
the and for with from this that have has had not your you our their these
those there here when where what which who whom how why because all any
its his her item product use used using also more most less other than just
new cost free buy bought made make piece pieces set sets pack two one three
inch inches cm mm gram fl oz lb pounds black white grey gray
are can will would could should closure imported machine women men kids ladies
amazon brand sleeve sleeves shirt shirts dress dresses pants jacket jackets
""".split())



_GENERIC_MINED_TOKENS = {
    "are", "can", "will", "would", "could", "should", "closure", "imported",
    "machine", "women", "woman", "men", "kids", "ladies", "perfect", "nice",
    "great", "good", "comfortable", "casual", "design", "wear", "wash",
    "fit", "fabric", "material", "polyester", "cotton", "soft",
}

_AMBIGUOUS_SINGLE_TOKENS = {
    "cheap", "perfect", "comfortable", "design", "style", "fit", "wash",
}
def _tokenize(text: str) -> List[str]:
    return re.findall(r"[a-z][a-z\-]+", text.lower())


def extend_aspect_dict_from_metadata(
    meta_texts: Iterable[str],
    aspect_dict: Dict[str, List[str]] = None,
    top_k: int = 20,
    min_count: int = 5,
) -> Dict[str, List[str]]:
    """Mine frequent domain words from metadata text to extend aspect seed dict.

    For each aspect, if a metadata token co-occurs frequently with any seed word
    in the same metadata blob, add it to the aspect's vocabulary.
    """
    if aspect_dict is None:
        aspect_dict = get_aspect_dict()

    # Build per-aspect co-occurrence counter
    aspect_cooc = {a: Counter() for a in aspect_dict}

    for text in meta_texts:
        if not isinstance(text, str) or not text:
            continue
        tokens = set(_tokenize(text))
        if not tokens:
            continue
        # For each aspect, check if any seed appears in this blob
        for aspect, seeds in aspect_dict.items():
            seed_set = set(_tokenize(" ".join(seeds)))
            if tokens & seed_set:
                # Co-occurring tokens -> aspect candidates
                for t in tokens:
                    if t in _STOPWORDS or t in seed_set:
                        continue
                    if t in _GENERIC_MINED_TOKENS or t in _AMBIGUOUS_SINGLE_TOKENS:
                        continue
                    if len(t) < 4:
                        continue
                    aspect_cooc[aspect][t] += 1

    # Add top-k high frequency co-occurring tokens to each aspect
    extended = {a: list(v) for a, v in aspect_dict.items()}
    for aspect, counter in aspect_cooc.items():
        new_words = [w for w, c in counter.most_common(top_k * 3) if c >= min_count]
        # Filter words that look generic or ambiguous outside local context.
        new_words = [
            w for w in new_words
            if w not in _STOPWORDS
            and w not in _GENERIC_MINED_TOKENS
            and w not in _AMBIGUOUS_SINGLE_TOKENS
        ][:top_k]
        extended[aspect].extend(new_words)

    # Dedup
    for a in extended:
        extended[a] = sorted(set(extended[a]))

    return extended


def save_aspect_dict(aspect_dict: Dict[str, List[str]], path=ASPECT_DICT_PATH):
    with open(path, "w", encoding="utf-8") as f:
        json.dump(aspect_dict, f, indent=2)


def load_aspect_dict(path=ASPECT_DICT_PATH) -> Dict[str, List[str]]:
    with open(path, encoding="utf-8") as f:
        return json.load(f)


def compile_aspect_patterns(aspect_dict: Dict[str, List[str]]):
    """Compile regex patterns for each aspect (whole-phrase matching, case-insensitive)."""
    patterns = {}
    for aspect, keywords in aspect_dict.items():
        # Sort longest first so multi-word phrases match before single tokens
        sorted_kws = sorted(set(keywords), key=lambda s: -len(s))
        escaped = [re.escape(kw) for kw in sorted_kws]
        pattern = r"\b(?:" + "|".join(escaped) + r")\b"
        patterns[aspect] = re.compile(pattern, flags=re.IGNORECASE)
    return patterns