GitHub Actions
Deploy from GitHub: 3ce2b54ba6295c2a6246b4b9c84f78e3167bdeb9
552301b
Raw
History Blame Contribute Delete
9.94 kB
import asyncio
import logging
from fastapi import APIRouter, Request, HTTPException
from pydantic import BaseModel
logger = logging.getLogger(__name__)
router = APIRouter()
# ── Request models ────────────────────────────────────────────────────────────
class AnalyzeFlowRequest(BaseModel):
imageBase64: str
mimeType: str
class SearchFlowRequest(BaseModel):
imageBase64: str
mimeType: str
class RecommendFlowRequest(BaseModel):
productId: str
# ── Helpers ───────────────────────────────────────────────────────────────────
def _build_filter(fashion_attrs: dict) -> dict | None:
clauses = []
if fashion_attrs.get("graphical_appearance"):
clauses.append({"graphical_appearance": {"$eq": fashion_attrs["graphical_appearance"]}})
if fashion_attrs.get("product_group"):
clauses.append({"product_group": {"$eq": fashion_attrs["product_group"]}})
if fashion_attrs.get("index_group"):
clauses.append({"index_group": {"$eq": fashion_attrs["index_group"]}})
return {"$and": clauses} if clauses else None
# ── Step 1: Analyze ──────────────────────────────────────────────────────────
@router.post("/analyze")
async def analyze(request: Request, dto: AnalyzeFlowRequest):
"""
Step 1 β€” Analyze uploaded image.
Claude classifies first (fashion / food / unknown).
If fashion, YOLO detects and crops individual items.
"""
yolo_service = request.app.state.yolo_service
vision_service = request.app.state.vision_service
logger.info("Step 1 β€” analyze: asking Claude to classify image type")
loop = asyncio.get_event_loop()
# Claude classifies first β€” YOLO is fashion-only and will hallucinate on non-fashion images
claude_result = await loop.run_in_executor(
None, vision_service.analyze_image, dto.imageBase64, dto.mimeType
)
logger.info(f"Claude vision result type: {claude_result.get('type')}")
if claude_result.get("type") == "food":
logger.info(f"Food detected: {claude_result.get('foodAnalysis', {}).get('dishName')}")
return {"type": "food", "foodAnalysis": claude_result.get("foodAnalysis")}
if claude_result.get("type") != "fashion":
logger.warning("Image could not be classified as fashion or food β€” returning unknown")
return {"type": "unknown"}
# Confirmed fashion β€” run YOLO to detect and crop individual items
logger.info("Fashion confirmed β€” running YOLO detection")
yolo_result = await loop.run_in_executor(
None, yolo_service.detect_and_crop, dto.imageBase64, dto.mimeType
)
detected_items = yolo_result.get("detectedItems", [])
logger.info(f"YOLO detected {len(detected_items)} item(s)")
real_items = [i for i in detected_items if i.get("label") != "unknown"]
if real_items:
# Deduplicate by label β€” keep highest confidence per label
by_label: dict = {}
for item in real_items:
label = item["label"]
if label not in by_label or item["confidence"] > by_label[label]["confidence"]:
by_label[label] = item
deduped = list(by_label.values())
logger.info(f"Fashion items ({len(real_items)} β†’ {len(deduped)} after dedup)")
return {
"type": "fashion",
"items": [
{
"label": item["label"],
"confidence": item["confidence"],
"boundingBox": item["boundingBox"],
"croppedImageBase64": item["croppedImageBase64"],
"mimeType": item["mimeType"],
}
for item in deduped
],
}
# YOLO found nothing despite fashion classification β€” return full image as single item
logger.warning("YOLO found no items in confirmed fashion image β€” returning full image")
fashion_attrs = claude_result.get("fashionAttributes", {})
return {
"type": "fashion",
"items": [{
"label": fashion_attrs.get("category", "unknown"),
"confidence": 1.0,
"boundingBox": {
"x": 0,
"y": 0,
"width": yolo_result.get("imageWidth", 0),
"height": yolo_result.get("imageHeight", 0),
},
"croppedImageBase64": dto.imageBase64,
"mimeType": dto.mimeType,
}],
}
# ── Step 2: Search ────────────────────────────────────────────────────────────
@router.post("/search")
async def search(request: Request, dto: SearchFlowRequest):
"""
Step 2 β€” Search by selected crop.
Generates CLIP image vector + Claude fashion attributes in parallel,
then queries Pinecone (top 6).
"""
clip_service = request.app.state.clip_service
vision_service = request.app.state.vision_service
pinecone_service = request.app.state.pinecone_service
logger.info("Step 2 β€” search: generating CLIP vector and analyzing fashion attributes in parallel")
loop = asyncio.get_event_loop()
vector, fashion_attrs = await asyncio.gather(
loop.run_in_executor(None, clip_service.generate_image_vector, dto.imageBase64),
loop.run_in_executor(None, vision_service.analyze_fashion_for_search, dto.imageBase64, dto.mimeType),
)
logger.info(f"Fashion attributes from Claude: {fashion_attrs}")
pinecone_filter = _build_filter(fashion_attrs)
logger.info(f"Pinecone filter: {pinecone_filter}")
results = await loop.run_in_executor(
None, lambda: pinecone_service.query(vector, 6, pinecone_filter)
)
logger.info(f"Results: {len(results)} from Pinecone")
return {
"fashionAttributes": fashion_attrs,
"results": [
{"id": r["id"], "score": r["score"], "metadata": r["metadata"]}
for r in results
],
}
# ── Step 3: Recommend ─────────────────────────────────────────────────────────
@router.post("/recommend")
async def recommend(request: Request, dto: RecommendFlowRequest):
"""
Step 3 β€” Get outfit recommendations for a product.
Fetches product from Pinecone, generates outfit ideas via Claude,
then matches each outfit item to a real product via CLIP text vector + Pinecone.
"""
clip_service = request.app.state.clip_service
vision_service = request.app.state.vision_service
pinecone_service = request.app.state.pinecone_service
logger.info(f"Step 3 β€” recommend: fetching product \"{dto.productId}\" from Pinecone")
loop = asyncio.get_event_loop()
product = await loop.run_in_executor(None, pinecone_service.fetch_by_id, dto.productId)
if not product:
logger.error(f"Product \"{dto.productId}\" not found in Pinecone")
raise HTTPException(status_code=404, detail=f"Product \"{dto.productId}\" not found in Pinecone")
meta = product.get("metadata", {})
logger.info(f"Product found: {meta.get('name')}")
description_parts = [
meta.get("name"),
f"in {meta['colour']}" if meta.get("colour") else None,
f"β€” {meta['description']}" if meta.get("description") else None,
]
description = " ".join(p for p in description_parts if p)
logger.info(f"Requesting outfit ideas from Claude for: \"{description}\"")
outfit_result = await loop.run_in_executor(
None, vision_service.get_outfit_ideas_from_description, description
)
outfit = outfit_result.get("outfit", {})
items = outfit.get("items", [])
logger.info(f"Claude suggested outfit: \"{outfit.get('title')}\" with {len(items)} item(s)")
# Match each outfit item to a real product via CLIP text vector + Pinecone top 1
logger.info("Matching each outfit item to a real product via CLIP text vector + Pinecone")
async def match_item(item: dict) -> dict:
query = f"{item.get('color', '')} {item.get('item', '')}".strip()
logger.info(f" Matching outfit item: \"{query}\"")
item_filter = _build_filter(item)
logger.info(
f" Filters: graphical_appearance={item.get('graphical_appearance', 'none')}, "
f"product_group={item.get('product_group', 'none')}, "
f"index_group={item.get('index_group', 'none')}"
)
text_vector = await loop.run_in_executor(None, clip_service.generate_text_vector, query)
matches = await loop.run_in_executor(
None, lambda: pinecone_service.query(text_vector, 1, item_filter)
)
matched = matches[0] if matches else None
logger.info(
f" β†’ Matched: {matched['metadata'].get('name') if matched else 'none'} "
f"(score: {matched['score'] if matched else '-'})"
)
return {
**item,
"matchedProduct": (
{"id": matched["id"], "score": matched["score"], "metadata": matched["metadata"]}
if matched else None
),
}
enriched_items = await asyncio.gather(*[match_item(item) for item in items])
logger.info("Step 3 β€” recommend complete")
return {
"success": True,
"detectedItem": outfit_result.get("detectedItem"),
"claudeRawResponse": outfit_result,
"outfit": {
**outfit,
"items": list(enriched_items),
},
}