step 2

app.py

@@ -86,7 +86,7 @@ async def forward_request(
         service = AIServiceFactory.get_service(ai_vendor)
 
         try:
-            json_attributes = await service.extract_attributes_with_validation(
+            json_attributes, reevaluated = await service.extract_attributes_with_validation(
                 Product,  # type: ignore
                 ai_model,
                 None,
@@ -101,7 +101,7 @@ async def forward_request(
         shutil.rmtree(request_temp_folder)
 
     gr.Info("Process completed!")
-    return json_attributes
+    return json_attributes, reevaluated
 
 
 def add_attribute_schema(attributes, attr_name, attr_desc, attr_type, allowed_values):
@@ -380,6 +380,9 @@ with gr.Blocks(title="Internal Demo for Attribution") as demo:
             output_json = gr.Json(
                 label="Extracted Attributes", value={}, show_indices=False
             )
+            reevaluated_output_json = gr.Json(
+                label="Extracted Attributes", value={}, show_indices=False
+            )
 
     # add_btn.click(
     #     add_attribute_schema,
@@ -390,7 +393,7 @@ with gr.Blocks(title="Internal Demo for Attribution") as demo:
     submit_btn.click(
         forward_request,
         inputs=[attributes, product_taxnomy, product_data, ai_model, gallery],
-        outputs=output_json,
+        outputs=[output_json, reevaluated_output_json],
     )
 
 

app/core/prompts.py

@@ -28,6 +28,13 @@ You should use the following product data to assist you, if available:
 If an attribute appears in both the image and the product data, use the value from the product data.
 """
 
+REEVALUATE_SYSTEM = "You are an expert in structured data extraction. You will be given an image or a set of images of a product and set of attributes and should reevaluate certainity of the attributes into the given structure."
+
+REEVALUATE_HUMAN = """Reevaluate the following attributes of the main product (or {product_taxonomy}) shown in the images. Here are the attributes to reevaluate:
+{product_data}
+
+If an attribute can have multiple values, do not need to reevaluate the values, just the attribute itself. If an attribute can have only one value, reevaluate the top three values. 
+"""
 
 class Prompts(BaseSettings):
     EXTRACT_INFO_SYSTEM_MESSAGE: str = EXTRACT_INFO_SYSTEM
@@ -42,6 +49,10 @@ class Prompts(BaseSettings):
 
     GET_PERCENTAGE_HUMAN_MESSAGE: str = GET_PERCENTAGE_HUMAN
 
+    REEVALUATE_SYSTEM_MESSAGE: str = REEVALUATE_SYSTEM
+    
+    REEVALUATE_HUMAN_MESSAGE: str = REEVALUATE_HUMAN
+
 
 # Create a cached instance of settings
 @lru_cache

app/services/base.py

@@ -11,6 +11,116 @@ from app.schemas.schema_tools import (
 )
 
 
+example_data = example_data = {
+    "length": {
+        "maxi": 100,
+        "knee_length": 0,
+        "mini": 0,
+        "midi": 0
+    },
+    "style": {
+        "a_line": 0,
+        "bodycon": 0,
+        "shirt_dress": 0,
+        "wrap_dress": 0,
+        "slip": 0,
+        "smock": 0,
+        "corset": 100,
+        "jumper_dress": 0,
+        "blazer_dress": 0,
+        "asymmetric": 0,
+        "shift": 0,
+        "drop_waist": 0,
+        "empire": 0,
+        "modest": 0
+    },
+    "sleeve_length": {
+        "sleeveless": 0,
+        "three_quarters_sleeve": 0,
+        "long_sleeve": 0,
+        "short_sleeve": 0,
+        "strapless": 100
+    },
+    "neckline": {
+        "v_neck": 0,
+        "sweetheart": 100,
+        "round_neck": 0,
+        "halter_neck": 0,
+        "square_neck": 0,
+        "high_neck": 0,
+        "crew_neck": 0,
+        "turtle_neck": 0,
+        "off_the_shoulder": 0,
+        "one_shoulder": 0,
+        "boat_neck": 0
+    },
+    "pattern": {
+        "floral": 0,
+        "stripe": 0,
+        "leopard_print": 0,
+        "plain": 100,
+        "geometric": 0,
+        "logo": 0,
+        "graphic_print": 0,
+        "other": 0
+    },
+    "fabric": {
+        "cotton": 0,
+        "denim": 0,
+        "linen": 0,
+        "satin": 0,
+        "silk": 0,
+        "sequin": 0,
+        "leather": 0,
+        "velvet": 100,
+        "knit": 0,
+        "lace": 0,
+        "suede": 0,
+        "sheer": 0,
+        "polyester": 0,
+        "viscose": 0
+    },
+    "features": {
+        "pockets": 0,
+        "lined": 0,
+        "cut_out": 0,
+        "backless": 0,
+        "none": 100
+    },
+    "closure": {
+        "button": 0,
+        "zip": 0,
+        "press_stud": 0,
+        "clasp": 0
+    },
+    "body_fit": {
+        "petite": 0,
+        "maternity": 0,
+        "regular": 100,
+        "tall": 0,
+        "plus_size": 0
+    },
+    "occasion": {
+        "beach": 0,
+        "casual": 0,
+        "cocktail": 0,
+        "day": 0,
+        "evening": 100,
+        "mother_of_the_bride": 0,
+        "party": 0,
+        "prom": 0,
+        "wedding_guest": 0,
+        "work": 0,
+        "sportswear": 0
+    },
+    "season": {
+        "spring": 0,
+        "summer": 0,
+        "autumn": 0,
+        "winter": 100
+    }
+}
+
 class BaseAttributionService(ABC):
     @abstractmethod
     async def extract_attributes(
@@ -23,6 +133,17 @@ class BaseAttributionService(ABC):
     ) -> Dict[str, Any]:
         pass
 
+    @abstractmethod
+    async def reevaluate_atributes(
+        self,
+        attributes_model: Type[BaseModel],
+        ai_model: str,
+        img_urls: List[str],
+        product_taxonomy: str,
+        pil_images: List[Any] = None,
+    ) -> Dict[str, Any]:
+        pass
+
     @abstractmethod
     async def follow_schema(
         self, schema: Dict[str, Any], data: Dict[str, Any]
@@ -52,9 +173,37 @@ class BaseAttributionService(ABC):
             # pil_images=pil_images, # temporarily removed to save cost
             img_paths=img_paths,
         )
+        # data = example_data
         validate_json_data(data, schema)
 
-        return data
+        str_data = str(data)
+        reevaluate_data = await self.reevaluate_atributes(
+            attributes_model,
+            ai_model,
+            img_urls,
+            product_taxonomy if product_taxonomy != "" else "main",
+            str_data,
+            # pil_images=pil_images, # temporarily removed to save cost
+            img_paths=img_paths,
+        )
+
+        init_reevaluate_data = {}
+        for field_name, field in attributes_model.model_fields.items(): # type: ignore
+            print(f"{field_name}: {field.description}")
+            if "single value" in field.description.lower():
+                max_percentage = 0
+                for k, v in reevaluate_data[field_name].items():
+                    if v > max_percentage:
+                        max_percentage = v
+                        init_reevaluate_data[field_name] = k
+            elif "multiple values" in field.description.lower():
+                init_list = []
+                for k, v in reevaluate_data[field_name].items():
+                    if v >= 60:
+                        init_list.append(k)
+                init_reevaluate_data[field_name] = init_list
+
+        return data, init_reevaluate_data
 
     async def follow_schema_with_validation(
         self, schema: Dict[str, Any], data: Dict[str, Any]

app/services/service_openai.py

@@ -147,6 +147,95 @@ class OpenAIService(BaseAttributionService):
             raise VendorError(errors.VENDOR_ERROR_INVALID_JSON)
 
         return parsed_data
+    
+    async def reevaluate_atributes(
+        self,
+        attributes_model: Type[BaseModel],
+        ai_model: str,
+        img_urls: List[str],
+        product_taxonomy: str,
+        product_data: str,
+        pil_images: List[Any] = None,  # do not remove, this is for weave
+        img_paths: List[str] = None,
+    ) -> Dict[str, Any]:
+
+        print("Prompt: ")
+        print(prompts.REEVALUATE_HUMAN_MESSAGE.format(product_taxonomy=product_taxonomy, product_data=product_data))
+        
+        text_content = [
+            {
+                "type": "text",
+                "text": prompts.REEVALUATE_HUMAN_MESSAGE.format(
+                    product_taxonomy=product_taxonomy,
+                    product_data=product_data,
+                ),
+            },
+        ]
+        if img_urls is not None:
+            base64_data_list = []
+            data_format_list = []
+
+            for img_url in img_urls:
+                base64_data, data_format = get_image_base64_and_type(img_url)
+                base64_data_list.append(base64_data)
+                data_format_list.append(data_format)
+
+            image_content = [
+                {
+                    "type": "image_url",
+                    "image_url": {
+                        "url": f"data:image/{data_format};base64,{base64_data}",
+                    },
+                }
+                for base64_data, data_format in zip(base64_data_list, data_format_list)
+            ]
+        elif img_paths is not None:
+            image_content = [
+                {
+                    "type": "image_url",
+                    "image_url": {
+                        "url": f"data:image/{get_data_format(img_path)};base64,{get_image_data(img_path)}",
+                    },
+                }
+                for img_path in img_paths
+            ]
+
+        try:
+            logger.info("Extracting info via OpenAI...")
+            response = await self.client.beta.chat.completions.parse(
+                model=ai_model,
+                messages=[
+                    {
+                        "role": "system",
+                        "content": prompts.REEVALUATE_SYSTEM_MESSAGE,
+                    },
+                    {
+                        "role": "user",
+                        "content": text_content + image_content,
+                    },
+                ],
+                max_tokens=1000,
+                response_format=attributes_model,
+                logprobs=False,
+                # top_logprobs=2,
+                # temperature=0.0,
+                top_p=1e-45,
+            )
+        except openai.BadRequestError as e:
+            error_message = exception_to_str(e)
+            raise BadRequestError(error_message)
+        except Exception as e:
+            raise VendorError(
+                errors.VENDOR_THROW_ERROR.format(error_message=exception_to_str(e))
+            )
+
+        try:
+            content = response.choices[0].message.content
+            parsed_data = json.loads(content)
+        except:
+            raise VendorError(errors.VENDOR_ERROR_INVALID_JSON)
+
+        return parsed_data
 
     @weave.op
     async def follow_schema(