Improve quality of generated description. Add ratings, comments in debug mode

app.py

@@ -1,7 +1,9 @@
+import random
 import traceback
 import gradio as gr 
 import numpy as np
 import os
+from langchain_core.output_parsers import JsonOutputParser
 
 from langchain_openai.chat_models import ChatOpenAI
 from langchain.schema import HumanMessage, SystemMessage, AIMessage
@@ -205,9 +207,9 @@ def get_language(struct_lang, copy_lang):
 
 def get_model(model_name):
     if model_name.startswith("gpt"):
-        chat = ChatOpenAI(model=model_name)
+        chat = ChatOpenAI(model=model_name, max_tokens=8192)
     elif model_name.startswith("claude"):
-        chat = ChatAnthropic(model_name=model_name, anthropic_api_key=os.environ["ANTHROPIC_API_KEY"])
+        chat = ChatAnthropic(model_name=model_name, anthropic_api_key=os.environ["ANTHROPIC_API_KEY"], max_tokens_to_sample=4096)
     else:
         chat = ChatGroq(model_name=model_name, api_key=os.environ["GROQ_API_KEY"])
     return chat
@@ -217,8 +219,8 @@ def generate(*data):
     global visible
     print("visible", visible)
     
-    nargs = 8 
-    feature, image, garment_type, model, temperature, excluded_words, included_words, glossary = data[:nargs]
+    nargs = 9 
+    feature, image, garment_type, model, temperature, excluded_words, included_words, glossary, debug = data[:nargs]
     struct_ref = data[nargs:]
 
     print(f"{feature=}")
@@ -228,8 +230,9 @@ def generate(*data):
     print(f"{temperature=}")
     print(f"{excluded_words=}")
     print(f"{included_words=}")
+    print(f"{debug=}")
     print(f"{glossary=}")
-    print(f"{struct_ref=}")
+    # print(f"{struct_ref=}")
 
     chat = get_model(model)
 
@@ -237,9 +240,9 @@ def generate(*data):
     copy_languages = detect_language([struct_ref[2 * i + 1] for i in range(visible + 1)], model=chat) 
     languages = [get_language(struct_lang=struct_lang, copy_lang=copy_lang) for struct_lang, copy_lang in zip(struct_languages, copy_languages)]
 
-    print("Struct languages--------------------------------------------\n", struct_languages)
-    print("Copy languages--------------------------------------------\n", copy_languages)
-    print("Languages--------------------------------------------\n", languages)
+    # print("Struct languages--------------------------------------------\n", struct_languages)
+    # print("Copy languages--------------------------------------------\n", copy_languages)
+    # print("Languages--------------------------------------------\n", languages)
     # print("Types--------------------------------------------", types)
 
     image_features, base64_images = detect_features(image, garment_type)
@@ -257,56 +260,158 @@ def generate(*data):
         structure = struct_ref[2 * i]
         copy = struct_ref[2 * i + 1]  
         if len((structure + copy).strip()) > 0:
-            if len(copy.strip()) > 0:
+            if len(copy.strip()) > 0 and len(structure.strip()) > 0:    
+                print('------------')
+                print("Using both copy and structure")
+                messages = [
+                    SystemMessage(content=f"""You are a helpful assistant that writes about products for ecommerce websites. Make sure to write in {languages[i]} language."""),
+#                     HumanMessage(content=f"""Write a product description with the following features.
+# Make sure that the structure of the output follows the reference structure.
+# Make sure to use the tone of voice, rythm, cadence and style of the reference copy for the output.
+# Use markdown format for the output.
+# Do not include any part of the reference structure in the output.
+# Do not use any of the excluded words in the output.
+# Make sure to include all of the included words in the output.
+# Output the product description only.
+# Do not hallucinate any information.
+# Generate 5 versions of the product description and rate the quality of each version based on the following criteria:
+#     - how well it follows the reference copy's tone of voice, rythm, cadence and style.
+#     - how well it follows the reference structure.
+#     - how faithful it describes the product features.
+                    HumanMessage(content=f"""Generate 5 versions of the product description for a product with the following information.
+Make sure that the structure of each output follows the reference structure.
+Make sure to use the tone of voice, rythm, cadence and style of the reference copy for each output.
+Use markdown format for each output.
+Do not include any part of the reference structure in the output.
+Do not use any of the excluded words in the output.
+Include all included words in the output.
+Do not hallucinate any information.
+Use creative language in each output.                                 
+Rate the quality of each version based on the following criteria:
+    - how well it follows the reference copy's tone of voice, rythm, cadence and style.
+    - how well it follows the reference structure.
+    - how faithful it describes the product features. 
+    - how creative the language is.                                    
+The score should be a number between 0 and 10 with 10 being the best quality.                              
+Return the result in the following JSON format:
+[
+    {{
+        "id": 1,
+        "content": The first product description,
+        "score": The score of the first product description,
+        "explanation": A less than 20 word explanation of the score of the first product description                         
+    }},
+    {{
+        "id": 2,
+        "content": The second product description,
+        "score": The score of the second product description
+        "explanation": A less than 20 word explanation of the score of the second product description                         
+    }},
+]                              
+Make sure that the output is in JSON format, no extra text should be included in the output.                                
+                                 
+Product information:
+Key features: {feature + detected_features} 
+Intended use: {intended_use} 
+Reference structure: {structure}
+Reference copy: {copy}
+Included words: {included_words}
+Excluded words: {excluded_words}"""),]
+                
+            elif len(copy.strip()) > 0:
                 print('------------')
                 print("Using copy")
                 messages = [
                     SystemMessage(content=f"""You are a helpful assistant that writes about products for ecommerce websites. Make sure to write in {languages[i]} language."""),
-                    HumanMessage(content=f"""Write a product description with the following features.
-Make sure that the output follows the structure and tone of voice of the reference copy.
-Make sure that the output really sounds like the reference copy.
-Use markdown format for the output.
-Output the product description only, do not include any preceeding text like "Here is your product description".
-Use a consistent tone of voice throughout the text. 
+                    HumanMessage(content=f"""Generate 5 versions of the product description for a product with the following information.
+Make sure that the structure of each output follows the structure of the reference copy.
+Make sure to use the tone of voice, rythm, cadence and style of the reference copy for each output.
+Use markdown format for each output.
 Do not include any part of the reference structure in the output.
 Do not use any of the excluded words in the output.
-Make sure to include all of the included words in the output.
+Include all included words in the output.
 Do not hallucinate any information.
-
-{feature + detected_features} 
-{intended_use} 
+Use creative language in each output.                                 
+Rate the quality of each version based on the following criteria:
+    - how well it follows the reference copy's tone of voice, rythm, cadence and style.
+    - how well it follows the reference copy's structure.
+    - how faithful it describes the product features. 
+    - how creative the language is.                                    
+The score should be a number between 0 and 10 with 10 being the best quality.                              
+Return the result in the following JSON format:
+[
+    {{
+        "id": 1,
+        "content": The first product description,
+        "score": The score of the first product description,
+        "explanation": A less than 20 word explanation of the score of the first product description                         
+    }},
+    {{
+        "id": 2,
+        "content": The second product description,
+        "score": The score of the second product description
+        "explanation": A less than 20 word explanation of the score of the second product description                         
+    }},
+]                              
+Make sure that the output is in JSON format, no extra text should be included in the output.                                
+                                 
+Product information:
+Key features: {feature + detected_features} 
+Intended use: {intended_use} 
 Reference copy: {copy}
 Included words: {included_words}
 Excluded words: {excluded_words}"""),]
                 print(messages[1].content)
                 print('------------')
+
             elif len(structure.strip()) > 0:
                 print('------------')
                 print("Using structure")
                 messages = [
                     SystemMessage(content=f"""You are a helpful assistant that writes about products for ecommerce websites. Make sure to write in {languages[i]} language."""),
-                    HumanMessage(content=f"""Write a product description with the following features.
-Make sure that the description follows the structure of the reference structure. 
-Make sure to use markdown format for the output.
-Make sure that the entire output is written entirely in language defined in the reference structure. 
-Make sure to output the product description only, do not include any preceeding text like "Here is your product description".
-Use language that is suitable for the type of document specified in the reference structure. 
-Use a consistent tone of voice throughout the text. 
+                    HumanMessage(content=f"""Generate 5 versions of the product description for a product with the following information.
+Make sure that the structure of each output follows the reference structure.
+Use markdown format for each output.
 Do not include any part of the reference structure in the output.
 Do not use any of the excluded words in the output.
-Make sure to include all of the included words in the output.
+Include all included words in the output.
 Do not hallucinate any information.
-
-{feature + detected_features} 
-{intended_use} 
+Use creative language in each output.                                 
+Rate the quality of each version based on the following criteria:
+    - how well it follows the reference tone of voice, rythm, cadence and style.
+    - how well it follows the reference structure.
+    - how faithful it describes the product features. 
+    - how creative the language is.                                    
+The score should be a number between 0 and 10 with 10 being the best quality.                              
+Return the result in the following JSON format:
+[
+    {{
+        "id": 1,
+        "content": The first product description,
+        "score": The score of the first product description,
+        "explanation": A less than 20 word explanation of the score of the first product description                         
+    }},
+    {{
+        "id": 2,
+        "content": The second product description,
+        "score": The score of the second product description
+        "explanation": A less than 20 word explanation of the score of the second product description                         
+    }},
+]                              
+Make sure that the output is in JSON format, no extra text should be included in the output.                                
+                                 
+Product information:
+Key features: {feature + detected_features} 
+Intended use: {intended_use} 
 Reference structure: {structure}
+Reference copy: {copy}
 Included words: {included_words}
 Excluded words: {excluded_words}"""),]
                 print(messages[1].content)
                 print('------------')
             batch.append(messages)
 
-    description = ""
+    descriptions = ""
     
     response = chat.batch(batch, temperature=temperature)
     print(response)
@@ -344,13 +449,28 @@ Excluded words: {excluded_words}"""),]
     # response = [re_response[rewrite_map[i]] if i in rewrite_map else response[i] for i in range(visible + 1)]
     # print("Done rewriting")
 
-    description = "\n---\n".join([msg.content for msg in response])
-    md_content = description 
+    parser = JsonOutputParser()
+    jresponse = [parser.parse(msg.content) for msg in response]
+    descriptions = []
+    for jr in jresponse:
+        bests = 0
+        bestd = ""
+        for d in jr:
+            print(f'{d["score"]=}, {d["id"]=}, {bests=}')
+            if d["score"] > bests:
+                bests = d["score"]
+                bestd = d["content"] + (f"\n\nDebug info:\n\nScore: {d['score']}\n\nExplanation: {d['explanation']}" if debug else "")
+            elif d["score"] == bests and random.random() > 0.5:
+                bestd = d["content"] + (f"\n\nDebug info:\n\nScore: {d['score']}\n\nExplanation: {d['explanation']}" if debug else "")
+
+        descriptions.append(bestd)
+    # description = "\n\n\n\n".join([msg.content for msg in response])
+    md_content = "\n\n\n".join(descriptions)
 
     alt_texts_str = '\n\n### Alt text\n\n' + '\n- ' + '\n- '.join(alt_texts) if len(alt_texts) > 0 else ""
 
     alt_text_dict = {k[0]: v for (k, v) in zip(image, alt_texts)} if len(alt_texts) > 0 else {}
-    result_json = {"outputs": [msg.content for msg in response], "alt_text": alt_text_dict}
+    result_json = {"outputs": descriptions, "alt_text": alt_text_dict}
     result_md = md_content + alt_texts_str + '\n'.join([f'![Product photo](data:image/png;base64,{base64_image} "{alt_text}")' if base64_image != "" else "" for (base64_image, alt_text) in zip(base64_images, alt_texts)])
     return result_md, result_json
 
@@ -402,11 +522,12 @@ with gr.Blocks() as demo:
             garment_type = gr.Textbox(label="Garment Type", value="all", lines=1, interactive=True) 
             # language = gr.Dropdown(languages, value="American English", interactive=True, label="Language")
             with gr.Accordion(label="Advanced Options", open=False):
-                model = gr.Dropdown(models, value="gpt-4-turbo", interactive=True, label="Model", visible=True)
+                model = gr.Dropdown(models, value="claude-3-5-sonnet-20240620", interactive=True, label="Model", visible=True)
                 temperature = gr.Slider(minimum=0., maximum=1.0, value=0., interactive=True, label="Temperature", visible=True)
                 excluded_words = gr.Textbox(label="Excluded words", interactive=True, lines=2)
                 included_words = gr.Textbox(label="Included words", interactive=True, lines=2)
                 glossary = gr.Dataframe(row_count = (2, "dynamic"), col_count=(2,"static"), headers=["Description", "Way of writing"], label="Glossary", interactive=True)
+                debug = gr.Checkbox(label="Debug", interactive=True, value=True)
             with gr.Row():
                 submit = gr.Button(value="Submit")
                 # advanced = gr.Button(value="Advanced")
@@ -426,7 +547,8 @@ with gr.Blocks() as demo:
         with gr.Column():
             md_output = gr.Markdown(label="Output", show_label=True)
             json_output = gr.JSON(label="JSON Output")  
-        submit.click(generate, inputs=[feature, image, garment_type, model, temperature, excluded_words, included_words, glossary, *struct_ref], 
+        submit.click(generate, inputs=[feature, image, garment_type, model, temperature, 
+                                       excluded_words, included_words, glossary, debug, *struct_ref], 
                      outputs=[md_output, json_output])
         # advanced.click(show_advanced, inputs=[], outputs=[model, temperature])