added ux_testing api

.gitignore

@@ -7,4 +7,6 @@ eval/__pycache__/
 eval/synthetic/
 test_lite_llm.py
 test_gemini.py
-test/
+test/
+ux_test.py
+images.json

app.py

@@ -10,7 +10,8 @@ from schemas import (
     GenerateParametersRequest, GenerateParametersResponse,
     GenerateTwinRequest, GenerateTwinResponse,
     ChatWithTwinRequest, ChatWithTwinResponse,
-    GenerateUsersAnswersRequest
+    GenerateUsersAnswersRequest,
+    UXTestingRequest, UXTestingResponse
 )
 import uvicorn
 import logging
@@ -229,4 +230,35 @@ def chat_with_twin_endpoint(request: ChatWithTwinRequest, x_api_key: str = Heade
     # Return the results in the format defined by ChatWithTwinResponse
     return ChatWithTwinResponse(answer=answer, new_memory=new_memory)
 
+# Endpoint for UX Testing with Images
+@app.post("/ux-testing", 
+    response_model=UXTestingResponse,
+    summary="Conduct UX Testing with Finite State Machine",
+    description="Conduct interactive UX testing with a persona using images. The persona will choose an action to take, provide reasoning, and indicate if the task is complete."
+)
+def ux_testing_endpoint(request: UXTestingRequest, x_api_key: str = Header(...)):
+    if x_api_key != API_KEY:
+        logger.warning("Unauthorized access attempt to /ux-testing.")
+        raise HTTPException(status_code=403, detail="Invalid API Key")
+    
+    # Import the core function
+    from core import ux_testing_fsm
+
+    logger.info(f"Starting UX FSM testing with persona: {request.user.get('Name', 'Unknown')}")
+    logger.info(f"Task: {request.task}")
+    logger.info(f"Available actions: {request.available_actions}")
+    
+    # Call the core function
+    result = ux_testing_fsm(
+        persona=request.user,
+        task=request.task,
+        image=request.image,
+        available_actions=request.available_actions,
+        session_history=request.session_history
+    )
+    
+    logger.info("UX FSM testing completed successfully.")
+    # Return the results in the format defined by UXTestingResponse
+    return UXTestingResponse(**result)
+
 if __name__ == "__main__": uvicorn.run("app:app", host="0.0.0.0", port=8000, reload=False)

core.py

@@ -12,6 +12,7 @@ from prompts import (
     GENERATE_REPORT_PROMPT,
     CHAT_WITH_REPORT_PROMPT,
     GENERATE_AUDIENCE_NAME_PROMPT,
+    UX_FSM_SIMPLE_PROMPT,
     persona_schema,
     answers_schema
 )
@@ -318,17 +319,120 @@ def chat_with_report(users: List[dict], question: str, questions: List[str]) ->
 
 def generate_audience_name(audience: str, scope: str) -> str:
     """
-    Generate a concise, descriptive name for the audience based on the research scope.
+    Generate a concise audience name based on the provided audience description and scope.
     
     Args:
-        audience: The target audience description
-        scope: The research scope
-    
+        audience: Detailed audience description
+        scope: Research scope
+        
     Returns:
-        A concise, descriptive name for the audience
+        String containing a concise audience name
     """
     prompt = GENERATE_AUDIENCE_NAME_PROMPT.format(
         audience=audience,
         scope=scope
     )
-    return call_llm(prompt=prompt,model_type="low",temperature=0)
+    
+    audience_name = call_llm(prompt=prompt, temperature=0, model_type="low")
+    return audience_name.strip()
+
+def ux_testing_fsm(persona: dict, task: str, image: str, available_actions: list, session_history: list = None) -> dict:
+    """
+    Conduct simple FSM-based UX testing with a persona.
+    
+    Args:
+        persona: User persona to conduct testing with
+        task: The task the persona needs to accomplish
+        image: URL of the current interface image
+        available_actions: List of available actions in current state
+        session_history: List of previous steps in this session
+        
+    Returns:
+        Dictionary with action_taken, thought, task_finished, and task_difficulty
+    """
+    
+    # Format available actions
+    actions_text = ", ".join(available_actions)
+    
+    # Format session history
+    if session_history:
+        history_text = "Previous steps in this session:\n"
+        for i, step in enumerate(session_history, 1):
+            history_text += f"Step {i}: Action '{step.get('action_taken', 'unknown')}' - {step.get('thought', 'No thought recorded')}\n"
+    else:
+        history_text = "This is the first step of the session."
+    
+    prompt = UX_FSM_SIMPLE_PROMPT.format(
+        persona=persona,
+        task=task,
+        available_actions=actions_text,
+        session_history=history_text
+    )
+    
+    # Define response format for structured JSON
+    response_format = {
+        "type": "json_schema",
+        "json_schema": {
+            "name": "ux_testing_response",
+            "schema": {
+                "type": "object",
+                "properties": {
+                    "action_taken": {
+                        "type": "string",
+                        "description": "The action chosen from available actions",
+                        "enum": available_actions
+                    },
+                    "thought": {
+                        "type": "string", 
+                        "description": "Reasoning for the action"
+                    },
+                    "task_finished": {
+                        "type": "boolean",
+                        "description": "Whether the task is complete"
+                    },
+                    "task_difficulty": {
+                        "type": ["number", "null"],
+                        "minimum": 1.0,
+                        "maximum": 5.0,
+                        "description": "Difficulty rating if task is finished"
+                    }
+                },
+                "required": ["action_taken", "thought", "task_finished", "task_difficulty"],
+                "additionalProperties": False
+            },
+            "strict": True
+        }
+    }
+    
+    try:
+        # Call LLM with the image and structured response format
+        response = call_llm(
+            prompt=prompt,
+            temperature=0.7,
+            model_type="mid",
+            images=[image],
+            response_format=response_format
+        )
+        
+        # Parse JSON response
+        parsed_response = json.loads(response)["ux_testing_response"]
+        
+        # Validate action is in available actions
+        if parsed_response.get("action_taken") not in available_actions:
+            logger.warning(f"Persona chose invalid action: {parsed_response.get('action_taken')}. Using first available action.")
+            parsed_response["action_taken"] = available_actions[0] if available_actions else "unknown"
+        
+        logger.info(f"UX FSM testing completed for persona: {persona.get('Name', 'Unknown')}")
+        logger.info(f"Action taken: {parsed_response.get('action_taken')}")
+        logger.info(f"Task finished: {parsed_response.get('task_finished')}")
+        
+        return parsed_response
+        
+    except Exception as e:
+        logger.error(f"Error during UX FSM testing for persona {persona.get('Name', 'Unknown')}: {e}")
+        return {
+            "action_taken": available_actions[0] if available_actions else "unknown",
+            "thought": f"Error occurred during testing: {str(e)}",
+            "task_finished": False,
+            "task_difficulty": None
+        }

prompts.py

@@ -528,30 +528,97 @@ def persona_schema(n):
     return persona_schema
 
 def answers_schema(n):
-    answers_schema={
-        "type": "json_schema",
-        "json_schema": {
-                "name": "answers_list",
-                "schema": {
-                    "type": "object",
-                    "properties": {
-                    "answers": {
-                        "type": "array",
-                        "description": f"A list of answers to questions, with exactly {n} elements.",
-                        "items": {
+    return {
+        "type": "object",
+        "properties": {
+            "answers": {
+                "type": "array",
+                "items": {"type": "string"},
+                "minItems": n,
+                "maxItems": n,
+                "description": f"Array of exactly {n} answers to the questions"
+            }
+        },
+        "required": ["answers"],
+        "additionalProperties": False
+    }
+
+def ux_fsm_schema(actions):
+    return{
+        "type": "object",
+        "properties": {
+            "ux_response": {
+                "type": "object",
+                "description": "The response from the UX testing",
+                "properties": {
+                    "action_taken": {
                         "type": "string",
-                        "description": "Each answer corresponding to a question."
-                        },
-                        "minItems": n,
-                        "maxItems": n
-                    }
+                        "description": "The action the persona decided to take",
+                        "enum": actions
+                    },
+                    "thought": {
+                        "type": "string",
+                        "description": "The persona's reasoning for taking this action"
+                    },
+                    "task_finished": {
+                        "type": "boolean",
+                        "description": "Whether the persona believes they have completed the task"
                     },
-                    "required": [
-                    "answers"
-                    ],
-                    "additionalProperties": False
+                    "task_difficulty": {
+                        "type": "number",
+                        "description": "The difficulty of the task, from 1 (very easy) to 5 (very difficult). If not finished, set to None.",
+                        "enum": [1, 2, 3, 4, 5, None]
+                    }
                 },
-                "strict": True
             }
+        },
+        "required": ["ux_response"],
+        "additionalProperties": False
     }
-    return answers_schema
+
+# Simple FSM UX Testing Prompt
+UX_FSM_SIMPLE_PROMPT = """
+You are conducting a UX testing session as the detailed user persona provided below. You are looking at a specific interface screen and need to decide what action to take to accomplish your given task.
+
+**Persona Profile:**
+{persona}
+
+**Task to Accomplish:**
+{task}
+
+**Current Screen:**
+You are viewing the interface shown in the provided image.
+
+**Available Actions:**
+Choose ONE of these actions: {available_actions}
+
+**Session History:**
+{session_history}
+
+**Your Response:**
+
+Provide your response as a JSON object with exactly these fields:
+
+{{
+  "action_taken": "your_chosen_action",
+  "thought": "your reasoning for this action",
+  "task_finished": true/false,
+  "task_difficulty": 1.0-5.0 (only if task_finished is true, otherwise null)
+}}
+
+**Guidelines:**
+* **action_taken**: Select exactly ONE action from the available actions list
+* **thought**: Explain your reasoning for choosing this action. Why does this action make sense for accomplishing your task? What do you expect to happen? Consider your previous actions from the session history.
+* **task_finished**: Answer true ONLY if you have completely finished the task (e.g., item successfully added to cart and you're at checkout), false if you need to continue
+* **task_difficulty**: If task_finished is true, rate the overall difficulty from 1.0 (very easy) to 5.0 (very difficult). If task is not finished, set to null.
+
+**Important Instructions:**
+* Respond as the persona - use their knowledge, experience, and perspective
+* Consider your session history when making decisions - don't repeat unnecessary actions
+* Choose the action that best helps accomplish the given task
+* Be specific and clear in your reasoning
+* Consider your persona's technical comfort level and preferences
+* Only choose actions from the provided available actions list
+* **Task Completion**: Only mark task_finished as true when you have FULLY completed the entire task (not just found a product, but actually added it to cart and reached checkout)
+* Respond with valid JSON only - no additional text
+"""

schemas.py

@@ -287,10 +287,71 @@ class ChatWithTwinResponse(BaseModel):
         example="Based on my knowledge, my favorite color isn't specified, but I appreciate aesthetics in design."
     )
     new_memory: List[Dict[str, Any]] = Field(...,
-        description="The updated memory of the digital twin, including the latest interaction.",
+        description="The updated memory list of the digital twin after the conversation.",
         example=[
-            # ... previous memory items ...
-            {'role': 'user', 'content': {'stimuli': [{'type': 'SPEECH', 'content': 'What is your favorite color?'}]}},
-            {'role': 'assistant', 'content': {'actions': [{'type': 'TALK', 'content': 'Based on my knowledge... aesthetics in design.'}]}}
+            {
+                "id": 1,
+                "kind": "chat_message",
+                "content": "The user asked about my favorite color, and I responded that it's not specified but I appreciate aesthetics.",
+                "importance": 2
+            }
         ]
     )
+
+class UXTestingRequest(BaseModel):
+    user: Dict[str, Any] = Field(..., 
+        description="User persona to conduct UX testing with", 
+        example={
+            "Name": "Sarah Chen",
+            "Age": "32",
+            "Profession": "UX Designer",
+            "Location": "San Francisco"
+        }
+    )
+    task: str = Field(..., 
+        description="The UX task the persona needs to accomplish", 
+        example="Find and add a red t-shirt to your shopping cart"
+    )
+    image: str = Field(...,
+        description="URL of the current screen/interface image",
+        example="https://example.com/current-screen.png"
+    )
+    available_actions: List[str] = Field(...,
+        description="List of actions the persona can take in the current state",
+        example=["scroll_down", "click_search", "click_menu", "click_product"]
+    )
+    session_history: List[Dict] = Field(default=[], 
+        description="Previous steps in this UX testing session",
+        example=[
+            {
+                "image": "https://example.com/homepage.png",
+                "action_taken": "click_search",
+                "thought": "I clicked search to find the product"
+            },
+            {
+                "image": "https://example.com/search-page.png", 
+                "action_taken": "type_search_term",
+                "thought": "I typed 'red t-shirt' to find what I need"
+            }
+        ]
+    )
+
+class UXTestingResponse(BaseModel):
+    action_taken: str = Field(...,
+        description="The action the persona decided to take",
+        example="click_search"
+    )
+    thought: str = Field(...,
+        description="The persona's reasoning and thought process for taking this action",
+        example="I need to find a red t-shirt, so clicking the search button seems like the most direct way to locate what I'm looking for."
+    )
+    task_finished: bool = Field(...,
+        description="Whether the persona believes they have completed the task",
+        example=False
+    )
+    task_difficulty: Optional[float] = Field(default=None,
+        description="If task is finished, rate the difficulty from 1.0 (very easy) to 5.0 (very difficult)",
+        ge=1.0,
+        le=5.0,
+        example=2.5
+    )

utils.py

@@ -14,7 +14,7 @@ model_low="openai/gpt-4.1-nano"
 model_mid="openai/gpt-4.1-mini"
 model_high="openai/gpt-4.1"
 
-def call_llm(prompt: str, temperature: float,model_type: str,response_format=None,tools=None,shuffle=False,return_tokens=False) -> str:
+def call_llm(prompt: str, temperature: float, model_type: str, response_format=None, tools=None, shuffle=False, return_tokens=False, images=None) -> str:
     if shuffle:
         if model_type=="low":
             model = random.choice(models_low)
@@ -31,9 +31,19 @@ def call_llm(prompt: str, temperature: float,model_type: str,response_format=Non
         elif model_type=="high":
             model = model_high
 
-    messages=[
-        {"role": "user", "content": prompt},
-    ]
+    # Create message content - support both text-only and multimodal
+    if images:
+        # Multimodal message with images
+        content = [{"type": "text", "text": prompt}]
+        for image_url in images:
+            content.append({
+                "type": "image_url",
+                "image_url": {"url": image_url}
+            })
+        messages = [{"role": "user", "content": content}]
+    else:
+        # Text-only message
+        messages = [{"role": "user", "content": prompt}]
     
     completion_args = {
         "model": model,