Update app.py

app.py

@@ -330,6 +330,106 @@ async def search_tavily(
             "query": query
         }
 
+##### EVALS #####
+async def evaluate_answer_relevancy(
+    query: str,
+    response: str,
+) -> float:
+    """Evaluate how relevant the answer is to the query using AnswerRelevancyEvaluator.
+    
+    Args:
+        query: Original user query (required)
+        response: Generated response to evaluate (required)
+        
+    Returns:
+        float: Relevancy score between 0 and 1 (higher is better)
+    """
+    try:
+        from llama_index.core.evaluation import AnswerRelevancyEvaluator
+        
+        # Initialize the evaluator
+        evaluator = AnswerRelevancyEvaluator(llm=llm)
+        
+        # Perform the evaluation
+        eval_result = evaluator.evaluate(query=query, response=response)
+        
+        # Return the score as a float
+        return float(eval_result.score)
+        
+    except Exception as e:
+        # In case of error, return 0.0 (minimum score) and log the error
+        print(f"Error in relevancy evaluation: {str(e)}")
+        return 0.0
+
+async def evaluate_context_relevancy(
+    context: str,
+    query: str,
+    response: str
+) -> float:
+    """Evaluates the relevance of the response considering both the query and the context.
+    
+    Args:
+        context: Contextual information / knowledge base (required)
+        query: Original user query (required)
+        response: Generated response to evaluate (required)
+        
+    Returns:
+        float: Relevance score between 0 and 1 (higher is better)
+    """
+    try:
+        from llama_index.core.evaluation import ContextRelevancyEvaluator
+
+        # Initialize the relevancy evaluator with context
+        evaluator = ContextRelevancyEvaluator(llm=llm)
+        
+        # Perform the evaluation (adapted to handle context)
+        eval_result = evaluator.evaluate(
+            query=query,
+            response=response,
+            contexts=[context]
+        )
+        
+        return float(eval_result.score)
+        
+    except Exception as e:
+        print(f"Error during context relevancy evaluation: {str(e)}")
+        return 0.0
+
+async def evaluate_faithfulness(
+    query: str,
+    response: str,
+    context: str
+) -> float:
+    """Evaluate how faithful (factually consistent) the response is to the provided context.
+    
+    Args:
+        query: Original user query (required)
+        response: Generated response to evaluate (required)
+        context: Source context/knowledge base used for the response (required)
+        
+    Returns:
+        float: Faithfulness score between 0 and 1 (higher is better)
+    """
+    try:
+        from llama_index.core.evaluation import FaithfulnessEvaluator
+        
+        # Initialize evaluator
+        evaluator = FaithfulnessEvaluator(llm=llm)
+        
+        # Perform evaluation
+        eval_result = evaluator.evaluate(
+            query=query,
+            response=response,
+            contexts=[context]
+        )
+        
+        # Return score as float
+        return float(eval_result.score)
+        
+    except Exception as e:
+        # On error, return 0.0 (minimum score) and log the error
+        print(f"Error in faithfulness evaluation: {str(e)}")
+        return 0.0
 
 
 
@@ -399,10 +499,51 @@ with gr.Blocks(title="MCP Tools", theme=gr.themes.Base()) as retrieve_tab:
         api_name="_retrieve"
     )
 
+with gr.Blocks(title="MCP Tools", theme=gr.themes.Base()) as asw_relevance_tab:
+    relevancy_interface = gr.Interface(
+        fn=evaluate_answer_relevancy,
+        inputs=[
+            gr.Textbox(label="Original Query", placeholder="E.g.: How does photosynthesis work?"),
+            gr.Textbox(label="Answer to Evaluate", placeholder="Paste the generated answer here", lines=5),
+        ],
+        outputs=gr.Number(label="Relevancy Score (0-1)", precision=3),
+        title="Relevancy Evaluator (Query-Answer)",
+        description="Evaluates how relevant an answer is to the original query (1 = perfectly relevant).",
+        api_name="_evaluate_relevancy"
+    )
+
+with gr.Blocks(title="MCP Tools", theme=gr.themes.Base()) as ctx_relevance_tab:
+    context_relevancy_interface = gr.Interface(
+        fn=evaluate_context_relevancy,
+        inputs=[
+            gr.Textbox(label="Context", placeholder="Relevant text / knowledge base", lines=3),
+            gr.Textbox(label="Original Query", placeholder="What question is being answered?"),
+            gr.Textbox(label="Generated Answer", placeholder="The answer to evaluate", lines=5),
+        ],
+        outputs=gr.Number(label="Relevancy Score (0-1)", precision=3),
+        title="Relevancy Evaluator (Context-Query-Answer)",
+        description="Evaluates how relevant the answer is considering both the query and the reference context.",
+        api_name="_evaluate_context_relevancy"
+    )
+
+with gr.Blocks(title="MCP Tools", theme=gr.themes.Base()) as faithfulness_tab:
+    faithfulness_interface = gr.Interface(
+        fn=evaluate_faithfulness,
+        inputs=[
+            gr.Textbox(label="Original Query", placeholder="E.g.: What are the causes of climate change?"),
+            gr.Textbox(label="Answer to Evaluate", placeholder="Paste the generated answer here", lines=5),
+            gr.Textbox(label="Context", placeholder="Reference text / knowledge base", lines=3),
+        ],
+        outputs=gr.Number(label="Faithfulness Score (0-1)", precision=3),
+        title="Faithfulness Evaluator",
+        description="Evaluates how faithful/factually consistent the answer is with respect to the provided context (1 = perfectly faithful).",
+        api_name="_evaluate_faithfulness"
+    )
+
 # Create the interface with separate tabs
 demo = gr.TabbedInterface(
-    [arxiv_tab, tavily_tab, list_retrievers_tab, retrieve_tab],
-    ["ArXiv", "Tavily", "List Retrievers", "Retrieve"]
+    [arxiv_tab, tavily_tab, list_retrievers_tab, retrieve_tab, asw_relevance_tab, ctx_relevance_tab, faithfulness_tab],
+    ["ArXiv", "Tavily", "List Retrievers", "Retrieve", "Answer Relevance", "Context Relevance", "Faithfulness"]
 )
 
 demo.launch(mcp_server=True)