Update utils.py

utils.py

@@ -13,6 +13,11 @@ from   langchain.text_splitter     import RecursiveCharacterTextSplitter
 from   langchain.chains            import RetrievalQA
 from   langchain.document_loaders  import TextLoader
 from   langchain.embeddings        import HuggingFaceEmbeddings, OpenAIEmbeddings
+from   langchain.chains            import LLMChain
+from   langchain.evaluation        import StringEvaluator
+from   typing                      import Any, Optional
+from   langsmith                   import Client
+from   langchain.smith             import RunEvalConfig, run_on_dataset
 import streamlit as st
 import together
 import textwrap
@@ -24,6 +29,7 @@ import re
 #os.environ["TOGETHER_API_KEY"] = "6101599d6e33e3bda336b8d007ca22e35a64c72cfd52c2d8197f663389fc50c5"
 #os.environ["OPENAI_API_KEY"]   = "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"
 os.environ["LANGCHAIN_TRACING_V2"] = "true"
+client = Client()
 
 # -- LLM class
 class TogetherLLM(LLM):
@@ -114,6 +120,51 @@ PREGUNTA:""", cleaned_prompt, re.DOTALL)
         text = self.clean_duplicates(text)
         return text
 
+# -- Langchain evaluator
+class RelevanceEvaluator(StringEvaluator):
+    """An LLM-based relevance evaluator."""
+
+    def __init__(self):
+        llm = ChatOpenAI(model="gpt-3.5-turbo", temperature=0)
+
+        template = """En una escala del 0 al 100, ¿Como de relevante es la siguiente salida con respecto a la siguiente entrada?
+        --------
+        ENTRADA: {input}
+        --------
+        SALIDA: {prediction}
+        --------
+        Razona paso a paso porqué el score que has elegido es apropiado y despues muestra la puntuacion al final."""
+
+        self.eval_chain = LLMChain.from_string(llm=llm, template=template)
+
+    @property
+    def requires_input(self) -> bool:
+        return True
+
+    @property
+    def requires_reference(self) -> bool:
+        return False
+
+    @property
+    def evaluation_name(self) -> str:
+        return "scored_relevance"
+
+    def _evaluate_strings(
+        self,
+        prediction: str,
+        input: Optional[str] = None,
+        reference: Optional[str] = None,
+        **kwargs: Any
+    ) -> dict:
+        evaluator_result = self.eval_chain(
+            dict(input=input, prediction=prediction), **kwargs
+        )
+        reasoning, score = evaluator_result["text"].split("\n", maxsplit=1)
+        score = re.search(r"\d+", score).group(0)
+        if score is not None:
+            score = float(score.strip()) / 100.0
+        return {"score": score, "reasoning": reasoning.strip()}
+
 # -- Get GPT response
 def get_gpt_response(transcription_path, query):
     template = """Eres un asistente. Su misión es proporcionar respuestas precisas a preguntas relacionadas con la transcripción de una entrevista de YouTube.
@@ -145,7 +196,29 @@ def get_gpt_response(transcription_path, query):
         | llm
         | StrOutputParser()
     )
-    return rag_chain.invoke(query)
+    llm_output = rag_chain.invoke(query)
+    dataset    = client.create_dataset(dataset_name="Sample LLM dataset", description="A dataset with LLM inputs and outputs", data_type="llm")
+
+    client.create_example(
+      inputs={"input": query},
+      outputs={"output": llm_output},
+      dataset_id=dataset.id,
+    )
+
+    # -- Run custom evaluator
+    evaluation_config = RunEvalConfig(
+        custom_evaluators = [RelevanceEvaluator()],
+    )
+    eval_output = run_on_dataset(
+       dataset_name="Sample LLM dataset",
+       llm_or_chain_factory=rag_chain,
+       evaluation=evaluation_config,
+       client=client,
+    )
+    print("Eval output!!!!")
+    print(eval_output)
+    
+    return llm_output
 
 # -- Python function to setup basic features: SpaCy pipeline and LLM model
 @st.cache_resource