tests(suggest_expectations): write test cases

app/tests/suggest_expectations_test.py

@@ -0,0 +1,84 @@
+from langchain_openai import ChatOpenAI
+from app.workflows.courses.suggest_expectations import SuggestExpectations
+from langsmith.evaluation import LangChainStringEvaluator, evaluate
+from langsmith.schemas import Example, Run
+from typing import Any, Optional, TypedDict
+
+database_name = "course-learn-suggest-expectations"
+evaluator_llm = ChatOpenAI(model="gpt-4o-mini", temperature=0)
+
+
+class SingleEvaluatorInput(TypedDict):
+    """The input to a `StringEvaluator`."""
+
+    prediction: str
+    """The prediction string."""
+    reference: Optional[Any]
+    """The reference string."""
+    input: Optional[str]
+    """The input string."""
+
+
+def generate_expectations(example: dict):
+    chain = SuggestExpectations()._build_chain()
+    response = chain.invoke({
+        "course": example["course"], "module": example["module"], "tasks": example["tasks"],
+        "format_instructions": example["format_instructions"],
+        "existing_expectations": example["existing_expectations"]
+    })
+    return response
+
+
+def similarity_search(org_str, test_strs):
+    most_similar = None
+    min_similarity = float('inf')
+
+    similarity_qa_evaluator = LangChainStringEvaluator(
+        "embedding_distance",
+        config={"distance_metric": "cosine"},
+    )
+
+    for test_itr in test_strs:
+        eval_inputs = SingleEvaluatorInput(
+            prediction=org_str,
+            reference=test_itr
+        )
+
+        result = similarity_qa_evaluator.evaluator.evaluate_strings(
+            **eval_inputs)
+        similarity_distance = result['score']
+
+        if abs(similarity_distance) < min_similarity:
+            similarity = 1 - similarity_distance
+            result['score'] = similarity
+            most_similar = {"key": "similarity", **result,
+                          "prediction": test_itr,
+                          "reference": org_str}
+            min_similarity = abs(similarity_distance)
+
+    if most_similar:
+        return most_similar
+
+
+def custom_evaluator(root_run: Run, example: Example) -> dict:
+    results = []
+    for output_expectation_obj in root_run.outputs['expectations']:
+        output_expectation = output_expectation_obj['expectation']
+        most_similar = similarity_search(
+            output_expectation,
+            [item["expectation"] for item in example.outputs["expectations"]]
+        )
+        results.append(most_similar)
+
+    return {"results": results}
+
+
+def build_evaluators():
+    response = evaluate(
+        generate_expectations,
+        data=database_name,
+        evaluators=[custom_evaluator],
+        experiment_prefix="alpha",
+    )
+
+build_evaluators()

app/workflows/courses/suggest_check_question.py

@@ -19,6 +19,7 @@ class Response(BaseModel):
     expectation: str
     check_question: str
 
+
 class CheckQuestion(BaseModel):
     check_question: str = Field(
         description="Targeted question that the course designer have developed to assess the learner's understanding of the learning outcomes.")
@@ -38,6 +39,20 @@ class SuggestCheckQuestion:
         }
 
     def _get_check_quesiton(self):
+        parser = JsonOutputParser(pydantic_object=CheckQuestion)
+        chain = self._build_chain()
+
+        with callbacks.collect_runs() as cb:
+            llm_response = chain.invoke({
+                "course": self.course, "module": self.module, "tasks": "* " + ("\n* ".join(self.tasks)),
+                "format_instructions": parser.get_format_instructions(),
+                "learning_outcome": self.learning_outcome
+            })
+            self.run_id = cb.traced_runs[0].id
+
+        return llm_response
+
+    def _build_chain(self):
         parser = JsonOutputParser(pydantic_object=CheckQuestion)
         prompt = hub.pull("course_learn_suggest_check_question")
         llm = ChatOpenAI(model=os.environ['OPENAI_MODEL'], temperature=0.2)
@@ -51,12 +66,4 @@ class SuggestCheckQuestion:
             }
         })
 
-        with callbacks.collect_runs() as cb:
-            llm_response = chain.invoke({
-                "course": self.course, "module": self.module, "tasks": "* " + ("\n* ".join(self.tasks)),
-                "format_instructions": parser.get_format_instructions(),
-                "learning_outcome": self.learning_outcome
-            })
-            self.run_id = cb.traced_runs[0].id
-
-        return llm_response
+        return chain

app/workflows/courses/suggest_expectations.py

@@ -45,17 +45,7 @@ class SuggestExpectations:
 
     def _get_suggestions(self):
         parser = JsonOutputParser(pydantic_object=Expectations)
-        prompt = hub.pull("course_learn_suggest_expectations_from_learner")
-        llm = ChatOpenAI(model=os.environ['OPENAI_MODEL'], temperature=0.2)
-        chain = (prompt | llm | parser).with_config({
-            "tags": ["course_learn", "suggest_expectations"], "run_name": "Suggest Module Expectations",
-            "metadata": {
-                "versoin": "v1.0.0",
-                "growth_activity": "course_learn",
-                "env": os.environ["ENV"],
-                "model": os.environ["OPENAI_MODEL"],
-            }
-        })
+        chain = self._build_chain()
 
         # Existing Expectations
         existing_expectations = []
@@ -81,6 +71,22 @@ class SuggestExpectations:
 
         return llm_response
 
+    def _build_chain(self):
+        parser = JsonOutputParser(pydantic_object=Expectations)
+        prompt = hub.pull("course_learn_suggest_expectations_from_learner")
+        llm = ChatOpenAI(model=os.environ['OPENAI_MODEL'], temperature=0.2)
+        chain = (prompt | llm | parser).with_config({
+            "tags": ["course_learn", "suggest_expectations"], "run_name": "Suggest Module Expectations",
+            "metadata": {
+                "versoin": "v1.0.0",
+                "growth_activity": "course_learn",
+                "env": os.environ["ENV"],
+                "model": os.environ["OPENAI_MODEL"],
+            }
+        })
+
+        return chain
+
 
 # Example usage
 # suggester = SuggestExpectations()