workflows(course_learn): Return run_id

endpoints.py

@@ -1,15 +1,15 @@
+import uvicorn
+from typing import List, Optional
+from pydantic import UUID4, BaseModel
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi import FastAPI, Query
+from .workflows.utils.feedback import Feedback, post_feedback
+from .workflows.til import TilCrew, TilFeedbackResponse
+from .workflows.courses.suggest_expectations import SuggestExpectations, Inputs as SuggestExpectationsInputs, Expectation, Response as SuggestExpectationsResponse
+from .workflows.courses.expectation_revision import ExpectationRevision, Inputs as ExpectationRevisionInputs, Response as ExpectationRevisionResponse
 from dotenv import load_dotenv
 load_dotenv()
 
-from .workflows.courses.expectation_revision import ExpectationRevision, Inputs as ExpectationRevisionInputs
-from .workflows.courses.suggest_expectations import SuggestExpectations, Inputs as SuggestExpectationsInputs, Expectations, Expectation
-from .workflows.til import TilCrew, TilFeedbackResponse
-from .workflows.utils.feedback import Feedback
-from fastapi import FastAPI, Query
-from fastapi.middleware.cors import CORSMiddleware
-from pydantic import UUID4, BaseModel
-from typing import List, Optional
-import uvicorn
 
 description = """
  API helps you do awesome stuff. 🚀
@@ -44,6 +44,7 @@ app.add_middleware(
     allow_headers=["*"],
 )
 
+
 @app.post("/til_feedback", tags=["til_feedback"])
 async def til_feedback_kickoff(content: List[str]) -> TilFeedbackResponse:
     separator = "\n* "
@@ -52,15 +53,17 @@ async def til_feedback_kickoff(content: List[str]) -> TilFeedbackResponse:
     result = TilCrew().kickoff(inputs)
     return result
 
+
 @app.post("/til_feedback/{run_id}/feedback", tags=["til_feedback"])
 async def capture_feedback(run_id: UUID4, feedback: Feedback) -> str:
-    print("Helful Score: ", feedback.helpful_score)
+    print("Metric Type: ", feedback.metric_type)
     print("Feedback On: ", feedback.feedback_on)
-    TilCrew.post_feedback(run_id=run_id, feedback=feedback)
+    post_feedback(run_id=run_id, feedback=feedback)
     return "ok"
 
+
 @app.post("/course_learn/suggest_expectations", tags=["course_learn"])
-async def course_learn_suggest_expectations(inputs: SuggestExpectationsInputs) -> Expectations:
+async def course_learn_suggest_expectations(inputs: SuggestExpectationsInputs) -> SuggestExpectationsResponse:
     print("Inputs: ", inputs)
     result = SuggestExpectations().kickoff(inputs={
         "course": inputs.course,
@@ -70,8 +73,17 @@ async def course_learn_suggest_expectations(inputs: SuggestExpectationsInputs) -
     })
     return result
 
+
+@app.post("/course_learn/suggest_expectations/{run_id}/feedback", tags=["course_learn"])
+async def capture_suggest_expectations_feedback(run_id: UUID4, feedback: Feedback) -> str:
+    print("Helful Score: ", feedback.metric_type)
+    print("Feedback On: ", feedback.feedback_on)
+    post_feedback(run_id=run_id, feedback=feedback)
+    return "ok"
+
+
 @app.post("/course_learn/expectation_revision", tags=["course_learn"])
-async def course_learn_expectation_revision(inputs: ExpectationRevisionInputs) -> Expectation:
+async def course_learn_expectation_revision(inputs: ExpectationRevisionInputs) -> ExpectationRevisionResponse:
     print("Inputs: ", inputs)
     result = ExpectationRevision().kickoff(inputs={
         "expectation": inputs.expectation,
@@ -80,12 +92,19 @@ async def course_learn_expectation_revision(inputs: ExpectationRevisionInputs) -
     })
     return result
 
+
+@app.post("/course_learn/expectation_revision/{run_id}/feedback", tags=["course_learn"])
+async def capture_expectation_revision_feedback(run_id: UUID4, feedback: Feedback) -> str:
+    print("Helful Score: ", feedback.metric_type)
+    print("Feedback On: ", feedback.feedback_on)
+    post_feedback(run_id=run_id, feedback=feedback)
+    return "ok"
+
+
 @app.get("/healthcheck")
 async def read_root():
-     return {"status": "ok"}
+    return {"status": "ok"}
 
 
 if __name__ == "__main__":
     uvicorn.run(app, host="127.0.0.1", port=8080)
-
-

workflows/courses/expectation_revision.py

@@ -1,9 +1,8 @@
-from langchain import hub
+from langchain import hub, callbacks
 from langchain_core.output_parsers import JsonOutputParser
 from langchain_openai import ChatOpenAI
-from pydantic import BaseModel
+from pydantic import BaseModel, UUID4
 from .suggest_expectations import Expectation
-from typing import List, Optional
 import os
 
 class Inputs(BaseModel):
@@ -11,12 +10,22 @@ class Inputs(BaseModel):
     check_question: str
     request: str
 
+class Response(BaseModel):
+    run_id: UUID4
+    expectation: str
+    check_question: str
+
 class ExpectationRevision:
     def kickoff(self, inputs={}):
         self.learning_outcome = inputs["expectation"]
         self.check_question = inputs["check_question"]
         self.request = inputs["request"]
-        return self._get_suggestion()
+        llm_response = self._get_suggestion()
+        return {
+            "run_id": self.run_id,
+            "expectation": llm_response["expectation"],
+            "check_question": llm_response["check_question"]
+        }
 
     def _get_suggestion(self):
         parser = JsonOutputParser(pydantic_object=Expectation)
@@ -32,10 +41,12 @@ class ExpectationRevision:
             }
         })
 
-        response = chain.invoke({
-            "learning_outcome": self.learning_outcome, "check_question": self.check_question, "request": self.request,
-            "format_instructions": parser.get_format_instructions()
-          })
+        with callbacks.collect_runs() as cb:
+            response = chain.invoke({
+                "learning_outcome": self.learning_outcome, "check_question": self.check_question, "request": self.request,
+                "format_instructions": parser.get_format_instructions()
+              })
+            self.run_id = cb.traced_runs[0].id
 
         return response
 

workflows/courses/suggest_expectations.py

@@ -1,7 +1,7 @@
-from langchain import hub
+from langchain import hub, callbacks
 from langchain_core.output_parsers import JsonOutputParser
 from langchain_openai import ChatOpenAI
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, UUID4
 from typing import List
 from pydantic import BaseModel
 from typing import List, Optional
@@ -19,6 +19,11 @@ class Expectations(BaseModel):
     expectations: List[Expectation]
 
 
+class Response(BaseModel):
+    run_id: UUID4
+    expectations: List[Expectation]
+
+
 class Inputs(BaseModel):
     course: str
     module: str
@@ -32,7 +37,11 @@ class SuggestExpectations:
         self.module = inputs["module"]
         self.existing_expectations = inputs["existing_expectations"]
         self.tasks = inputs["tasks"]
-        return self._get_suggestions()
+        llm_response = self._get_suggestions()
+        return {
+          "run_id": self.run_id,
+          "expectations": llm_response["expectations"]
+        }
 
     def _get_suggestions(self):
         parser = JsonOutputParser(pydantic_object=Expectations)
@@ -62,13 +71,15 @@ class SuggestExpectations:
             existing_expectations_str += "\n".join(existing_expectations)
             existing_expectations_str += "\n```"
 
-        response = chain.invoke({
-            "course": self.course, "module": self.module, "tasks": "* " + ("\n* ".join(self.tasks)),
-            "format_instructions": parser.get_format_instructions(),
-            "existing_expectations": existing_expectations_str
-        })
+        with callbacks.collect_runs() as cb:
+            llm_response = chain.invoke({
+                "course": self.course, "module": self.module, "tasks": "* " + ("\n* ".join(self.tasks)),
+                "format_instructions": parser.get_format_instructions(),
+                "existing_expectations": existing_expectations_str
+            })
+            self.run_id = cb.traced_runs[0].id
 
-        return response
+        return llm_response
 
 
 # Example usage

workflows/til.py

@@ -1,5 +1,4 @@
-from langchain import callbacks
-from langchain import hub
+from langchain import callbacks, hub
 from langchain.agents import AgentExecutor, create_react_agent
 from langchain_community.tools.tavily_search import TavilyAnswer
 from langchain_core.messages import SystemMessage
@@ -13,6 +12,7 @@ from .utils.feedback import Feedback
 import os
 import pprint
 
+
 class TilCrew:
     def kickoff(self, inputs={}):
         print("Human Message:")
@@ -21,17 +21,6 @@ class TilCrew:
         self._gather_feedback()
         return self._final_call_on_feedback()
 
-    def post_feedback(run_id: UUID4, feedback: Feedback):
-        client = Client()
-        client.create_feedback(
-            str(run_id),
-            key=feedback.metric_type,
-            score=feedback.metric_score,
-            source_info={"til": feedback.feedback_on},
-            type="api",
-        )
-
-
     def _final_call_on_feedback(self):
         final_results = []
         for feedback in self.feedback_results:
@@ -70,7 +59,7 @@ class TilCrew:
             result["feedback"] = "ok"
             final_results = final_results + [result]
 
-        response = {"feedback": final_results, "run_id": self.run_id }
+        response = {"feedback": final_results, "run_id": self.run_id}
         print("Final Results:")
         pprint.pp(response)
         return response
@@ -79,22 +68,25 @@ class TilCrew:
         feedback_chain = self._build_feedback_chain()
         pprint.pp("Analysing the TIL.....")
         with callbacks.collect_runs() as cb:
-          self.feedback_results = feedback_chain.invoke({"til_content": self.content})['tils']
-          self.run_id = cb.traced_runs[0].id
-          print("Run ID: ", self.run_id)
+            self.feedback_results = feedback_chain.invoke(
+                {"til_content": self.content})['tils']
+            self.run_id = cb.traced_runs[0].id
+            print("Run ID: ", self.run_id)
 
         print("Feedback: ")
         pprint.pp(self.feedback_results)
 
     # Deprecated: Not using this as we are getting similar results by using or without using this
     def _gather_facts(self):
-        facts_prompt = PromptTemplate.from_template("What are the facts on the topics mentioned the following user's TILs: {content}")
+        facts_prompt = PromptTemplate.from_template(
+            "What are the facts on the topics mentioned the following user's TILs: {content}")
         tools = [TavilyAnswer()]
         llm = ChatOpenAI(model=os.environ['OPENAI_MODEL'], temperature=0.2)
         prompt = hub.pull("hwchase17/react")
         agent = create_react_agent(llm, tools, prompt)
         agent_executor = AgentExecutor(agent=agent, tools=tools, verbose=True)
-        self.facts = agent_executor.invoke({"input": facts_prompt.format(content=self.content)})['output']
+        self.facts = agent_executor.invoke(
+            {"input": facts_prompt.format(content=self.content)})['output']
         print("Gathered Facts: ")
         pprint.pp(self.facts)
 
@@ -129,7 +121,7 @@ class TilCrew:
         llm = ChatOpenAI(model=os.environ['OPENAI_MODEL'], temperature=0.2)
         analysis_chain = (feedback_prompt | llm | feedback_parser).with_config({
             "tags": ["til"], "run_name": "Analysing TIL",
-            "metadata" : {
+            "metadata": {
                 "versoin": "v1.0.0",
                 "growth_activity": "til",
                 "env": os.environ["ENV"],
@@ -141,19 +133,24 @@ class TilCrew:
 
 
 class TilFeedbackResult(BaseModel):
-    til: str = Field(description="TIL as exactly captured by the user without any modifications.")
+    til: str = Field(
+        description="TIL as exactly captured by the user without any modifications.")
     insightful_categorization: str = Field(
         description="TIL categorization as High/Medium/Low based on correctness on the insightful metric.")
-    insightful_reason: str = Field(description="Reason for your assessment in one or two sentences on insightful metric for the user.")
+    insightful_reason: str = Field(
+        description="Reason for your assessment in one or two sentences on insightful metric for the user.")
     factuality_categorization: str = Field(
         description="TIL categorization as High/Medium/Low based on correctness on the factuality metric.")
-    factuality_reason: str = Field(description="Reason for your assessment in one or two sentences on factuality metric for the user.")
+    factuality_reason: str = Field(
+        description="Reason for your assessment in one or two sentences on factuality metric for the user.")
     simplicity_categorization: str = Field(
         description="TIL categorization as High/Medium/Low based on correctness on the simplicity metric.")
-    simplicity_reason: str = Field(description="Reason for your assessment in one or two sentences on simplicity metric for the user.")
+    simplicity_reason: str = Field(
+        description="Reason for your assessment in one or two sentences on simplicity metric for the user.")
     grammatical_categorization: str = Field(
         description="TIL categorization as High/Medium/Low based on correctness on the grammatical metric.")
-    grammatical_reason: str = Field(description="Reason for your assessment in one or two sentences on grammatical metric for the user.")
+    grammatical_reason: str = Field(
+        description="Reason for your assessment in one or two sentences on grammatical metric for the user.")
     final_suggestion: str = Field(
         description="Rewrite the TIL if you were given the option to write it which should score High on all the metrics.")
 
@@ -161,6 +158,7 @@ class TilFeedbackResult(BaseModel):
 class TilFeedbackResults(BaseModel):
     tils: List[TilFeedbackResult]
 
+
 class TilFinalFeedback(BaseModel):
     til: str
     feedback: str
@@ -168,6 +166,7 @@ class TilFinalFeedback(BaseModel):
     reason: Optional[str] = None
     suggestion: Optional[str] = None
 
+
 class TilFeedbackResponse(BaseModel):
     run_id: UUID4
     feedback: List[TilFinalFeedback]

workflows/utils/feedback.py

@@ -1,7 +1,19 @@
-from pydantic import BaseModel
+from pydantic import BaseModel, UUID4
 from typing import List, Optional
+from langsmith import Client
 
 class Feedback(BaseModel):
     metric_type: Optional[str]
     metric_score: Optional[float]
     feedback_on: Optional[str]
+
+
+def post_feedback(run_id: UUID4, feedback: Feedback):
+    client = Client()
+    client.create_feedback(
+        str(run_id),
+        key=feedback.metric_type,
+        score=feedback.metric_score,
+        source_info={"content": feedback.feedback_on},
+        type="api",
+    )