initial TIL v2

endpoints.py

@@ -1,17 +1,21 @@
 from dotenv import load_dotenv
 load_dotenv()
 
+from fastapi import FastAPI, Query
+from fastapi.middleware.cors import CORSMiddleware
+from pydantic import UUID4
+from typing import List
+from tempenv import TemporaryEnvironment
 from .workflows.courses.expectation_revision import ExpectationRevision, Inputs as ExpectationRevisionInputs, Response as ExpectationRevisionResponse
 from .workflows.courses.suggest_check_question import SuggestCheckQuestion, Inputs as SuggestCheckQuestionInputs, Response as SuggestCheckQuestionResponse
 from .workflows.courses.suggest_expectations import SuggestExpectations, Inputs as SuggestExpectationsInputs, Expectation, Response as SuggestExpectationsResponse
 from .workflows.til import TilCrew, TilFeedbackResponse
+from .workflows.til_v2 import TilV2, TilV2FeedbackResponse
 from .workflows.utils.feedback import Feedback, post_feedback
-from fastapi import FastAPI, Query
-from fastapi.middleware.cors import CORSMiddleware
-from pydantic import UUID4, BaseModel
-from typing import List, Optional
 import uvicorn
 
+LANGSMITH_STAGING_PROJECT = "customer_agent"
+LANGSMITH_PROD_PROJECT = "growthy-agents"
 
 description = """
  API helps you do awesome stuff. 🚀
@@ -56,6 +60,26 @@ async def til_feedback_kickoff(content: List[str]) -> TilFeedbackResponse:
     return result
 
 
+def til_v2_logic(llm_model, langsmith_project, content) -> TilV2FeedbackResponse:
+    separator = "\n* "
+    content[0] = "* " + content[0]
+    inputs = {"content": separator.join(content)}
+    result = TilV2(llm_model, langsmith_project).kickoff(inputs)
+    return result
+
+
+@app.post("/v2/til_feedback", tags=["til_feedback"])
+async def til_v2_feedback_kickoff(content: List[str]) -> TilV2FeedbackResponse:
+    with TemporaryEnvironment({"LANGCHAIN_PROJECT": LANGSMITH_PROD_PROJECT}):
+        return til_v2_logic("gpt-4o", "growthy-agents", content)
+
+
+@app.post("/staging/v2/til_feedback", tags=["til_feedback", "staging"])
+async def til_v2_feedback_kickoff(content: List[str]) -> TilV2FeedbackResponse:
+    with TemporaryEnvironment({"LANGCHAIN_PROJECT": LANGSMITH_STAGING_PROJECT}):
+        return til_v2_logic("gpt-4o-mini", "customer_agent", content)
+
+
 @app.post("/til_feedback/{run_id}/feedback", tags=["til_feedback"])
 async def capture_feedback(run_id: UUID4, feedback: Feedback) -> str:
     print("Metric Type: ", feedback.metric_type)

requirements.txt

@@ -19,5 +19,6 @@ semanticscholar
 streamlit
 streamlit-extras
 tavily-python
+tempenv
 unstructured
 uvicorn

workflows/til_v2.py

@@ -0,0 +1,133 @@
+from langchain import callbacks
+from langchain_core.messages import SystemMessage
+from langchain_core.output_parsers import JsonOutputParser
+from langchain_core.prompts import ChatPromptTemplate, HumanMessagePromptTemplate
+from langchain_openai import ChatOpenAI
+from pydantic import BaseModel, Field, UUID4
+from typing import List, Optional
+import os
+import pprint
+
+
+class TilV2:
+
+    def __init__(self, llm_model, langsmith_project):
+        self.llm_model = llm_model
+        self.langsmith_project = langsmith_project
+
+    def kickoff(self, inputs={}):
+        print("Human Message:")
+        pprint.pp(inputs)
+        self.content = inputs["content"]
+        self._gather_feedback()
+        return self._final_call_on_feedback()
+
+    def _final_call_on_feedback(self):
+        final_results = []
+        for feedback in self.feedback_results:
+            print("Final analysis of:")
+            pprint.pp(feedback)
+            result = {
+                "til": feedback.get('til', ""),
+                "feedback": "not_ok",
+            }
+            if feedback["factuality_categorization"] != 'High':
+                result["feedback_criteria"] = "factuality_feedback"
+                result["reason"] = feedback["factuality_reason"]
+                final_results = final_results + [result]
+                continue
+
+            if feedback["insightful_categorization"] != 'High':
+                result["feedback_criteria"] = "insightful_feedback"
+                result["reason"] = feedback["insightful_reason"]
+                final_results = final_results + [result]
+                continue
+
+            result["feedback"] = "ok"
+            final_results = final_results + [result]
+
+        response = {"feedback": final_results, "run_id": self.run_id}
+        print("Final Results:")
+        pprint.pp(response)
+        return response
+
+    def _gather_feedback(self):
+        feedback_chain = self._build_feedback_chain()
+        pprint.pp("Analysing the TIL.....")
+        with callbacks.collect_runs() as cb:
+            self.feedback_results = feedback_chain.invoke(
+                {"til_content": self.content})['tils']
+            self.run_id = cb.traced_runs[0].id
+            print("Run ID: ", self.run_id)
+
+        print("Feedback: ")
+        pprint.pp(self.feedback_results)
+
+    def _build_feedback_chain(self):
+        feedback_parser = JsonOutputParser(pydantic_object=TilV2FeedbackResults)
+        feedback_prompt = ChatPromptTemplate.from_messages([
+            SystemMessage(
+                "You are a 'Personal TIL Reviewer' who works in a Product Engineering Services company. "
+                "You are an expert in writing TILs which are Insightful, Factually correct, Easy to read and grammatically correct."
+                "Your goal is to review user's TILs and categorize their correctness as High, Medium, or Low based on the following metrics:"
+                "1. Is the TIL insightful?"
+                "2. Is the TIL factually correct and accurate?"
+
+                "The criteria to use for assessing if they are insightful or not are:\n"
+                "* They TIL shouldn't just be a outright statement, it should contain even the reason on why the statement is true."
+                "* It should showcase the understanding of the user on the subject.\n\n"
+
+                "The criteria to use for assessing if they are factual or not are:\n"
+                "* They are related to facts."
+                "* You are able to find a source which agrees to the fact from reputable websites.\n\n"
+
+                "Give reason for your assessment in one or two sentences for each metric. "
+                "Evaluate each TIL in the context of all the user's TILs."
+                f"Formatting Instructions: {feedback_parser.get_format_instructions()}"
+            ),
+            HumanMessagePromptTemplate.from_template("{til_content}")
+        ])
+        print("Prompt: ")
+        pprint.pp(feedback_prompt, width=80)
+        llm = ChatOpenAI(model=self.llm_model, temperature=0.2)
+        analysis_chain = (feedback_prompt | llm | feedback_parser).with_config({
+            "tags": ["til"], "run_name": "Analysing TIL v2",
+            "metadata": {
+                "versoin": "v2.0.0",
+                "growth_activity": "til",
+                "env": os.environ["ENV"],
+                "model": self.llm_model,
+            }
+        })
+
+        return analysis_chain
+
+
+class TilV2FeedbackResult(BaseModel):
+    til: str = Field(
+        description="TIL as exactly captured by the user without any modifications.")
+    insightful_categorization: str = Field(
+        description="TIL categorization as High/Medium/Low based on correctness on the insightful metric.")
+    insightful_reason: str = Field(
+        description="Reason for your assessment in one or two sentences on insightful metric for the user.")
+    factuality_categorization: str = Field(
+        description="TIL categorization as High/Medium/Low based on correctness on the factuality metric.")
+    factuality_reason: str = Field(
+        description="Reason for your assessment in one or two sentences on factuality metric for the user.")
+
+
+class TilV2FeedbackResults(BaseModel):
+    tils: List[TilV2FeedbackResult]
+
+
+class TilV2FinalFeedback(BaseModel):
+    til: str
+    feedback: str
+    feedback_criteria: Optional[str] = None
+    reason: Optional[str] = None
+    suggestion: Optional[str] = None
+
+
+class TilV2FeedbackResponse(BaseModel):
+    run_id: UUID4
+    feedback: List[TilV2FinalFeedback]