Merge pull request #28 from beautiful-code/til_v2

endpoints.py

@@ -4,15 +4,20 @@ load_dotenv()
 from .workflows.courses.expectation_revision import ExpectationRevision, Inputs as ExpectationRevisionInputs, Response as ExpectationRevisionResponse
 from .workflows.courses.suggest_check_question import SuggestCheckQuestion, Inputs as SuggestCheckQuestionInputs, Response as SuggestCheckQuestionResponse
 from .workflows.courses.suggest_expectations import SuggestExpectations, Inputs as SuggestExpectationsInputs, Expectation, Response as SuggestExpectationsResponse
-from .workflows.til import TilCrew, TilFeedbackResponse
+from .workflows.til.analyse_til import TilCrew, TilFeedbackResponse
+from .workflows.til.analyse_til_v2 import AnalyseTilV2, TilV2FeedbackResponse
 from .workflows.utils.feedback import Feedback, post_feedback
 from fastapi import FastAPI, Query
 from fastapi.middleware.cors import CORSMiddleware
-from pydantic import UUID4, BaseModel
-from typing import List, Optional
+from pydantic import UUID4
+from tempenv import TemporaryEnvironment
+from typing import List
 import uvicorn
 
 
+LANGSMITH_STAGING_PROJECT = "customer_agent"
+LANGSMITH_PROD_PROJECT = "growthy-agents"
+
 description = """
  API helps you do awesome stuff. 🚀
 
@@ -64,8 +69,46 @@ async def capture_feedback(run_id: UUID4, feedback: Feedback) -> str:
     return "ok"
 
 
-@app.post("/course_learn/suggest_expectations", tags=["course_learn"])
-async def course_learn_suggest_expectations(inputs: SuggestExpectationsInputs) -> SuggestExpectationsResponse:
+@app.post("/staging/til_feedback", tags=["til_feedback", "staging"])
+async def staging_til_feedback_kickoff(content: List[str]) -> TilFeedbackResponse:
+    with TemporaryEnvironment({"LANGCHAIN_PROJECT": LANGSMITH_STAGING_PROJECT, "OPENAI_MODEL": "gpt-4o-mini"}):
+        separator = "\n* "
+        content[0] = "* " + content[0]
+        inputs = {"content": separator.join(content)}
+        result = TilCrew().kickoff(inputs)
+        return result
+
+
+@app.post("/staging/til_feedback/{run_id}/feedback", tags=["til_feedback", "staging"])
+async def staging_capture_feedback(run_id: UUID4, feedback: Feedback) -> str:
+    with TemporaryEnvironment({"LANGCHAIN_PROJECT": LANGSMITH_STAGING_PROJECT, "OPENAI_MODEL": "gpt-4o-mini"}):
+        print("Metric Type: ", feedback.metric_type)
+        print("Feedback On: ", feedback.feedback_on)
+        post_feedback(run_id=run_id, feedback=feedback)
+        return "ok"
+
+
+def til_v2_logic(content) -> TilV2FeedbackResponse:
+    separator = "\n* "
+    content[0] = "* " + content[0]
+    inputs = {"content": separator.join(content)}
+    result = AnalyseTilV2().kickoff(inputs)
+    return result
+
+
+@app.post("/v2/til_feedback", tags=["til_feedback"])
+async def til_v2_feedback_kickoff(content: List[str]) -> TilV2FeedbackResponse:
+    with TemporaryEnvironment({"LANGCHAIN_PROJECT": LANGSMITH_PROD_PROJECT, "OPENAI_MODEL": "gpt-4o"}):
+        return til_v2_logic(content)
+
+
+@app.post("/staging/v2/til_feedback", tags=["til_feedback", "staging"])
+async def staging_til_v2_feedback_kickoff(content: List[str]) -> TilV2FeedbackResponse:
+    with TemporaryEnvironment({"LANGCHAIN_PROJECT": LANGSMITH_STAGING_PROJECT, "OPENAI_MODEL": "gpt-4o-mini"}):
+        return til_v2_logic(content)
+
+
+def course_learn_suggest_expectations_logic(inputs) -> SuggestExpectationsResponse:
     print("Inputs: ", inputs)
     result = SuggestExpectations().kickoff(inputs={
         "course": inputs.course,
@@ -76,16 +119,38 @@ async def course_learn_suggest_expectations(inputs: SuggestExpectationsInputs) -
     return result
 
 
-@app.post("/course_learn/suggest_expectations/{run_id}/feedback", tags=["course_learn"])
-async def capture_suggest_expectations_feedback(run_id: UUID4, feedback: Feedback) -> str:
+def course_learn_suggest_expectations_feedback_logic(run_id: UUID4, feedback: Feedback) -> str:
     print("Helful Score: ", feedback.metric_type)
     print("Feedback On: ", feedback.feedback_on)
     post_feedback(run_id=run_id, feedback=feedback)
     return "ok"
 
 
-@app.post("/course_learn/expectation_revision", tags=["course_learn"])
-async def course_learn_expectation_revision(inputs: ExpectationRevisionInputs) -> ExpectationRevisionResponse:
+@app.post("/course_learn/suggest_expectations", tags=["course_learn"])
+async def course_learn_suggest_expectations(inputs: SuggestExpectationsInputs) -> SuggestExpectationsResponse:
+    with TemporaryEnvironment({"LANGCHAIN_PROJECT": LANGSMITH_PROD_PROJECT, "OPENAI_MODEL": "gpt-4o"}):
+        return course_learn_suggest_expectations_logic(inputs)
+
+
+@app.post("/staging/course_learn/suggest_expectations", tags=["course_learn", "staging"])
+async def staging_course_learn_suggest_expectations(inputs: SuggestExpectationsInputs) -> SuggestExpectationsResponse:
+    with TemporaryEnvironment({"LANGCHAIN_PROJECT": LANGSMITH_STAGING_PROJECT, "OPENAI_MODEL": "gpt-4o-mini"}):
+        return course_learn_suggest_expectations_logic(inputs)
+
+
+@app.post("/course_learn/suggest_expectations/{run_id}/feedback", tags=["course_learn"])
+async def capture_suggest_expectations_feedback(run_id: UUID4, feedback: Feedback) -> str:
+    with TemporaryEnvironment({"LANGCHAIN_PROJECT": LANGSMITH_PROD_PROJECT, "OPENAI_MODEL": "gpt-4o"}):
+        return course_learn_suggest_expectations_feedback_logic(run_id, feedback)
+
+
+@app.post("/staging/course_learn/suggest_expectations/{run_id}/feedback", tags=["course_learn", "staging"])
+async def staging_capture_suggest_expectations_feedback(run_id: UUID4, feedback: Feedback) -> str:
+    with TemporaryEnvironment({"LANGCHAIN_PROJECT": LANGSMITH_STAGING_PROJECT, "OPENAI_MODEL": "gpt-4o-mini"}):
+        return course_learn_suggest_expectations_feedback_logic(run_id, feedback)
+
+
+def course_learn_expectation_revision_logic(inputs: ExpectationRevisionInputs) -> ExpectationRevisionResponse:
     print("Inputs: ", inputs)
     result = ExpectationRevision().kickoff(inputs={
         "expectation": inputs.expectation,
@@ -95,16 +160,38 @@ async def course_learn_expectation_revision(inputs: ExpectationRevisionInputs) -
     return result
 
 
-@app.post("/course_learn/expectation_revision/{run_id}/feedback", tags=["course_learn"])
-async def capture_expectation_revision_feedback(run_id: UUID4, feedback: Feedback) -> str:
+@app.post("/course_learn/expectation_revision", tags=["course_learn"])
+async def course_learn_expectation_revision(inputs: ExpectationRevisionInputs) -> ExpectationRevisionResponse:
+    with TemporaryEnvironment({"LANGCHAIN_PROJECT": LANGSMITH_PROD_PROJECT, "OPENAI_MODEL": "gpt-4o"}):
+        return course_learn_expectation_revision_logic(inputs)
+
+
+@app.post("/staging/course_learn/expectation_revision", tags=["course_learn", "staging"])
+async def staging_course_learn_expectation_revision(inputs: ExpectationRevisionInputs) -> ExpectationRevisionResponse:
+    with TemporaryEnvironment({"LANGCHAIN_PROJECT": LANGSMITH_STAGING_PROJECT, "OPENAI_MODEL": "gpt-4o-mini"}):
+        return course_learn_expectation_revision_logic(inputs)
+
+
+def capture_expectation_revision_feedback_logic(run_id: UUID4, feedback: Feedback) -> str:
     print("Helful Score: ", feedback.metric_type)
     print("Feedback On: ", feedback.feedback_on)
     post_feedback(run_id=run_id, feedback=feedback)
     return "ok"
 
 
-@app.post("/course_learn/suggest_check_question", tags=["course_learn"])
-async def course_learn_suggest_check_question(inputs: SuggestCheckQuestionInputs) -> SuggestCheckQuestionResponse:
+@app.post("/course_learn/expectation_revision/{run_id}/feedback", tags=["course_learn"])
+async def capture_expectation_revision_feedback(run_id: UUID4, feedback: Feedback) -> str:
+    with TemporaryEnvironment({"LANGCHAIN_PROJECT": LANGSMITH_PROD_PROJECT, "OPENAI_MODEL": "gpt-4o"}):
+        return capture_expectation_revision_feedback_logic(run_id, feedback)
+
+
+@app.post("/staging/course_learn/expectation_revision/{run_id}/feedback", tags=["course_learn", "staging"])
+async def staging_capture_expectation_revision_feedback(run_id: UUID4, feedback: Feedback) -> str:
+    with TemporaryEnvironment({"LANGCHAIN_PROJECT": LANGSMITH_STAGING_PROJECT, "OPENAI_MODEL": "gpt-4o-mini"}):
+        return capture_expectation_revision_feedback_logic(run_id, feedback)
+
+
+def course_learn_suggest_check_question_logic(inputs: SuggestCheckQuestionInputs) -> SuggestCheckQuestionResponse:
     print("Inputs: ", inputs)
     result = SuggestCheckQuestion().kickoff(inputs={
         "course": inputs.course,
@@ -115,14 +202,37 @@ async def course_learn_suggest_check_question(inputs: SuggestCheckQuestionInputs
     return result
 
 
-@app.post("/course_learn/suggest_check_question/{run_id}/feedback", tags=["course_learn"])
-async def course_learn_suggest_check_question_feedback(run_id: UUID4, feedback: Feedback) -> str:
+@app.post("/course_learn/suggest_check_question", tags=["course_learn"])
+async def course_learn_suggest_check_question(inputs: SuggestCheckQuestionInputs) -> SuggestCheckQuestionResponse:
+    with TemporaryEnvironment({"LANGCHAIN_PROJECT": LANGSMITH_PROD_PROJECT, "OPENAI_MODEL": "gpt-4o"}):
+        return course_learn_suggest_check_question_logic(inputs)
+
+
+@app.post("/staging/course_learn/suggest_check_question", tags=["course_learn", "staging"])
+async def staging_course_learn_suggest_check_question(inputs: SuggestCheckQuestionInputs) -> SuggestCheckQuestionResponse:
+    with TemporaryEnvironment({"LANGCHAIN_PROJECT": LANGSMITH_STAGING_PROJECT, "OPENAI_MODEL": "gpt-4o-mini"}):
+        return course_learn_suggest_check_question_logic(inputs)
+
+
+def course_learn_suggest_check_question_feedback_logic(run_id: UUID4, feedback: Feedback) -> str:
     print("Helful Score: ", feedback.metric_type)
     print("Feedback On: ", feedback.feedback_on)
     post_feedback(run_id=run_id, feedback=feedback)
     return "ok"
 
 
+@app.post("/course_learn/suggest_check_question/{run_id}/feedback", tags=["course_learn"])
+async def course_learn_suggest_check_question_feedback(run_id: UUID4, feedback: Feedback) -> str:
+    with TemporaryEnvironment({"LANGCHAIN_PROJECT": LANGSMITH_PROD_PROJECT, "OPENAI_MODEL": "gpt-4o"}):
+        return course_learn_suggest_check_question_feedback_logic(run_id, feedback)
+
+
+@app.post("/staging/course_learn/suggest_check_question/{run_id}/feedback", tags=["course_learn", "staging"])
+async def staging_course_learn_suggest_check_question_feedback(run_id: UUID4, feedback: Feedback) -> str:
+    with TemporaryEnvironment({"LANGCHAIN_PROJECT": LANGSMITH_STAGING_PROJECT, "OPENAI_MODEL": "gpt-4o-mini"}):
+        return course_learn_suggest_check_question_feedback_logic(run_id, feedback)
+
+
 @app.get("/healthcheck")
 async def read_root():
     return {"status": "ok"}

requirements.txt

@@ -18,6 +18,8 @@ python-dotenv
 semanticscholar
 streamlit
 streamlit-extras
+streamlit_router
 tavily-python
+tempenv
 unstructured
 uvicorn

ui/til_feedback.py

@@ -1,8 +1,7 @@
 import streamlit as st
 from dotenv import load_dotenv
-from workflows.til import TilCrew
+from workflows.til.analyse_til import TilCrew
 from streamlit_extras.capture import stdout
-from langsmith import Client
 from workflows.utils.feedback import Feedback
 
 load_dotenv()
@@ -21,7 +20,7 @@ def feedback_main():
         [data-testid="stToolbar"]{
             right: 2rem;
         }
-       
+
 
         </style>
         '''
@@ -86,7 +85,7 @@ def feedback_main():
                     st.markdown(f"**Reason:** {result['reason']}")
                     if result.get('suggestion') is not None:
                         st.markdown(f"**Suggestion:** {result['suggestion']}")
-         
+
                 feedback_key = result['til'].replace(' ', '_')
                 feedback_given_key = f"{feedback_key}_feedback_given"
 
@@ -111,14 +110,14 @@ def give_feedback(feedback_key, is_helpful):
         feedback_data = Feedback(
             metric_type=metric_type,
             metric_score=metric_score,
-            feedback_on=feedback_key.replace('_', ' ').title() 
+            feedback_on=feedback_key.replace('_', ' ').title()
         )
         try:
             TilCrew.post_feedback(run_id, feedback_data)
             st.success("Feedback submitted successfully!")
         except Exception as e:
             st.error(f"Failed to submit feedback: {e}")
-        
+
         st.session_state[f"{feedback_key}_feedback_given"] = True
 
 def clear_feedback_state(results):
@@ -128,4 +127,4 @@ def clear_feedback_state(results):
 
 
 if __name__ == "__main__":
-    feedback_main()
+    feedback_main()

workflows/{til.py → til/analyse_til.py}

@@ -5,10 +5,8 @@ from langchain_core.messages import SystemMessage
 from langchain_core.output_parsers import JsonOutputParser
 from langchain_core.prompts import ChatPromptTemplate, HumanMessagePromptTemplate, PromptTemplate
 from langchain_openai import ChatOpenAI
-from langsmith import Client
 from pydantic import BaseModel, Field, UUID4
 from typing import List, Optional
-from .utils.feedback import Feedback
 import os
 import pprint
 

workflows/til/analyse_til_v2.py

@@ -0,0 +1,129 @@
+from langchain import callbacks
+from langchain_core.messages import SystemMessage
+from langchain_core.output_parsers import JsonOutputParser
+from langchain_core.prompts import ChatPromptTemplate, HumanMessagePromptTemplate
+from langchain_openai import ChatOpenAI
+from pydantic import BaseModel, Field, UUID4
+from typing import List, Optional
+import os
+import pprint
+
+
+class AnalyseTilV2:
+
+    def kickoff(self, inputs={}):
+        print("Human Message:")
+        pprint.pp(inputs)
+        self.content = inputs["content"]
+        self._gather_feedback()
+        return self._final_call_on_feedback()
+
+    def _final_call_on_feedback(self):
+        final_results = []
+        for feedback in self.feedback_results:
+            print("Final analysis of:")
+            pprint.pp(feedback)
+            result = {
+                "til": feedback.get('til', ""),
+                "feedback": "not_ok",
+            }
+            if feedback["factuality_categorization"] != 'High':
+                result["feedback_criteria"] = "factuality_feedback"
+                result["reason"] = feedback["factuality_reason"]
+                final_results = final_results + [result]
+                continue
+
+            if feedback["insightful_categorization"] != 'High':
+                result["feedback_criteria"] = "insightful_feedback"
+                result["reason"] = feedback["insightful_reason"]
+                final_results = final_results + [result]
+                continue
+
+            result["feedback"] = "ok"
+            final_results = final_results + [result]
+
+        response = {"feedback": final_results, "run_id": self.run_id}
+        print("Final Results:")
+        pprint.pp(response)
+        return response
+
+    def _gather_feedback(self):
+        feedback_chain = self._build_feedback_chain()
+        pprint.pp("Analysing the TIL.....")
+        with callbacks.collect_runs() as cb:
+            self.feedback_results = feedback_chain.invoke(
+                {"til_content": self.content})['tils']
+            self.run_id = cb.traced_runs[0].id
+            print("Run ID: ", self.run_id)
+
+        print("Feedback: ")
+        pprint.pp(self.feedback_results)
+
+    def _build_feedback_chain(self):
+        feedback_parser = JsonOutputParser(pydantic_object=TilV2FeedbackResults)
+        feedback_prompt = ChatPromptTemplate.from_messages([
+            SystemMessage(
+                "You are a 'Personal TIL Reviewer' who works in a Product Engineering Services company. "
+                "You are an expert in writing TILs which are Insightful, Factually correct, Easy to read and grammatically correct."
+                "Your goal is to review user's TILs and categorize their correctness as High, Medium, or Low based on the following metrics:"
+                "1. Is the TIL insightful?"
+                "2. Is the TIL factually correct and accurate?"
+
+                "The criteria to use for assessing if they are insightful or not are:\n"
+                "* They TIL shouldn't just be a outright statement, it should contain even the reason on why the statement is true."
+                "* It should showcase the understanding of the user on the subject.\n\n"
+
+                "The criteria to use for assessing if they are factual or not are:\n"
+                "* They are related to facts."
+                "* You are able to find a source which agrees to the fact from reputable websites.\n\n"
+
+                "Give reason for your assessment in one or two sentences for each metric. "
+                "Evaluate each TIL in the context of all the user's TILs."
+                f"Formatting Instructions: {feedback_parser.get_format_instructions()}"
+            ),
+            HumanMessagePromptTemplate.from_template("{til_content}")
+        ])
+        print("Prompt: ")
+        pprint.pp(feedback_prompt, width=80)
+        llm = ChatOpenAI(model=os.environ["OPENAI_MODEL"], temperature=0.2)
+        analysis_chain = (feedback_prompt | llm | feedback_parser).with_config({
+            "tags": ["til"], "run_name": "Analysing TIL v2",
+            "metadata": {
+                "versoin": "v2.0.0",
+                "growth_activity": "til",
+                "env": os.environ["ENV"],
+                "model": os.environ["OPENAI_MODEL"]
+            }
+        })
+
+        return analysis_chain
+
+
+class TilV2FeedbackResult(BaseModel):
+    til: str = Field(
+        description="TIL as exactly captured by the user without any modifications.")
+    insightful_categorization: str = Field(
+        description="TIL categorization as High/Medium/Low based on correctness on the insightful metric.")
+    insightful_reason: str = Field(
+        description="Reason for your assessment in one or two sentences on insightful metric for the user.")
+    factuality_categorization: str = Field(
+        description="TIL categorization as High/Medium/Low based on correctness on the factuality metric.")
+    factuality_reason: str = Field(
+        description="Reason for your assessment in one or two sentences on factuality metric for the user.")
+
+
+class TilV2FeedbackResults(BaseModel):
+    tils: List[TilV2FeedbackResult]
+
+
+class TilV2FinalFeedback(BaseModel):
+    til: str
+    feedback: str
+    feedback_criteria: Optional[str] = None
+    reason: Optional[str] = None
+    suggestion: Optional[str] = None
+
+
+class TilV2FeedbackResponse(BaseModel):
+    run_id: UUID4
+    feedback: List[TilV2FinalFeedback]