workflows(tilv2): migrate tilv2 prompt to langsmith

ui/til_feedback.py

@@ -1,6 +1,7 @@
 import streamlit as st
 from dotenv import load_dotenv
 from workflows.til.analyse_til import TilCrew
+from workflows.til.analyse_til_v2 import AnalyseTilV2
 from streamlit_extras.capture import stdout
 from workflows.utils.feedback import Feedback
 
@@ -63,7 +64,7 @@ def feedback_main():
                 with st.container(height=500, border=False):
                     log_container = st.empty()
                     with stdout(log_container.code, terminator=""):
-                        feedback = TilCrew()
+                        feedback = AnalyseTilV2()
                         inputs = {"content": til_content}
                         results = feedback.kickoff(inputs=inputs)
                 status.update(
@@ -81,7 +82,6 @@ def feedback_main():
                 st.markdown(f"#### TIL: {result['til']}")
                 st.markdown(f"**Feedback:** {result['feedback']}")
                 if result['feedback'] == "not_ok":
-                    st.markdown(f"**Criteria:** {result['feedback_criteria']}")
                     st.markdown(f"**Reason:** {result['reason']}")
                     if result.get('suggestion') is not None:
                         st.markdown(f"**Suggestion:** {result['suggestion']}")

ui_main.py

@@ -1,6 +1,6 @@
 from dotenv import load_dotenv
 from streamlit_extras.stylable_container import stylable_container
-from ui.til_feedback import  feedback_main
+from ui.til_feedback import feedback_main
 from ui.course_learn_suggest_expectations import course_suggester_main
 import math
 import streamlit as st
@@ -42,12 +42,12 @@ def show_main_page():
         }
         [data-testid="column"] {
                 text-align: center;
-                                } 
-    
+                                }
+
         </style>
         '''
     st.markdown(page_bg_img, unsafe_allow_html=True)
-    
+
     css = load_css("ui/main.css")
 
     st.markdown(f"<style>{css}</style>", unsafe_allow_html=True)
@@ -77,7 +77,7 @@ def show_main_page():
                         with st.container(height=280,border=False):
                             with stylable_container(
                                 key="inside_container_with_border",
-                                
+
                                 css_styles="""
                                         {
                                             background-color: #e9eaec;
@@ -97,7 +97,7 @@ def show_main_page():
                                     st.markdown(
                                         f"""
                                         <div style="
-                                            font-size: 4rem; 
+                                            font-size: 4rem;
                                             margin-bottom: 10px;
                                         ">
                                             {card['image']}
@@ -125,14 +125,14 @@ def show_main_page():
                                     #     border-radius: 50%;
                                     #     border: 2px solid white;
 
-                                    #  } 
+                                    #  }
 
                                     [data-testid="column"] div[data-testid="column"]{
                                             margin: 10px;
-                                        }                            
+                                        }
                                     </style>
                                     """, unsafe_allow_html=True
-                                            )      
+                                            )
 
 
 def feedback_main_wrapper():
@@ -144,7 +144,7 @@ def course_suggester_main_wrapper():
     course_suggester_main()
 
 
-                   
+
 router = sr.StreamlitRouter()
 
 router.register(show_main_page, '/')

workflows/til/analyse_til_v2.py

@@ -1,4 +1,4 @@
-from langchain import callbacks
+from langchain import callbacks, hub
 from langchain_core.messages import SystemMessage
 from langchain_core.output_parsers import JsonOutputParser
 from langchain_core.prompts import ChatPromptTemplate, HumanMessagePromptTemplate
@@ -28,14 +28,12 @@ class AnalyseTilV2:
                 "feedback": "not_ok",
             }
             if feedback["factuality_categorization"] != 'High':
-                result["feedback_criteria"] = "factuality_feedback"
-                result["reason"] = feedback["factuality_reason"]
+                result["reason"] = feedback["assessment_reason"]
                 final_results = final_results + [result]
                 continue
 
             if feedback["insightful_categorization"] != 'High':
-                result["feedback_criteria"] = "insightful_feedback"
-                result["reason"] = feedback["insightful_reason"]
+                result["reason"] = feedback["assessment_reason"]
                 final_results = final_results + [result]
                 continue
 
@@ -49,10 +47,11 @@ class AnalyseTilV2:
 
     def _gather_feedback(self):
         feedback_chain = self._build_feedback_chain()
+        feedback_parser = JsonOutputParser(pydantic_object=TilV2FeedbackResults)
         pprint.pp("Analysing the TIL.....")
         with callbacks.collect_runs() as cb:
             self.feedback_results = feedback_chain.invoke(
-                {"til_content": self.content})['tils']
+                {"til_content": self.content, "format_instructions": feedback_parser.get_format_instructions()})['tils']
             self.run_id = cb.traced_runs[0].id
             print("Run ID: ", self.run_id)
 
@@ -61,28 +60,7 @@ class AnalyseTilV2:
 
     def _build_feedback_chain(self):
         feedback_parser = JsonOutputParser(pydantic_object=TilV2FeedbackResults)
-        feedback_prompt = ChatPromptTemplate.from_messages([
-            SystemMessage(
-                "You are a 'Personal TIL Reviewer' who works in a Product Engineering Services company. "
-                "You are an expert in writing TILs which are Insightful, Factually correct, Easy to read and grammatically correct."
-                "Your goal is to review user's TILs and categorize their correctness as High, Medium, or Low based on the following metrics:"
-                "1. Is the TIL insightful?"
-                "2. Is the TIL factually correct and accurate?"
-
-                "The criteria to use for assessing if they are insightful or not are:\n"
-                "* They TIL shouldn't just be a outright statement, it should contain even the reason on why the statement is true."
-                "* It should showcase the understanding of the user on the subject.\n\n"
-
-                "The criteria to use for assessing if they are factual or not are:\n"
-                "* They are related to facts."
-                "* You are able to find a source which agrees to the fact from reputable websites.\n\n"
-
-                "Give reason for your assessment in one or two sentences for each metric. "
-                "Evaluate each TIL in the context of all the user's TILs."
-                f"Formatting Instructions: {feedback_parser.get_format_instructions()}"
-            ),
-            HumanMessagePromptTemplate.from_template("{til_content}")
-        ])
+        feedback_prompt = hub.pull("til_analysis")
         print("Prompt: ")
         pprint.pp(feedback_prompt, width=80)
         llm = ChatOpenAI(model=os.environ["OPENAI_MODEL"], temperature=0.2)
@@ -104,12 +82,10 @@ class TilV2FeedbackResult(BaseModel):
         description="TIL as exactly captured by the user without any modifications.")
     insightful_categorization: str = Field(
         description="TIL categorization as High/Medium/Low based on correctness on the insightful metric.")
-    insightful_reason: str = Field(
-        description="Reason for your assessment in one or two sentences on insightful metric for the user.")
     factuality_categorization: str = Field(
         description="TIL categorization as High/Medium/Low based on correctness on the factuality metric.")
-    factuality_reason: str = Field(
-        description="Reason for your assessment in one or two sentences on factuality metric for the user.")
+    assessment_reason: str = Field(
+        description="Reason for your assessment in one or two sentences on factuality metric and insightful metric for the user if they are not High. The assesment should be from FPV")
 
 
 class TilV2FeedbackResults(BaseModel):
@@ -119,9 +95,7 @@ class TilV2FeedbackResults(BaseModel):
 class TilV2FinalFeedback(BaseModel):
     til: str
     feedback: str
-    feedback_criteria: Optional[str] = None
     reason: Optional[str] = None
-    suggestion: Optional[str] = None
 
 
 class TilV2FeedbackResponse(BaseModel):