endpoints(llm_feedback): add generic llm feedback endpoints

endpoints.py

@@ -1,25 +1,27 @@
 from dotenv import load_dotenv
 load_dotenv()
 
-import uvicorn
-from typing import List
-from tempenv import TemporaryEnvironment
-from pydantic import UUID4
-from fastapi.middleware.cors import CORSMiddleware
+from .workflows.til.suggest_headlines_v2 import SuggestHeadlinesV2, Response as SuggestHeadlinesResponse
+from .workflows.til.rewrite_til_v2 import RewriteTilV2, Response as RewriteTilResponse
+from .workflows.courses.expectation_revision import ExpectationRevision, Inputs as ExpectationRevisionInputs, Response as ExpectationRevisionResponse
+from .workflows.courses.suggest_check_question import SuggestCheckQuestion, Inputs as SuggestCheckQuestionInputs, Response as SuggestCheckQuestionResponse
+from .workflows.courses.suggest_expectations import SuggestExpectations, Inputs as SuggestExpectationsInputs, Expectation, Response as SuggestExpectationsResponse
+from .workflows.til.analyse_til import TilCrew, TilFeedbackResponse
+from .workflows.til.analyse_til_v2 import AnalyseTilV2, TilV2FeedbackResponse
+from .workflows.utils.feedback import Feedback, post_feedback
+from .utils.endpoints_utils import CreateTilInputs
 from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+from pydantic import UUID4
+from tempenv import TemporaryEnvironment
+from typing import List
+import uvicorn
 
-from .utils.endpoints_utils import CreateTilInputs
-from .workflows.utils.feedback import Feedback, post_feedback
-from .workflows.til.analyse_til_v2 import AnalyseTilV2, TilV2FeedbackResponse
-from .workflows.til.analyse_til import TilCrew, TilFeedbackResponse
-from .workflows.courses.suggest_expectations import SuggestExpectations, Inputs as SuggestExpectationsInputs, Expectation, Response as SuggestExpectationsResponse
-from .workflows.courses.suggest_check_question import SuggestCheckQuestion, Inputs as SuggestCheckQuestionInputs, Response as SuggestCheckQuestionResponse
-from .workflows.courses.expectation_revision import ExpectationRevision, Inputs as ExpectationRevisionInputs, Response as ExpectationRevisionResponse
-from .workflows.til.rewrite_til_v2 import RewriteTilV2, Response as RewriteTilResponse
-from .workflows.til.suggest_headlines_v2 import SuggestHeadlinesV2, Response as SuggestHeadlinesResponse
 
 LANGSMITH_STAGING_PROJECT = "customer_agent"
 LANGSMITH_PROD_PROJECT = "growthy-agents"
+LANGSMITH_STAGING_MODEL = "gpt-4o-mini"
+LANGSMITH_PROD_MODEL = "gpt-4o"
 
 description = """
  API helps you do awesome stuff. 🚀
@@ -65,15 +67,13 @@ async def til_feedback_kickoff(content: List[str]) -> TilFeedbackResponse:
 
 @app.post("/til_feedback/{run_id}/feedback", tags=["til_feedback"])
 async def capture_feedback(run_id: UUID4, feedback: Feedback) -> str:
-    print("Metric Type: ", feedback.metric_type)
-    print("Feedback On: ", feedback.feedback_on)
     post_feedback(run_id=run_id, feedback=feedback)
     return "ok"
 
 
 @app.post("/staging/til_feedback", tags=["til_feedback", "staging"])
 async def staging_til_feedback_kickoff(content: List[str]) -> TilFeedbackResponse:
-    with TemporaryEnvironment({"LANGCHAIN_PROJECT": LANGSMITH_STAGING_PROJECT, "OPENAI_MODEL": "gpt-4o-mini"}):
+    with TemporaryEnvironment({"LANGCHAIN_PROJECT": LANGSMITH_STAGING_PROJECT, "OPENAI_MODEL": LANGSMITH_STAGING_MODEL}):
         separator = "\n* "
         content[0] = "* " + content[0]
         inputs = {"content": separator.join(content)}
@@ -83,9 +83,7 @@ async def staging_til_feedback_kickoff(content: List[str]) -> TilFeedbackRespons
 
 @app.post("/staging/til_feedback/{run_id}/feedback", tags=["til_feedback", "staging"])
 async def staging_capture_feedback(run_id: UUID4, feedback: Feedback) -> str:
-    with TemporaryEnvironment({"LANGCHAIN_PROJECT": LANGSMITH_STAGING_PROJECT, "OPENAI_MODEL": "gpt-4o-mini"}):
-        print("Metric Type: ", feedback.metric_type)
-        print("Feedback On: ", feedback.feedback_on)
+    with TemporaryEnvironment({"LANGCHAIN_PROJECT": LANGSMITH_STAGING_PROJECT, "OPENAI_MODEL": LANGSMITH_STAGING_MODEL}):
         post_feedback(run_id=run_id, feedback=feedback)
         return "ok"
 
@@ -98,56 +96,43 @@ def til_v2_analyze_logic(content) -> TilV2FeedbackResponse:
 
 @app.post("/v2/til_feedback", tags=["til_feedback"])
 async def til_v2_feedback_kickoff(content: List[str]) -> TilV2FeedbackResponse:
-    with TemporaryEnvironment({"LANGCHAIN_PROJECT": LANGSMITH_PROD_PROJECT, "OPENAI_MODEL": "gpt-4o"}):
+    with TemporaryEnvironment({"LANGCHAIN_PROJECT": LANGSMITH_PROD_PROJECT, "OPENAI_MODEL": LANGSMITH_PROD_MODEL}):
         return til_v2_analyze_logic(content)
 
 
 @app.post("/staging/v2/til_feedback", tags=["til_feedback", "staging"])
 async def staging_til_v2_feedback_kickoff(content: List[str]) -> TilV2FeedbackResponse:
-    with TemporaryEnvironment({"LANGCHAIN_PROJECT": LANGSMITH_STAGING_PROJECT, "OPENAI_MODEL": "gpt-4o-mini"}):
+    with TemporaryEnvironment({"LANGCHAIN_PROJECT": LANGSMITH_STAGING_PROJECT, "OPENAI_MODEL": LANGSMITH_STAGING_MODEL}):
         return til_v2_analyze_logic(content)
 
 
-@app.post("/v2/til_feedback/{run_id}/feedback", tags=["til_feedback"])
-async def capture_feedback(run_id: UUID4, feedback: Feedback) -> str:
-    print("Metric Type: ", feedback.metric_type)
-    print("Feedback On: ", feedback.feedback_on)
-    post_feedback(run_id=run_id, feedback=feedback)
-    return "ok"
-
-
-@app.post("/staging/v2/til_feedback/{run_id}/feedback", tags=["til_feedback", "staging"])
-async def capture_feedback(run_id: UUID4, feedback: Feedback) -> str:
-    print("Metric Type: ", feedback.metric_type)
-    print("Feedback On: ", feedback.feedback_on)
-    post_feedback(run_id=run_id, feedback=feedback)
-    return "ok"
-
-
 @app.post("/v2/til_rewrite", tags=["til_readability"])
 async def til_v2_rewrite_kickoff(content: List[str]) -> RewriteTilResponse:
-    with TemporaryEnvironment({"LANGCHAIN_PROJECT": LANGSMITH_PROD_PROJECT, "OPENAI_MODEL": "gpt-4o"}):
+    with TemporaryEnvironment({"LANGCHAIN_PROJECT": LANGSMITH_PROD_PROJECT, "OPENAI_MODEL": LANGSMITH_PROD_MODEL}):
         inputs = CreateTilInputs(content)
         result = RewriteTilV2().kickoff(inputs)
         return result
 
+
 @app.post("/staging/v2/til_rewrite", tags=["til_readability", "staging"])
 async def staging_til_v2_rewrite_kickoff(content: List[str]) -> RewriteTilResponse:
-    with TemporaryEnvironment({"LANGCHAIN_PROJECT": LANGSMITH_STAGING_PROJECT, "OPENAI_MODEL": "gpt-4o-mini"}):
+    with TemporaryEnvironment({"LANGCHAIN_PROJECT": LANGSMITH_STAGING_PROJECT, "OPENAI_MODEL": LANGSMITH_STAGING_MODEL}):
         inputs = CreateTilInputs(content)
         result = RewriteTilV2().kickoff(inputs)
         return result
 
+
 @app.post("/v2/til_headlines", tags=["til_headlines"])
 async def til_v2_suggest_headlines(content: List[str]) -> SuggestHeadlinesResponse:
-    with TemporaryEnvironment({"LANGCHAIN_PROJECT": LANGSMITH_PROD_PROJECT, "OPENAI_MODEL": "gpt-4o"}):
+    with TemporaryEnvironment({"LANGCHAIN_PROJECT": LANGSMITH_PROD_PROJECT, "OPENAI_MODEL": LANGSMITH_PROD_MODEL}):
         inputs = CreateTilInputs(content)
         result = SuggestHeadlinesV2().kickoff(inputs)
         return result
 
+
 @app.post("/staging/v2/til_headlines", tags=["til_headlines", "staging"])
 async def staging_til_v2_suggest_headlines(content: List[str]) -> SuggestHeadlinesResponse:
-    with TemporaryEnvironment({"LANGCHAIN_PROJECT": LANGSMITH_STAGING_PROJECT, "OPENAI_MODEL": "gpt-4o-mini"}):
+    with TemporaryEnvironment({"LANGCHAIN_PROJECT": LANGSMITH_STAGING_PROJECT, "OPENAI_MODEL": LANGSMITH_STAGING_MODEL}):
         inputs = CreateTilInputs(content)
         result = SuggestHeadlinesV2().kickoff(inputs)
         return result
@@ -174,25 +159,25 @@ def course_learn_suggest_expectations_feedback_logic(run_id: UUID4, feedback: Fe
 
 @app.post("/course_learn/suggest_expectations", tags=["course_learn"])
 async def course_learn_suggest_expectations(inputs: SuggestExpectationsInputs) -> SuggestExpectationsResponse:
-    with TemporaryEnvironment({"LANGCHAIN_PROJECT": LANGSMITH_PROD_PROJECT, "OPENAI_MODEL": "gpt-4o"}):
+    with TemporaryEnvironment({"LANGCHAIN_PROJECT": LANGSMITH_PROD_PROJECT, "OPENAI_MODEL": LANGSMITH_PROD_MODEL}):
         return course_learn_suggest_expectations_logic(inputs)
 
 
 @app.post("/staging/course_learn/suggest_expectations", tags=["course_learn", "staging"])
 async def staging_course_learn_suggest_expectations(inputs: SuggestExpectationsInputs) -> SuggestExpectationsResponse:
-    with TemporaryEnvironment({"LANGCHAIN_PROJECT": LANGSMITH_STAGING_PROJECT, "OPENAI_MODEL": "gpt-4o-mini"}):
+    with TemporaryEnvironment({"LANGCHAIN_PROJECT": LANGSMITH_STAGING_PROJECT, "OPENAI_MODEL": LANGSMITH_STAGING_MODEL}):
         return course_learn_suggest_expectations_logic(inputs)
 
 
 @app.post("/course_learn/suggest_expectations/{run_id}/feedback", tags=["course_learn"])
 async def capture_suggest_expectations_feedback(run_id: UUID4, feedback: Feedback) -> str:
-    with TemporaryEnvironment({"LANGCHAIN_PROJECT": LANGSMITH_PROD_PROJECT, "OPENAI_MODEL": "gpt-4o"}):
+    with TemporaryEnvironment({"LANGCHAIN_PROJECT": LANGSMITH_PROD_PROJECT, "OPENAI_MODEL": LANGSMITH_PROD_MODEL}):
         return course_learn_suggest_expectations_feedback_logic(run_id, feedback)
 
 
 @app.post("/staging/course_learn/suggest_expectations/{run_id}/feedback", tags=["course_learn", "staging"])
 async def staging_capture_suggest_expectations_feedback(run_id: UUID4, feedback: Feedback) -> str:
-    with TemporaryEnvironment({"LANGCHAIN_PROJECT": LANGSMITH_STAGING_PROJECT, "OPENAI_MODEL": "gpt-4o-mini"}):
+    with TemporaryEnvironment({"LANGCHAIN_PROJECT": LANGSMITH_STAGING_PROJECT, "OPENAI_MODEL": LANGSMITH_STAGING_MODEL}):
         return course_learn_suggest_expectations_feedback_logic(run_id, feedback)
 
 
@@ -208,13 +193,13 @@ def course_learn_expectation_revision_logic(inputs: ExpectationRevisionInputs) -
 
 @app.post("/course_learn/expectation_revision", tags=["course_learn"])
 async def course_learn_expectation_revision(inputs: ExpectationRevisionInputs) -> ExpectationRevisionResponse:
-    with TemporaryEnvironment({"LANGCHAIN_PROJECT": LANGSMITH_PROD_PROJECT, "OPENAI_MODEL": "gpt-4o"}):
+    with TemporaryEnvironment({"LANGCHAIN_PROJECT": LANGSMITH_PROD_PROJECT, "OPENAI_MODEL": LANGSMITH_PROD_MODEL}):
         return course_learn_expectation_revision_logic(inputs)
 
 
 @app.post("/staging/course_learn/expectation_revision", tags=["course_learn", "staging"])
 async def staging_course_learn_expectation_revision(inputs: ExpectationRevisionInputs) -> ExpectationRevisionResponse:
-    with TemporaryEnvironment({"LANGCHAIN_PROJECT": LANGSMITH_STAGING_PROJECT, "OPENAI_MODEL": "gpt-4o-mini"}):
+    with TemporaryEnvironment({"LANGCHAIN_PROJECT": LANGSMITH_STAGING_PROJECT, "OPENAI_MODEL": LANGSMITH_STAGING_MODEL}):
         return course_learn_expectation_revision_logic(inputs)
 
 
@@ -227,13 +212,13 @@ def capture_expectation_revision_feedback_logic(run_id: UUID4, feedback: Feedbac
 
 @app.post("/course_learn/expectation_revision/{run_id}/feedback", tags=["course_learn"])
 async def capture_expectation_revision_feedback(run_id: UUID4, feedback: Feedback) -> str:
-    with TemporaryEnvironment({"LANGCHAIN_PROJECT": LANGSMITH_PROD_PROJECT, "OPENAI_MODEL": "gpt-4o"}):
+    with TemporaryEnvironment({"LANGCHAIN_PROJECT": LANGSMITH_PROD_PROJECT, "OPENAI_MODEL": LANGSMITH_PROD_MODEL}):
         return capture_expectation_revision_feedback_logic(run_id, feedback)
 
 
 @app.post("/staging/course_learn/expectation_revision/{run_id}/feedback", tags=["course_learn", "staging"])
 async def staging_capture_expectation_revision_feedback(run_id: UUID4, feedback: Feedback) -> str:
-    with TemporaryEnvironment({"LANGCHAIN_PROJECT": LANGSMITH_STAGING_PROJECT, "OPENAI_MODEL": "gpt-4o-mini"}):
+    with TemporaryEnvironment({"LANGCHAIN_PROJECT": LANGSMITH_STAGING_PROJECT, "OPENAI_MODEL": LANGSMITH_STAGING_MODEL}):
         return capture_expectation_revision_feedback_logic(run_id, feedback)
 
 
@@ -250,13 +235,13 @@ def course_learn_suggest_check_question_logic(inputs: SuggestCheckQuestionInputs
 
 @app.post("/course_learn/suggest_check_question", tags=["course_learn"])
 async def course_learn_suggest_check_question(inputs: SuggestCheckQuestionInputs) -> SuggestCheckQuestionResponse:
-    with TemporaryEnvironment({"LANGCHAIN_PROJECT": LANGSMITH_PROD_PROJECT, "OPENAI_MODEL": "gpt-4o"}):
+    with TemporaryEnvironment({"LANGCHAIN_PROJECT": LANGSMITH_PROD_PROJECT, "OPENAI_MODEL": LANGSMITH_PROD_MODEL}):
         return course_learn_suggest_check_question_logic(inputs)
 
 
 @app.post("/staging/course_learn/suggest_check_question", tags=["course_learn", "staging"])
 async def staging_course_learn_suggest_check_question(inputs: SuggestCheckQuestionInputs) -> SuggestCheckQuestionResponse:
-    with TemporaryEnvironment({"LANGCHAIN_PROJECT": LANGSMITH_STAGING_PROJECT, "OPENAI_MODEL": "gpt-4o-mini"}):
+    with TemporaryEnvironment({"LANGCHAIN_PROJECT": LANGSMITH_STAGING_PROJECT, "OPENAI_MODEL": LANGSMITH_STAGING_MODEL}):
         return course_learn_suggest_check_question_logic(inputs)
 
 
@@ -269,16 +254,30 @@ def course_learn_suggest_check_question_feedback_logic(run_id: UUID4, feedback:
 
 @app.post("/course_learn/suggest_check_question/{run_id}/feedback", tags=["course_learn"])
 async def course_learn_suggest_check_question_feedback(run_id: UUID4, feedback: Feedback) -> str:
-    with TemporaryEnvironment({"LANGCHAIN_PROJECT": LANGSMITH_PROD_PROJECT, "OPENAI_MODEL": "gpt-4o"}):
+    with TemporaryEnvironment({"LANGCHAIN_PROJECT": LANGSMITH_PROD_PROJECT, "OPENAI_MODEL": LANGSMITH_PROD_MODEL}):
         return course_learn_suggest_check_question_feedback_logic(run_id, feedback)
 
 
 @app.post("/staging/course_learn/suggest_check_question/{run_id}/feedback", tags=["course_learn", "staging"])
 async def staging_course_learn_suggest_check_question_feedback(run_id: UUID4, feedback: Feedback) -> str:
-    with TemporaryEnvironment({"LANGCHAIN_PROJECT": LANGSMITH_STAGING_PROJECT, "OPENAI_MODEL": "gpt-4o-mini"}):
+    with TemporaryEnvironment({"LANGCHAIN_PROJECT": LANGSMITH_STAGING_PROJECT, "OPENAI_MODEL": LANGSMITH_STAGING_MODEL}):
         return course_learn_suggest_check_question_feedback_logic(run_id, feedback)
 
 
+@app.post("/llm_feedback/{run_id}/feedback", tags=["llm_feedback"])
+async def capture_llm_feedback(run_id: UUID4, feedback: Feedback) -> str:
+    with TemporaryEnvironment({"LANGCHAIN_PROJECT": LANGSMITH_PROD_PROJECT, "OPENAI_MODEL": LANGSMITH_PROD_MODEL}):
+        post_feedback(run_id=run_id, feedback=feedback)
+        return "ok"
+
+
+@app.post("/staging/llm_feedback/{run_id}/feedback", tags=["llm_feedback", "staging"])
+async def staging_capture_llm_feedback(run_id: UUID4, feedback: Feedback) -> str:
+    with TemporaryEnvironment({"LANGCHAIN_PROJECT": LANGSMITH_STAGING_PROJECT, "OPENAI_MODEL": LANGSMITH_STAGING_MODEL}):
+        post_feedback(run_id=run_id, feedback=feedback)
+        return "ok"
+
+
 @app.get("/healthcheck")
 async def read_root():
     return {"status": "ok"}

workflows/utils/feedback.py

@@ -9,6 +9,9 @@ class Feedback(BaseModel):
 
 
 def post_feedback(run_id: UUID4, feedback: Feedback):
+    print("Metric Type: ", feedback.metric_type)
+    print("Feedback On: ", feedback.feedback_on)
+
     client = Client()
     client.create_feedback(
         str(run_id),