add llmtracing

.gitignore

@@ -1,3 +1,4 @@
 .streamlit
 *.log
-__pycache__
+__pycache__
+*.pyc

app.py

@@ -3,7 +3,11 @@ from writer import write_article, incorporate_feedback, _template, evaluate_post
 import hmac
 import dspy
 from dsp.modules import Claude
-from pydantic import BaseModel
+
+import phoenix as px
+
+my_traces = px.Client().get_trace_dataset().save()
+px.launch_app(trace=px.TraceDataset.load(my_traces))
 
 
 def check_password():
@@ -34,45 +38,6 @@ if not check_password():
     st.stop()
 
 
-class LmChoice(BaseModel, arbitrary_types_allowed=True):
-    gpt_35_turbo: dspy.dsp.LM = dspy.OpenAI(
-        model="gpt-3.5-turbo",
-        max_tokens=3800,
-        api_key=st.secrets["OpenAI"],
-    )
-    gpt_4_turbo: dspy.dsp.LM = dspy.OpenAI(
-        model="gpt-4-0125-preview",
-        max_tokens=3800,
-        api_key=st.secrets["OpenAI"],
-    )
-    gpt_4: dspy.dsp.LM = dspy.OpenAI(
-        model="gpt-4-0125-preview",
-        max_tokens=3800,
-        api_key=st.secrets["OpenAI"],
-    )
-    # command_r_plus: dspy.dsp.LM = dspy.Cohere(
-    #     model="command-r-plus",
-    #     max_tokens=4000,
-    #     api_key=st.secrets["Cohere"],
-    # )
-
-    claude_3_opus: dspy.dsp.LM = Claude(
-        model="claude-3-opus-20240229",
-        api_key=st.secrets["Claude"],
-        max_tokens=4096,
-    )
-    claude_3_sonnet: dspy.dsp.LM = Claude(
-        model="claude-3-sonnet-20240229",
-        api_key=st.secrets["Claude"],
-        max_tokens=4096,
-    )
-    claude_3_haiku: dspy.dsp.LM = Claude(
-        model="claude-3-haiku-20240307",
-        api_key=st.secrets["Claude"],
-        max_tokens=4096,
-    )
-
-
 st.title("Linkedin shill")
 
 #! I hate this
@@ -106,23 +71,47 @@ with st.sidebar:
             "claude3 sonnet",
             "claude3 haiku",
             "claude3 opus",
-            "command-r-plus",
+            # "command-r-plus",
         ),
     )
     if lm_choice == "gpt-4-turbo":
-        lm = LmChoice.gpt_4_turbo
+        lm = dspy.OpenAI(
+            model="gpt-4-0125-preview",
+            max_tokens=3800,
+            api_key=st.secrets["OpenAI"],
+        )
     elif lm_choice == "gpt-3.5-turbo":
-        lm = LmChoice.gpt_35_turbo
+        lm = dspy.OpenAI(
+            model="gpt-3.5-turbo",
+            max_tokens=3800,
+            api_key=st.secrets["OpenAI"],
+        )
     elif lm_choice == "claude3 sonnet":
-        lm = LmChoice.claude_3_sonnet
+        lm = Claude(
+            model="claude-3-sonnet-20240229",
+            api_key=st.secrets["Claude"],
+            max_tokens=4096,
+        )
     elif lm_choice == "claude3 haiku":
-        lm = LmChoice.claude_3_haiku
+        lm = Claude(
+            model="claude-3-haiku-20240307",
+            api_key=st.secrets["Claude"],
+            max_tokens=4096,
+        )
     elif lm_choice == "claude3 opus":
-        lm = LmChoice.claude_3_opus
+        lm = Claude(
+            model="claude-3-opus-20240229",
+            api_key=st.secrets["Claude"],
+            max_tokens=4096,
+        )
     # elif lm_choice == "command-r-plus":
     #     lm = LmChoice.command_r_plus
     else:
-        lm = LmChoice.gpt_4
+        lm = dspy.OpenAI(
+            model="gpt-4",
+            max_tokens=3800,
+            api_key=st.secrets["OpenAI"],
+        )
 
     with st.form("my_form"):
         topic = st.text_input("topic", "Oil future")

requirements.txt

@@ -1,2 +1,5 @@
 dspy-ai
 anthropic
+arize-phoenix
+openinference-instrumentation-dspy 
+opentelemetry-exporter-otlp

writer.py

@@ -3,6 +3,23 @@ import dspy
 from dspy import Signature, InputField, Module, ChainOfThought, Predict
 import streamlit as st
 from typing import Optional
+from openinference.instrumentation.dspy import DSPyInstrumentor
+from opentelemetry import trace as trace_api
+from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
+from opentelemetry.sdk import trace as trace_sdk
+from opentelemetry.sdk.resources import Resource
+from opentelemetry.sdk.trace.export import SimpleSpanProcessor
+
+endpoint = "http://127.0.0.1:6006/v1/traces"
+resource = Resource(attributes={})
+tracer_provider = trace_sdk.TracerProvider(resource=resource)
+span_otlp_exporter = OTLPSpanExporter(endpoint=endpoint)
+tracer_provider.add_span_processor(
+    SimpleSpanProcessor(span_exporter=span_otlp_exporter)
+)
+trace_api.set_tracer_provider(tracer_provider=tracer_provider)
+DSPyInstrumentor().instrument()
+
 
 base_lm = dspy.OpenAI(
     model="gpt-4",
@@ -136,7 +153,7 @@ class Feedback(Module):
 class Evaluator(dspy.Signature):
     """You are a creative writing coach, evaluate this linkedin post"""
 
-    linkedin_post = InputField(desc="A linkedin post to evaluate")
+    linkedin_post = InputField(prefix="A linkedin post to evaluate")
     output = dspy.OutputField(
         prefix="A comprehensive evaluation of the input post, styled in markdown"
     )
@@ -147,8 +164,8 @@ class SelfEval(Module):
         super().__init__()
         self.self_eval = ChainOfThought(Evaluator)
 
-    def forward(self, content):
-        eval = self.self_eval(content=content).output
+    def forward(self, linkedin_post):
+        eval = self.self_eval(linkedin_post=linkedin_post).output
         return eval
 
 
@@ -171,5 +188,5 @@ def incorporate_feedback(
 
 def evaluate_post(content: str, lm: Optional[dspy.dsp.LM] = base_lm) -> str:
     with dspy.context(lm=lm):
-        eval = SelfEval().forward(content=content)
+        eval = SelfEval().forward(linkedin_post=content)
     return eval