File size: 2,473 Bytes
787c984
 
 
 
 
d7fd43f
787c984
 
d7fd43f
787c984
d7fd43f
 
787c984
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
d7fd43f
 
 
 
787c984
d7fd43f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
787c984
 
 
 
 
 
 
d7fd43f
787c984
d7fd43f
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
# LiteLLM custom callback: capture every prompt->response pair for V3 data.
#
# Writes to a writable local file and periodically pushes it to a private HF
# dataset repo, so traces survive the Space having no persistent storage and
# can be viewed/downloaded from the Hub.
import json
import os
import threading
import time

from litellm.integrations.custom_logger import CustomLogger

OUT = "/tmp/traces.jsonl"
DATASET = "aisquared/bolt-hackathon-traces"
FLUSH_EVERY = 10          # push to the Hub every N records
_lock = threading.Lock()
_count = 0


def _push():
    try:
        from huggingface_hub import HfApi
        HfApi(token=os.environ.get("HF_TOKEN")).upload_file(
            path_or_fileobj=OUT, path_in_repo="traces.jsonl",
            repo_id=DATASET, repo_type="dataset")
    except Exception as e:
        with open("/tmp/traces_push.err", "a") as f:
            f.write(f"{type(e).__name__}: {e}\n")


class TraceLogger(CustomLogger):
    def _write(self, kwargs, response_obj, start, end):
        global _count
        try:
            meta = (kwargs.get("litellm_params", {}) or {}).get("metadata", {}) or {}
            try:
                lat = (end - start).total_seconds()
            except Exception:
                lat = None
            rec = {
                "ts": time.time(),
                "team": meta.get("user_api_key_alias") or meta.get("user_api_key"),
                "messages": kwargs.get("messages"),
                "tools": (kwargs.get("optional_params", {}) or {}).get("tools"),
                "response": response_obj.model_dump()
                if hasattr(response_obj, "model_dump") else str(response_obj),
                "latency_s": lat,
            }
            with _lock:
                with open(OUT, "a") as f:
                    f.write(json.dumps(rec, default=str) + "\n")
                _count += 1
                due = (_count % FLUSH_EVERY == 0)
            if due:
                threading.Thread(target=_push, daemon=True).start()
        except Exception as e:
            with open("/tmp/traces.err", "a") as f:
                f.write(f"{type(e).__name__}: {e}\n")

    async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
        self._write(kwargs, response_obj, start_time, end_time)

    def log_success_event(self, kwargs, response_obj, start_time, end_time):
        self._write(kwargs, response_obj, start_time, end_time)


trace_logger = TraceLogger()