Update messagers/message_outputer.py

messagers/message_outputer.py

@@ -1,5 +1,6 @@
 import json
 import time
+import tiktoken
 
 class OpenaiStreamOutputer:
     """
@@ -86,8 +87,10 @@ class OpenaiStreamOutputer:
             ]
 
         # Update token counts
+        tokens_used = len(content.split())
+        self.token_manager.add_tokens(tokens_used)  # hypothetical method
         data["usage"]["prompt_tokens"] += tokens_count
-        data["usage"]["completion_tokens"] += len(content.split())
-        data["usage"]["total_tokens"] = data["usage"]["prompt_tokens"] + data["usage"]["completion_tokens"]
+        data["usage"]["completion_tokens"] += tokens_used
+        data["usage"]["total_tokens"] = self.token_manager.get_total_tokens()  # hypothetical method
 
         return self.data_to_string(data, content_type)