feat: modify Global TPM calculation

app/utils/global_tpm_tracker.py

@@ -111,24 +111,38 @@ class GlobalTPMTracker:
         # Calculate elapsed time
         elapsed_time = current_time - self._start_time
         elapsed_seconds = elapsed_time.total_seconds()
+
+        
+        # Calculate time threshold for records within timeout window
+        time_threshold = current_time - timedelta(seconds=self._timeout_seconds)
         
         # Get cumulative token counts
-        cumulative_prompt = self._global_tokens['prompt_tokens']
-        cumulative_total = self._global_tokens['total_tokens']
+        total_cumulative_prompt = self._global_tokens['prompt_tokens']
+        total_cumulative_total = self._global_tokens['total_tokens']
+
+
+        # Get cumulative token counts from records within timeout window
+        cumulative_prompt = 0
+        cumulative_total = 0
+        
+        for record in self._records:
+            # Parse the record time string back to datetime for comparison
+            record_time = datetime.strptime(record['current_time'], '%Y-%m-%d %H:%M:%S.%f')
+            
+            # Check if record is within the timeout window
+            if record_time >= time_threshold:
+                cumulative_prompt += record['prompt_tokens']
+                cumulative_total += record['total_tokens']
         
         print(f"Global TPM - Current usage: Prompt tokens: {current_prompt_tokens}, Total tokens: {current_total_tokens}")
-        print(f"Global TPM - Global cumulative: Prompt tokens: {cumulative_prompt}, Total tokens: {cumulative_total}")
+        print(f"Global TPM - Global cumulative: Prompt tokens: {total_cumulative_prompt}, Total tokens: {total_cumulative_total}")
         print(f"Global TPM - Elapsed time: {elapsed_seconds:.2f} seconds")
         
         # Calculate TPM (only if more than 1 second has elapsed)
-        if elapsed_seconds > 1:
-            elapsed_minutes = elapsed_seconds / 60
-            prompt_tpm = cumulative_prompt / elapsed_minutes
-            total_tpm = cumulative_total / elapsed_minutes
-            print(f"Global TPM - Prompt TPM: {prompt_tpm:.2f} tokens/minute")
-            print(f"Global TPM - Total TPM: {total_tpm:.2f} tokens/minute")
-        else:
-            print(f"Global TPM - TPM: N/A (insufficient time elapsed)")
+        prompt_tpm = cumulative_prompt
+        total_tpm = cumulative_total
+        print(f"Global TPM - Prompt TPM: {prompt_tpm:.2f} tokens/minute")
+        print(f"Global TPM - Total TPM: {total_tpm:.2f} tokens/minute")
         
         print("-" * 50)