Add support for fallback on LLM error or downtown

chatxbt-assistant.py

@@ -1,6 +1,7 @@
-import os
+import os, json
 from typing import Optional, Dict
 from dotenv import load_dotenv
+
 load_dotenv()
 
 import chainlit as cl
@@ -9,13 +10,16 @@ from phi.llm.openai import OpenAIChat
 from phi.tools.duckduckgo import DuckDuckGo
 from phi.tools.yfinance import YFinanceTools
 from src.databases.postgres import sqlalchemy_engine
+from src.knowledge_bases.combined import knowledge_base
 from src.tools.crypto_swap_toolkit import CryptoSwapTools
 from src.tools.crypto_data_toolkit import CryptoDataTools
-from src.tools.crypto_evm_wallet_toolkit import CryptoEVMWalletTools
-from src.tools.user_confirmation_pin_toolkit import UserConfirmationPinToolkit
+from portkey_ai import PORTKEY_GATEWAY_URL, createHeaders
+from src.config.portkey_config import generate_portkey_config
 from src.tools.user_profile_toolkit import UserProfileToolkit
 from phi.storage.assistant.postgres import PgAssistantStorage
-from src.knowledge_bases.combined import knowledge_base
+from src.tools.crypto_evm_wallet_toolkit import CryptoEVMWalletTools
+from src.tools.user_confirmation_pin_toolkit import UserConfirmationPinToolkit
+
 
 storage = PgAssistantStorage(
     # stores runs in the ai.assistant_runs table
@@ -85,14 +89,24 @@ async def set_starters():
 async def start():
     is_dev_mode = True if os.getenv("DEV_MODE") else False
 
+    portkey_local_gateway = True if os.getenv("PORTKEY_LOCAL_GATEWAY_URL") else False
+    portkey_config = generate_portkey_config(local=portkey_local_gateway)
+
     # Initialize the assistant
     cxbt_assistant = Assistant(
-        llm=OpenAIChat(model="gpt-4o"),
+        llm=OpenAIChat(
+            api_key=os.getenv("PORTKEY_API_KEY"),
+            base_url=os.getenv("PORTKEY_LOCAL_GATEWAY_URL") or PORTKEY_GATEWAY_URL,
+            default_headers=createHeaders(
+                api_key=os.getenv("PORTKEY_API_KEY") or None,
+                config=portkey_config
+            ),
+        ),
         tools=[
-            UserProfileToolkit(),
             DuckDuckGo(),
             CryptoDataTools(),
             CryptoSwapTools(),
+            UserProfileToolkit(),
             CryptoEVMWalletTools(),
             UserConfirmationPinToolkit(),
             YFinanceTools(stock_price=True)
@@ -104,7 +118,9 @@ async def start():
         search_knowledge=True,
         read_chat_history=True,
         add_references_to_prompt=True,
-        add_chat_history_to_prompt=True
+        add_chat_history_to_prompt=True,
+        prevent_hallucinations=True,
+        prevent_prompt_injection=True
     )
     cxbt_assistant.knowledge_base.load(recreate=False)
 

src/config/portkey_config.py

@@ -0,0 +1,108 @@
+import json
+import os
+
+def generate_portkey_config(local: bool = True):
+    if local:
+        config = {
+            "strategy": {
+                "mode": "fallback"
+            },
+            "cache": {
+                "mode": "semantic",
+                "max_age": 10000
+            },
+            "retry": {
+                "attempts": 3,
+                "on_status_codes": [429]
+            },
+            "targets": [
+                # {
+                #     "provider": "mistral-ai",
+                #     "api_key": os.getenv("MISTRAL_API_KEY"),
+                #     "override_params": {
+                #         "model": "codestral-latest",
+                #         "max_tokens": 1024,
+                #         "temperature": 0
+                #     }
+                # },
+                {
+                    "provider": "anthropic",
+                    "api_key": os.getenv("ANTHROPIC_API_KEY"),
+                    "override_params": {
+                        "model": "claude-3-5-sonnet-20240620",
+                        "max_tokens": 1024,
+                        "temperature": 0
+                    }
+                },
+                {
+                    "provider": "openai",
+                    "api_key": os.getenv("OPENAI_API_KEY"),
+                    "override_params": {
+                        "model": "gpt-4o",
+                        "max_tokens": 1024,
+                        "temperature": 0
+                    }
+                },
+                {
+                    "provider": "anthropic",
+                    "api_key": os.getenv("ANTHROPIC_API_KEY"),
+                    "override_params": {
+                        "model": "claude-3-opus-20240229",
+                        "max_tokens": 1024,
+                        "temperature": 0
+                    }
+                }
+            ]
+        }
+    else:
+        config = {
+            "strategy": {
+                "mode": "fallback"
+            },
+            "cache": {
+                "mode": "semantic",
+                "max_age": 10000
+            },
+            "retry": {
+                "attempts": 3,
+                "on_status_codes": [429]
+            },
+            "targets": [
+                
+                {
+                    "virtual_key": os.getenv("PORTKEY_OPENAI_VIRTUAL_KEY"),
+                    "override_params": {
+                        "model": "gpt-4o",
+                        "max_tokens": 1024,
+                        "temperature": 0
+                    }
+                },
+                {
+                    "provider": "anthropic",
+                    "api_key": os.getenv("POETRY_ANTHROPIC_API_KEY"),
+                    "override_params": {
+                        "model": "claude-3-5-sonnet-20240620",
+                        "max_tokens": 1024,
+                        "temperature": 0
+                    }
+                },
+                {
+                    "virtual_key": os.getenv("PORTKEY_ANTHROPIC_API_KEY"),
+                    "override_params": {
+                        "model": "claude-3-opus-20240229",
+                        "max_tokens": 1024,
+                        "temperature": 0
+                    }
+                },
+                {
+                    "virtual_key": os.getenv("PORTKEY_MISTRAL_API_KEY"),
+                    "override_params": {
+                        "model": "codestral-latest",
+                        "max_tokens": 1024,
+                        "temperature": 0
+                    }
+                }
+            ]
+        }
+    
+    return json.dumps(config)