Major knowledgebase update

chatxbt-assistant.py

@@ -11,7 +11,7 @@ from phi.tools.yfinance import YFinanceTools
 from src.databases.postgres import sqlalchemy_engine
 from src.tools.coin_data_toolkit import CryptoDataTools
 from phi.storage.assistant.postgres import PgAssistantStorage
-from src.knowledge_bases.knowledge_base import knowledge_base
+from src.knowledge_bases.combined import knowledge_base
 
 storage = PgAssistantStorage(
     # store runs in the ai.assistant_runs table

requirements.txt

@@ -1,9 +1,12 @@
+aiobotocore==2.13.0
 aiofiles==23.2.1
 aiohttp==3.9.5
+aioitertools==0.11.0
 aiosignal==1.3.1
 annotated-types==0.7.0
 anthropic==0.28.0
 anyio==3.7.1
+appdirs==1.4.4
 appnope==0.1.4
 asgiref==3.8.1
 asttokens==2.4.1
@@ -14,6 +17,8 @@ backcall==0.2.0
 beautifulsoup4==4.12.3
 bidict==0.23.1
 bleach==6.1.0
+boto3==1.34.131
+botocore==1.34.131
 certifi==2024.6.2
 chainlit==1.1.300
 charset-normalizer==3.3.2
@@ -35,6 +40,8 @@ filelock==3.15.1
 filetype==1.2.0
 frozendict==2.4.4
 frozenlist==1.4.1
+fs==2.4.16
+fs-s3fs==1.1.1
 fsspec==2024.6.0
 gitdb==4.0.11
 GitPython==3.1.43
@@ -53,6 +60,7 @@ ipython==8.12.3
 jedi==0.19.1
 Jinja2==3.1.4
 jiter==0.4.2
+jmespath==1.0.1
 jsonpatch==1.33
 jsonpointer==3.0.0
 jsonschema==4.22.0
@@ -147,7 +155,8 @@ regex==2024.5.15
 requests==2.32.3
 rich==13.7.1
 rpds-py==0.18.1
-setuptools==70.0.0
+s3fs==2024.6.0
+s3transfer==0.10.1
 shellingham==1.5.4
 simple-websocket==1.0.0
 six==1.16.0

run.py

@@ -1,51 +1,7 @@
 import json
 import pprint
-from src.libs.logger import logger
+import asyncio
 
-# from src.data_sources.coingecko import CoinGecko
-# from src.data_sources.cryptocompare import CryptoCompare
+import src.knowledge_bases.pipeline as Pipeline
 
-# cgc = CoinGecko()
-
-# result = cgc.get_coin_price(ids=["bitcoin", "ethereum"], vs_currencies=["usd"])
-# logger.info(f"Got token price: {result}")
-
-# result = cgc.get_coin_data(id="tron")
-# logger.info(f"Got token data: {result}")
-
-# pprint.pprint(result)
-
-# coingecko_instance = get_coingecko_instance()
-# print(coingecko_instance)
-
-# result = coingecko_instance.exchanges.get_list()
-# logger.info(f"Got exchange data: {result}")
-
-# pprint.pprint(result)
-
-
-# result = cgc.get_coin_price(ids=["pepe"], vs_currencies=["usd"])
-# # logger.info(f"Got token data: {result}")
-# pprint.pprint(result)
-
-# result1 = ccp.get_coin_price(ids=["portal"], vs_currencies=["usd"])
-# # logger.info(f"Got token data: {result1}")
-# pprint.pprint(result1)
-
-# from src.data_sources.cryptocompare import CryptoCompare
-# ccp = CryptoCompare()
-
-# result = ccp.get_overall_coin_data(symbol="bnb")
-# pprint.pprint(result)
-
-from src.data_sources.cryptocompare import CryptoCompare
-from src.data_sources.dexscreener import DexScreener
-
-crypto_compare = CryptoCompare()
-dex_screener = DexScreener()
-
-dexscreener_coin_data = dex_screener.search(query='PANDORA')
-cryptocompare_coin_data = crypto_compare.get_overall_coin_data(symbol='PANDORA')
-
-pprint.pprint(dexscreener_coin_data)
-pprint.pprint(cryptocompare_coin_data)
+asyncio.run(Pipeline.main())

src/data_sources/coin_gecko.py

@@ -11,6 +11,24 @@ redis_cache = REDIS_CACHED
 
 class CoinGecko:
     def __init__(self, pro_api: bool = False) -> None:
+        """
+        Initializes the CoinGecko object.
+
+        Args:
+            pro_api (bool, optional): A boolean value indicating whether to use the Pro API. Defaults to False.
+
+        Returns:
+            None
+
+        Raises:
+            ValueError: If the `pro_api` argument is not a boolean value.
+
+        Note:
+            This method initializes the CoinGecko object by creating an instance of either CoinGeckoProClient or CoinGeckoDemoClient based on the `pro_api` argument. If `pro_api` is `True`, it uses the Pro API with the provided API key; otherwise, it uses the Demo API.
+        """
+        if not isinstance(pro_api, bool):
+            raise ValueError("The `pro_api` argument must be a boolean value.")
+
         if pro_api:
             self.cgc = CoinGeckoProClient(api_key=os.getenv('COINGECKO_PRO_API_KEY'))
         else:
@@ -19,18 +37,54 @@ class CoinGecko:
     @redis_cache(ttl=ONE_MINUTE_IN_SECONDS)
     @logger.instrument()
     def get_coin_price(self, ids: list[str], vs_currencies: list[str], cache_ttl: int = None) -> dict:
-        # logger.debug(f"ids: {ids}")
-        # logger.debug(f"vs_currencies: {vs_currencies}")
+        """
+        Retrieves the current price of a list of coins in a specified currency.
+
+        Args:
+            ids (list[str]): A list of unique identifiers for the coins to retrieve prices for.
+            vs_currencies (list[str]): A list of currency codes to convert the coin prices to.
+            cache_ttl (int, optional): The time-to-live (in seconds) for the cached data. Defaults to None.
+
+        Returns:
+            dict: A dictionary containing the current prices of the specified coins in the specified currencies.
 
+        Example:
+            To get the current prices of a list of coins in USD, you can call:
+
+            ```python
+            coin_prices = coin_gecko.get_coin_price(ids=['bitcoin', 'ethereum'], vs_currencies=['usd'])
+            ```
+
+        Note:
+            The `get_coin_price` method uses the `redis_cache` decorator to cache the result for a specified time (`ONE_MINUTE_IN_SECONDS`). This means that if the same request is made within the cache time, the cached result will be returned instead of making a new API call.
+        """
         result = self.cgc.simple.get_price(ids=ids, vs_currencies=vs_currencies)
-        # logger.debug(f"Result: {result}")
 
         return result
     
     @redis_cache(ttl=ONE_HOUR_IN_SECONDS)
     @logger.instrument()
     def get_coin_data(self, id: str, cache_ttl: int = None) -> dict:
+        """
+        Retrieves detailed information about a specific coin.
+
+        Args:
+            id (str): The unique identifier of the coin.
+            cache_ttl (int, optional): The time-to-live (in seconds) for the cached data. Defaults to None.
+
+        Returns:
+            dict: A dictionary containing the detailed information about the specified coin.
 
+        Example:
+            To get data for the coin with id 'bitcoin', you can call:
+
+            ```python
+            coin_data = coin_gecko.get_coin_data(id='bitcoin')
+            ```
+
+        Note:
+            The `get_coin_data` method uses the `redis_cache` decorator to cache the result for a specified time (`ONE_HOUR_IN_SECONDS`). This means that if the same request is made within the cache time, the cached result will be returned instead of making a new API call.
+        """
         result = self.cgc.coins.get_id(id=id, localization=False, market_data=False, tickers=False, sparkline=False)
         logger.debug(f"Result: {result}")
 
@@ -39,7 +93,26 @@ class CoinGecko:
     @redis_cache(ttl=ONE_MONTH_IN_SECONDS)
     @logger.instrument()
     def get_coin_category_data(self, params: dict = None, cache_ttl: int = None) -> dict:
+        """
+        Retrieves data for all available coin categories.
+
+        Args:
+            params (dict, optional): A dictionary containing optional parameters for the request. Defaults to None.
+            cache_ttl (int, optional): The time-to-live (in seconds) for the cached data. Defaults to None.
 
+        Returns:
+            dict: A dictionary containing the data for all available coin categories.
+
+        Example:
+            To get data for all available coin categories, you can call:
+
+            ```python
+            category_data = coin_gecko.get_coin_category_data()
+            ```
+
+        Note:
+            The `get_coin_category_data` method uses the `redis_cache` decorator to cache the result for a specified time (`ONE_MONTH_IN_SECONDS`). This means that if the same request is made within the cache time, the cached result will be returned instead of making a new API call.
+        """
         params = params or {
             "order": "market_cap_desc",
         }
@@ -52,9 +125,25 @@ class CoinGecko:
     @redis_cache(ttl=ONE_MONTH_IN_SECONDS)
     @logger.instrument()
     def get_exchanges_list(self, cache_ttl: int = None) -> dict:
+        """
+        Retrieves a list of all available exchanges.
+
+        Args:
+            cache_ttl (int, optional): The time-to-live (in seconds) for the cached data. Defaults to None.
+
+        Returns:
+            dict: A dictionary containing a list of all available exchanges.
+
+        Example:
+            To get a list of all available exchanges, you can call:
+
+            ```python
+            exchange_list = coin_gecko.get_exchanges_list()
+            ```
 
-        # result = self.cgc.exchanges.get(per_page=10)
-        # result = self.cgc.exchanges.get_id(id=id)
+        Note:
+            The `get_exchanges_list` method uses the `redis_cache` decorator to cache the result for a specified time (`ONE_MONTH_IN_SECONDS`). This means that if the same request is made within the cache time, the cached result will be returned instead of making a new API call.
+        """
         result = self.cgc.exchanges.get_list()
         logger.debug(f"Result: {result}")
 
@@ -63,6 +152,31 @@ class CoinGecko:
     @redis_cache(ttl=ONE_HOUR_IN_SECONDS)
     @logger.instrument()
     def get_exchange_data(self, id: str = None, cache_ttl: int = None) -> dict:
+        """
+        Retrieves detailed information about a specific exchange.
+
+        Args:
+            id (str, optional): The unique identifier of the exchange. Defaults to None.
+            cache_ttl (int, optional): The time-to-live (in seconds) for the cached data. Defaults to None.
+
+        Returns:
+            dict: A dictionary containing the detailed information about the specified exchange.
+
+        Raises:
+            ValueError: If the `id` is not provided and `None` is passed as the argument.
+
+        Example:
+            To get data for the exchange with id 'binance', you can call:
+
+            ```python
+            exchange_data = coin_gecko.get_exchange_data(id='binance')
+            ```
+
+        Note:
+            The `get_exchange_data` method uses the `redis_cache` decorator to cache the result for a specified time (`ONE_HOUR_IN_SECONDS`). This means that if the same request is made within the cache time, the cached result will be returned instead of making a new API call.
+        """
+        if id is None:
+            raise ValueError("Exchange ID must be provided.")
 
         result = self.cgc.exchanges.get_id(id=id)
         logger.debug(f"Result: {result}")

src/data_sources/cryptocompare.py

@@ -3,10 +3,13 @@ from dotenv import load_dotenv
 load_dotenv()
 
 import httpx
+from enum import Enum
 from src.libs.logger import logger
 from src.databases.redis import REDIS_CACHED
 from src.libs.helper_functions import convert_to_snakecase
 from src.libs.constants import (
+    ONE_HOUR_IN_SECONDS,
+    ONE_MONTH_IN_SECONDS,
     ONE_MINUTE_IN_SECONDS,
     ONE_QUARTER_IN_SECONDS,
     CRYPTO_COMPARE_API_BASE_URL,
@@ -15,13 +18,29 @@ from src.libs.constants import (
 
 redis_cache = REDIS_CACHED
 
+class CryptoCompareNewsSortOrder(Enum):
+    LATEST = "latest"
+    POPULAR = "popular"
+
+    def __str__(self):
+        return self.value
+    
+class CryptoCompareLanguages(Enum):
+    English = "EN"
+    French = "FR"
+    Espanol = "ES"
+    Turkish = "TR"
+    Portuguese = "PT"
+
+    def __str__(self):
+        return self.value
 
 class CryptoCompare:
     def __init__(self, base_url: str = None, asset_data_base_url: str = None) -> None:
         self.CRYPTO_COMPARE_API_BASE_URL = base_url or CRYPTO_COMPARE_API_BASE_URL
         self.CRYPTO_COMPARE_ASSET_DATA_API_BASE_URL = asset_data_base_url or CRYPTO_COMPARE_ASSET_DATA_API_BASE_URL
 
-    @redis_cache(ttl=ONE_MINUTE_IN_SECONDS)
+    # @redis_cache(ttl=ONE_MINUTE_IN_SECONDS)
     @logger.instrument()
     def get_all_coins(self) -> dict:
         url = f"{self.CRYPTO_COMPARE_API_BASE_URL}data/all/coinlist"
@@ -91,4 +110,216 @@ class CryptoCompare:
         except httpx.HTTPError as e:
             logger.debug(f"An error occurred while making the request: {e}")
             # print(f"An error occurred while making the request: {e}")
-            return None
+            return None
+        
+    @redis_cache(ttl=ONE_QUARTER_IN_SECONDS)
+    @logger.instrument()
+    def get_news_categories(self, cache_ttl: int = None) -> dict:
+        """
+        Retrieves the news categories from the CryptoCompare API.
+       
+        Args:
+            cache_ttl (int, optional): The time-to-live (TTL) for the cached response. Defaults to None.
+
+        Returns:
+            dict: A dictionary containing the news categories from the CryptoCompare API.
+
+        Raises:
+            httpx.HTTPError: If an error occurs while making the request to the CryptoCompare API.
+        """
+        url = f"{self.CRYPTO_COMPARE_API_BASE_URL}data/news/categories"
+        logger.debug(f"Query URL: {url}")
+
+        params = {
+            "api_key": os.getenv("CRYPTOCOMPARE_API_KEY"),
+        }
+        logger.debug(f"Query params: {params}")
+
+        headers = {"Content-type":"application/json; charset=UTF-8"}
+        logger.debug(f"Query headers {headers}")
+
+        try:
+            with httpx.Client(timeout=30.0) as client:
+                response = client.get(url, params=params, headers=headers)
+                response.raise_for_status()  # Raise an exception if the request was unsuccessful
+                return response.json()
+        except httpx.HTTPError as e:
+            logger.debug(f"An error occurred while making the request: {e}")
+            # print(f"An error occurred while making the request: {e}")
+            return None
+        
+    @redis_cache(ttl=ONE_HOUR_IN_SECONDS)
+    @logger.instrument()
+    def get_latest_news_articles(
+        self,
+        language: CryptoCompareLanguages = CryptoCompareLanguages.English,
+        sort_order: CryptoCompareNewsSortOrder = CryptoCompareNewsSortOrder.LATEST,
+        cache_ttl: int = None
+    ) -> dict:
+        """
+        Retrieves the latest news articles from the CryptoCompare API.
+
+        Args:
+            language (CryptoCompareLanguages, optional): The language of the news articles. Defaults to English.
+            sort_order (CryptoCompareNewsSortOrder, optional): The sort order of the news articles. Defaults to 'latest'.
+            cache_ttl (int, optional): The time-to-live (TTL) for the cached response. Defaults to None.
+
+        Returns:
+            dict: A dictionary containing the latest news articles from the CryptoCompare API.
+
+        Raises:
+            httpx.HTTPError: If an error occurs while making the request to the CryptoCompare API.
+
+        Example:
+            # Get the latest news articles in English, sorted by popularity
+            latest_news = crypto_compare_instance.get_latest_news_articles(language=CryptoCompareLanguages.English, sort_order=CryptoCompareNewsSortOrder.POPULAR)
+        """
+        url = f"{self.CRYPTO_COMPARE_API_BASE_URL}data/v2/news/"
+        logger.debug(f"Query URL: {url}")
+
+        params = {
+            "api_key": os.getenv("CRYPTOCOMPARE_API_KEY"),
+            "lang": language,
+            "sortOrder": sort_order,
+        }
+        logger.debug(f"Query params: {params}")
+
+        headers = {"Content-type":"application/json; charset=UTF-8"}
+        logger.debug(f"Query headers {headers}")
+
+        try:
+            with httpx.Client(timeout=30.0) as client:
+                response = client.get(url, params=params, headers=headers)
+                response.raise_for_status()  # Raise an exception if the request was unsuccessful
+                return response.json()
+        except httpx.HTTPError as e:
+            logger.debug(f"An error occurred while making the request: {e}")
+            # print(f"An error occurred while making the request: {e}")
+            return None
+        
+    @redis_cache(ttl=ONE_MONTH_IN_SECONDS)
+    @logger.instrument()
+    def get_all_exchanges_general_info(self, cache_ttl: int = None) -> dict:
+        """
+        Retrieves general information about all supported exchanges from the CryptoCompare API.
+
+        Args:
+            cache_ttl (int, optional): The time-to-live (TTL) for the cached response. Defaults to None.
+
+        Returns:
+            dict: A dictionary containing general information about all supported exchanges from the CryptoCompare API.
+
+        Raises:
+            httpx.HTTPError: If an error occurs while making the request to the CryptoCompare API.
+
+        Example:
+            # Get general information about all supported exchanges
+            all_exchanges_info = crypto_compare_instance.get_all_exchanges_general_info()
+        """
+        url = f"{self.CRYPTO_COMPARE_API_BASE_URL}data/exchanges/general"
+        logger.debug(f"Query URL: {url}")
+
+        params = {
+            "api_key": os.getenv("CRYPTOCOMPARE_API_KEY"),
+        }
+        logger.debug(f"Query params: {params}")
+
+        headers = {"Content-type":"application/json; charset=UTF-8"}
+        logger.debug(f"Query headers {headers}")
+
+        try:
+            with httpx.Client(timeout=30.0) as client:
+                response = client.get(url, params=params, headers=headers)
+                response.raise_for_status()  # Raise an exception if the request was unsuccessful
+                return response.json()
+        except httpx.HTTPError as e:
+            logger.debug(f"An error occurred while making the request: {e}")
+            # print(f"An error occurred while making the request: {e}")
+            return None
+        
+    @redis_cache(ttl=ONE_MONTH_IN_SECONDS)
+    @logger.instrument()
+    def get_all_wallets_general_info(self, cache_ttl: int = None) -> dict:
+        """
+        Retrieves general information about all supported wallets from the CryptoCompare API.
+
+        Args:
+            cache_ttl (int, optional): The time-to-live (TTL) for the cached response. Defaults to None.
+
+        Returns:
+            dict: A dictionary containing general information about all supported wallets from the CryptoCompare API.
+
+        Raises:
+            httpx.HTTPError: If an error occurs while making the request to the CryptoCompare API.
+
+        Example:
+            # Get general information about all supported wallets
+            all_wallets_info = crypto_compare_instance.get_all_wallets_general_info()
+        """
+        url = f"{self.CRYPTO_COMPARE_API_BASE_URL}data/wallets/general"
+        logger.debug(f"Query URL: {url}")
+
+        params = {
+            "api_key": os.getenv("CRYPTOCOMPARE_API_KEY"),
+        }
+        logger.debug(f"Query params: {params}")
+
+        headers = {"Content-type":"application/json; charset=UTF-8"}
+        logger.debug(f"Query headers {headers}")
+
+        try:
+            with httpx.Client(timeout=30.0) as client:
+                response = client.get(url, params=params, headers=headers)
+                response.raise_for_status()  # Raise an exception if the request was unsuccessful
+                return response.json()
+        except httpx.HTTPError as e:
+            logger.debug(f"An error occurred while making the request: {e}")
+            # print(f"An error occurred while making the request: {e}")
+            return None
+        
+    # @redis_cache(ttl=ONE_MONTH_IN_SECONDS)
+    @logger.instrument()
+    def get_top_tier_exchanges_list(
+        self,
+        top_tier: bool = False,
+        cache_ttl: int = None
+    ) -> dict:
+        """
+        Retrieves a list of top-tier exchanges from the CryptoCompare API.
+
+        Args:
+            top_tier (bool, optional): A boolean flag indicating whether to retrieve top-tier exchanges. Defaults to False.
+            cache_ttl (int, optional): The time-to-live (TTL) for the cached response. Defaults to None.
+
+        Returns:
+            dict: A dictionary containing a list of top-tier exchanges from the CryptoCompare API.
+
+        Raises:
+            httpx.HTTPError: If an error occurs while making the request to the CryptoCompare API.
+
+        Example:
+            # Get a list of top-tier exchanges
+            top_tier_exchanges = crypto_compare_instance.get_top_tier_exchanges_list(top_tier=True)
+        """
+        url = f"{self.CRYPTO_COMPARE_API_BASE_URL}data/v4/all/exchanges"
+        logger.debug(f"Query URL: {url}")
+
+        params = {
+            "api_key": os.getenv("CRYPTOCOMPARE_API_KEY"),
+            "topTier": 'true' if top_tier else 'false',
+        }
+        logger.debug(f"Query params: {params}")
+
+        headers = {"Content-type":"application/json; charset=UTF-8"}
+        logger.debug(f"Query headers {headers}")
+
+        try:
+            with httpx.Client(timeout=30.0) as client:
+                response = client.get(url, params=params, headers=headers)
+                response.raise_for_status()  # Raise an exception if the request was unsuccessful
+                return response.json()
+        except httpx.HTTPError as e:
+            logger.debug(f"An error occurred while making the request: {e}")
+            # print(f"An error occurred while making the request: {e}")
+            return None
+        

src/data_sources/dexscreener.py

@@ -10,7 +10,6 @@ load_dotenv()
 
 redis_cache = REDIS_CACHED
 
-
 class DexScreener:
     """
     A class for interacting with the Dex Screener API.

src/knowledge_bases/{knowledge_base.py → combined.py}

@@ -1,6 +1,6 @@
 from phi.knowledge.combined import CombinedKnowledgeBase
 from phi.vectordb.pgvector import PgVector2
-from src.knowledge_bases.json_knowledge_base import json_knowledge_base
+from src.knowledge_bases.json import json_knowledge_base
 from src.databases.postgres import sqlalchemy_engine
 
 knowledge_base = CombinedKnowledgeBase(

src/knowledge_bases/data/coin_data/bnb.json

@@ -1,713 +0,0 @@
-{
-    "Data": {
-        "ID": 8,
-        "TYPE": "162",
-        "ID_LEGACY": 204788,
-        "ID_PARENT_ASSET": null,
-        "SYMBOL": "BNB",
-        "URI": "bnb",
-        "ASSET_TYPE": "BLOCKCHAIN",
-        "PARENT_ASSET_SYMBOL": null,
-        "CREATED_ON": 1661250250,
-        "UPDATED_ON": 1717070963,
-        "PUBLIC_NOTICE": "2024-05-30: The BNB Chain is currently in the final phase of the BNB Smart Chain Feynman Hardfork. This stage involves migrating assets from the BNB Beacon Chain to the BNB Smart Chain (BSC). Previous steps, including enabling native BSC validators, staking, cross-chain stake migration, and governance, have been completed. Stakeholders must take necessary actions before June 2024 to avoid risks associated with this migration process. For more details, visit the [BNB Chain Fusion page](https://www.bnbchain.org/en/bnb-chain-fusion).",
-        "NAME": "Binance Coin",
-        "LOGO_URL": "https://resources.cryptocompare.com/asset-management/8/1661250459982.png",
-        "LAUNCH_DATE": 1555545600,
-        "PREVIOUS_ASSET_SYMBOLS": null,
-        "ASSET_ALTERNATIVE_IDS": [
-            {
-                "NAME": "CMC",
-                "ID": 1839
-            },
-            {
-                "NAME": "CG",
-                "ID": "binancecoin"
-            },
-            {
-                "NAME": "ISIN",
-                "ID": "XT8N2VXJKB15"
-            },
-            {
-                "NAME": "VALOR",
-                "ID": "114384091"
-            },
-            {
-                "NAME": "DTI",
-                "ID": "8N2VXJKB1"
-            }
-        ],
-        "ASSET_DESCRIPTION_SNIPPET": "BNB, created by Changpeng Zhao, is integral to the Binance ecosystem, serving as a versatile utility token within and outside of the Binance platform.",
-        "ASSET_DECIMAL_POINTS": 8,
-        "SUPPORTED_PLATFORMS": [
-            {
-                "BLOCKCHAIN": "MATIC",
-                "TOKEN_STANDARD": "ERC20",
-                "BRIDGE_OPERATOR": "PORTAL_TOKEN_BRIDGE",
-                "IS_ASSET_ISSUER": true,
-                "EXPLORER_URL": "https://polygonscan.com/token/0xecdcb5b88f8e3c15f95c720c51c71c9e2080525d",
-                "SMART_CONTRACT_ADDRESS": "0xecdcb5b88f8e3c15f95c720c51c71c9e2080525d",
-                "LAUNCH_DATE": 1636416000,
-                "TRADING_AS": "WBNB",
-                "DECIMALS": 18
-            },
-            {
-                "BLOCKCHAIN": "SOL",
-                "TOKEN_STANDARD": "SPL",
-                "BRIDGE_OPERATOR": "PORTAL_TOKEN_BRIDGE",
-                "IS_ASSET_ISSUER": true,
-                "EXPLORER_URL": "https://solscan.io/token/9gP2kCy3wA1ctvYWQk75guqXuHfrEomqydHLtcTCqiLa",
-                "SMART_CONTRACT_ADDRESS": "9gP2kCy3wA1ctvYWQk75guqXuHfrEomqydHLtcTCqiLa",
-                "LAUNCH_DATE": 1633910400,
-                "TRADING_AS": "WBNB",
-                "DECIMALS": 8
-            },
-            {
-                "BLOCKCHAIN": "ETH",
-                "TOKEN_STANDARD": "ERC20",
-                "BRIDGE_OPERATOR": "NATIVE_ASSET",
-                "IS_ASSET_ISSUER": true,
-                "EXPLORER_URL": "https://etherscan.io/token/0xB8c77482e45F1F44dE1745F52C74426C631bDD52",
-                "SMART_CONTRACT_ADDRESS": "0xB8c77482e45F1F44dE1745F52C74426C631bDD52",
-                "LAUNCH_DATE": 1499212800,
-                "TRADING_AS": "BNB",
-                "DECIMALS": 18
-            },
-            {
-                "BLOCKCHAIN": "BNB",
-                "TOKEN_STANDARD": "BEP20",
-                "BRIDGE_OPERATOR": "BINANCE_BRIDGE",
-                "IS_ASSET_ISSUER": true,
-                "EXPLORER_URL": "https://bscscan.com/token/0xbb4CdB9CBd36B01bD1cBaEBF2De08d9173bc095c",
-                "SMART_CONTRACT_ADDRESS": "0xbb4CdB9CBd36B01bD1cBaEBF2De08d9173bc095c",
-                "LAUNCH_DATE": 1599091200,
-                "TRADING_AS": "WBNB",
-                "DECIMALS": 18
-            },
-            {
-                "BLOCKCHAIN": "AVAX",
-                "TOKEN_STANDARD": "ERC20",
-                "BRIDGE_OPERATOR": "MULTICHAIN",
-                "IS_ASSET_ISSUER": true,
-                "EXPLORER_URL": "https://snowtrace.io/token/0x264c1383ea520f73dd837f915ef3a732e204a493",
-                "SMART_CONTRACT_ADDRESS": "0x264c1383ea520f73dd837f915ef3a732e204a493",
-                "LAUNCH_DATE": 1623974400,
-                "RETIRE_DATE": 1684627200,
-                "TRADING_AS": "BNB",
-                "DECIMALS": 18
-            },
-            {
-                "BLOCKCHAIN": "WEMIX",
-                "TOKEN_STANDARD": "ERC20",
-                "BRIDGE_OPERATOR": "MULTICHAIN",
-                "IS_ASSET_ISSUER": true,
-                "EXPLORER_URL": "https://explorer.wemix.com/token/0xC1Be9a4D5D45BeeACAE296a7BD5fADBfc14602C4",
-                "SMART_CONTRACT_ADDRESS": "0xC1Be9a4D5D45BeeACAE296a7BD5fADBfc14602C4",
-                "LAUNCH_DATE": 1667433600,
-                "RETIRE_DATE": 1684627200,
-                "TRADING_AS": "BNB",
-                "DECIMALS": 18
-            },
-            {
-                "BLOCKCHAIN": "MOVR",
-                "TOKEN_STANDARD": "ERC20",
-                "BRIDGE_OPERATOR": "MULTICHAIN",
-                "IS_ASSET_ISSUER": true,
-                "EXPLORER_URL": "https://moonriver.moonscan.io/token/0x2bf9b864cdc97b08b6d79ad4663e71b8ab65c45c",
-                "SMART_CONTRACT_ADDRESS": "0x2bf9b864cdc97b08b6d79ad4663e71b8ab65c45c",
-                "LAUNCH_DATE": 1630627200,
-                "RETIRE_DATE": 1684627200,
-                "TRADING_AS": "BNB",
-                "DECIMALS": 18
-            },
-            {
-                "BLOCKCHAIN": "GLMR",
-                "TOKEN_STANDARD": "ERC20",
-                "BRIDGE_OPERATOR": "MULTICHAIN",
-                "IS_ASSET_ISSUER": true,
-                "EXPLORER_URL": "https://moonscan.io/token/0xc9baa8cfdde8e328787e29b4b078abf2dadc2055",
-                "SMART_CONTRACT_ADDRESS": "0xc9baa8cfdde8e328787e29b4b078abf2dadc2055",
-                "LAUNCH_DATE": 1641945600,
-                "RETIRE_DATE": 1684627200,
-                "TRADING_AS": "BNB",
-                "DECIMALS": 18
-            },
-            {
-                "BLOCKCHAIN": "ETC",
-                "TOKEN_STANDARD": "ERC20",
-                "BRIDGE_OPERATOR": "MULTICHAIN",
-                "IS_ASSET_ISSUER": true,
-                "EXPLORER_URL": "https://blockscout.com/etc/mainnet/token/0x0dCb0CB0120d355CdE1ce56040be57Add0185BAa",
-                "SMART_CONTRACT_ADDRESS": "0x0dCb0CB0120d355CdE1ce56040be57Add0185BAa",
-                "LAUNCH_DATE": 1654819200,
-                "RETIRE_DATE": 1684627200,
-                "TRADING_AS": "BNB",
-                "DECIMALS": 18
-            },
-            {
-                "BLOCKCHAIN": "DOGE",
-                "TOKEN_STANDARD": "ERC20",
-                "BRIDGE_OPERATOR": "MULTICHAIN",
-                "IS_ASSET_ISSUER": true,
-                "EXPLORER_URL": "https://explorer.dogechain.dog/token/0xA649325Aa7C5093d12D6F98EB4378deAe68CE23F",
-                "SMART_CONTRACT_ADDRESS": "0xA649325Aa7C5093d12D6F98EB4378deAe68CE23F",
-                "LAUNCH_DATE": 1660780800,
-                "RETIRE_DATE": 1684627200,
-                "TRADING_AS": "BNB",
-                "DECIMALS": 18
-            },
-            {
-                "BLOCKCHAIN": "KAI",
-                "TOKEN_STANDARD": "ERC20",
-                "BRIDGE_OPERATOR": "MULTICHAIN",
-                "IS_ASSET_ISSUER": true,
-                "EXPLORER_URL": "https://explorer.kardiachain.io/token/0xB44a9B6905aF7c801311e8F4E76932ee959c663C",
-                "SMART_CONTRACT_ADDRESS": "0xB44a9B6905aF7c801311e8F4E76932ee959c663C",
-                "LAUNCH_DATE": 1656547200,
-                "RETIRE_DATE": 1684627200,
-                "TRADING_AS": "BNB",
-                "DECIMALS": 18
-            },
-            {
-                "BLOCKCHAIN": "FTM",
-                "TOKEN_STANDARD": "ERC20",
-                "BRIDGE_OPERATOR": "SPOOKYSWAP_BRIDGE",
-                "IS_ASSET_ISSUER": true,
-                "EXPLORER_URL": "https://ftmscan.com/token/0xd67de0e0a0fd7b15dc8348bb9be742f3c5850454",
-                "SMART_CONTRACT_ADDRESS": "0xd67de0e0a0fd7b15dc8348bb9be742f3c5850454",
-                "LAUNCH_DATE": 1620691200,
-                "TRADING_AS": "BNB",
-                "DECIMALS": 18
-            },
-            {
-                "BLOCKCHAIN": "ETH",
-                "TOKEN_STANDARD": "ERC20",
-                "BRIDGE_OPERATOR": "REN_BRIDGE",
-                "IS_ASSET_ISSUER": true,
-                "EXPLORER_URL": "https://etherscan.io/token/0xb8f7762192C840A5E4A6661D5Fa2D4d103E1d16a",
-                "SMART_CONTRACT_ADDRESS": "0xb8f7762192C840A5E4A6661D5Fa2D4d103E1d16a",
-                "LAUNCH_DATE": 1646092800,
-                "TRADING_AS": "RENBNB",
-                "DECIMALS": 18,
-                "IS_INHERITED": true
-            },
-            {
-                "BLOCKCHAIN": "FTM",
-                "TOKEN_STANDARD": "ERC20",
-                "BRIDGE_OPERATOR": "REN_BRIDGE",
-                "IS_ASSET_ISSUER": true,
-                "EXPLORER_URL": "https://ftmscan.com/token/0xb8f7762192C840A5E4A6661D5Fa2D4d103E1d16a",
-                "SMART_CONTRACT_ADDRESS": "0xb8f7762192C840A5E4A6661D5Fa2D4d103E1d16a",
-                "LAUNCH_DATE": 1656979200,
-                "TRADING_AS": "RENBNB",
-                "DECIMALS": 18,
-                "IS_INHERITED": true
-            },
-            {
-                "BLOCKCHAIN": "MATIC",
-                "TOKEN_STANDARD": "ERC20",
-                "BRIDGE_OPERATOR": "REN_BRIDGE",
-                "IS_ASSET_ISSUER": true,
-                "EXPLORER_URL": "https://polygonscan.com/token/0xb8f7762192C840A5E4A6661D5Fa2D4d103E1d16a",
-                "SMART_CONTRACT_ADDRESS": "0xb8f7762192C840A5E4A6661D5Fa2D4d103E1d16a",
-                "LAUNCH_DATE": 1638144000,
-                "TRADING_AS": "RENBNB",
-                "DECIMALS": 18,
-                "IS_INHERITED": true
-            },
-            {
-                "BLOCKCHAIN": "AVAX",
-                "TOKEN_STANDARD": "ERC20",
-                "BRIDGE_OPERATOR": "REN_BRIDGE",
-                "IS_ASSET_ISSUER": true,
-                "EXPLORER_URL": "https://snowtrace.io/token/0xb8f7762192C840A5E4A6661D5Fa2D4d103E1d16a",
-                "SMART_CONTRACT_ADDRESS": "0xb8f7762192C840A5E4A6661D5Fa2D4d103E1d16a",
-                "LAUNCH_DATE": 1638230400,
-                "TRADING_AS": "RENBNB",
-                "DECIMALS": 18,
-                "IS_INHERITED": true
-            }
-        ],
-        "ASSET_CUSTODIANS": [
-            {
-                "NAME": "FIREBLOCKS"
-            },
-            {
-                "NAME": "BITGO"
-            },
-            {
-                "NAME": "MATRIXPORT"
-            },
-            {
-                "NAME": "CACTUS_CUSTODY"
-            },
-            {
-                "NAME": "COBO"
-            },
-            {
-                "NAME": "CYBAVO"
-            },
-            {
-                "NAME": "LEDGER_ENTERPRISE"
-            },
-            {
-                "NAME": "ETANA_CUSTODY"
-            },
-            {
-                "NAME": "BITCOIN_SUISSE"
-            },
-            {
-                "NAME": "COPPER"
-            }
-        ],
-        "CONTROLLED_ADDRESSES": null,
-        "ASSET_SECURITY_METRICS": [
-            {
-                "NAME": "CERTIK",
-                "OVERALL_SCORE": 92.11,
-                "OVERALL_RANK": 35,
-                "UPDATED_AT": 1718236800
-            }
-        ],
-        "SUPPLY_MAX": 100000000,
-        "SUPPLY_ISSUED": 202000000,
-        "SUPPLY_TOTAL": 147584208.501559,
-        "SUPPLY_CIRCULATING": 147584208.50155932,
-        "SUPPLY_FUTURE": 0,
-        "SUPPLY_LOCKED": 0,
-        "SUPPLY_BURNT": 54415791.498441,
-        "SUPPLY_STAKED": null,
-        "LAST_BLOCK_MINT": null,
-        "LAST_BLOCK_BURN": null,
-        "BURN_ADDRESSES": null,
-        "LOCKED_ADDRESSES": null,
-        "HAS_SMART_CONTRACT_CAPABILITIES": true,
-        "SMART_CONTRACT_SUPPORT_TYPE": "ADVANCED_EXECUTION_RIGHTS",
-        "TARGET_BLOCK_MINT": null,
-        "TARGET_BLOCK_TIME": null,
-        "LAST_BLOCK_NUMBER": 376559131,
-        "LAST_BLOCK_TIMESTAMP": null,
-        "LAST_BLOCK_TIME": null,
-        "LAST_BLOCK_SIZE": null,
-        "LAST_BLOCK_ISSUER": null,
-        "LAST_BLOCK_TRANSACTION_FEE_TOTAL": null,
-        "LAST_BLOCK_TRANSACTION_COUNT": null,
-        "LAST_BLOCK_HASHES_PER_SECOND": null,
-        "LAST_BLOCK_DIFFICULTY": null,
-        "SUPPORTED_STANDARDS": [
-            {
-                "NAME": "BEP2"
-            },
-            {
-                "NAME": "BEP20"
-            }
-        ],
-        "LAYER_TWO_SOLUTIONS": [
-            {
-                "NAME": "opBNB",
-                "WEBSITE_URL": "https://opbnb.bnbchain.org/en",
-                "DESCRIPTION": "Developing on opBNB, an optimistic rollup Layer 2 scaling solution for the BNB Smart Chain, is similar to building directly on the BNB Smart Chain itself. opBNB employs an Ethereum Virtual Machine (EVM) execution engine, which allows for easy migration of decentralized applications from Ethereum, BNB Smart Chain, Polygon, and other EVM-compatible chains with minimal code changes.\n\nThe platform scales the BNB Smart Chain by bundling transactions off-chain and using fraud proofs to ensure their validity, leading to significantly higher throughput compared to the underlying chain. This compatibility with the EVM means that any decentralized application (dApp), smart contract, or other application designed for the EVM can operate on opBNB with little to no modifications required.\n\nDevelopers can deploy the same applications on opBNB to benefit from Layer 2 scaling while maintaining the security features of the BNB Smart Chain. This offers the scalability and low costs associated with a rollup solution, combined with the robust security of a Layer 1 platform, providing an efficient environment for dApps to operate effectively.",
-                "CATEGORY": "ROLLUP"
-            },
-            {
-                "NAME": "Boba Network",
-                "WEBSITE_URL": "https://boba.network/",
-                "DESCRIPTION": "The Boba Network, functioning as a multichain Layer 2 optimistic rollup, seeks to harness the capabilities of rollup technology while facilitating interoperability between various blockchains and real-world applications. As a hybrid blockchain, it offers off-chain data and computational resources, making it suitable for the development of sophisticated applications geared towards widespread use. Fully compatible with EVM-based tools, the Boba Network has implemented multichain support for Ethereum, Avalanche, and BNB, providing a platform for rapid transactions and reduced fees.",
-                "CATEGORY": "ROLLUP"
-            }
-        ],
-        "PRIVACY_SOLUTIONS": null,
-        "CODE_REPOSITORIES": [
-            {
-                "URL": "https://github.com/binance-exchange"
-            },
-            {
-                "URL": "https://github.com/binance-exchange/binance-official-api-docs",
-                "OPEN_ISSUES": 2,
-                "CLOSED_ISSUES": 126,
-                "OPEN_PULL_REQUESTS": 0,
-                "CLOSED_PULL_REQUESTS": 126,
-                "CONTRIBUTORS": 0,
-                "FORKS": 2136,
-                "STARS": 3832,
-                "SUBSCRIBERS": 499,
-                "LAST_UPDATED_TS": 1690372964,
-                "LAST_PUSH_TS": 1682283934,
-                "CODE_SIZE_IN_BYTES": 1,
-                "IS_FORK": false,
-                "LANGUAGE": null,
-                "FORKED_ASSET_DATA": null
-            },
-            {
-                "URL": "https://github.com/binance-exchange/binance-api-node",
-                "OPEN_ISSUES": 1,
-                "CLOSED_ISSUES": 25,
-                "OPEN_PULL_REQUESTS": 0,
-                "CLOSED_PULL_REQUESTS": 25,
-                "CONTRIBUTORS": 102,
-                "FORKS": 176,
-                "STARS": 568,
-                "SUBSCRIBERS": 44,
-                "LAST_UPDATED_TS": 1690375176,
-                "LAST_PUSH_TS": 1686157630,
-                "CODE_SIZE_IN_BYTES": 739,
-                "IS_FORK": true,
-                "LANGUAGE": "JavaScript",
-                "FORKED_ASSET_DATA": null
-            },
-            {
-                "URL": "https://github.com/binance-exchange/go-binance",
-                "OPEN_ISSUES": 9,
-                "CLOSED_ISSUES": 10,
-                "OPEN_PULL_REQUESTS": 9,
-                "CLOSED_PULL_REQUESTS": 10,
-                "CONTRIBUTORS": 3,
-                "FORKS": 102,
-                "STARS": 214,
-                "SUBSCRIBERS": 22,
-                "LAST_UPDATED_TS": 1690375177,
-                "LAST_PUSH_TS": 1595796163,
-                "CODE_SIZE_IN_BYTES": 1599,
-                "IS_FORK": true,
-                "LANGUAGE": "Go",
-                "FORKED_ASSET_DATA": null
-            },
-            {
-                "URL": "https://github.com/binance-exchange/node-binance-api",
-                "OPEN_ISSUES": 18,
-                "CLOSED_ISSUES": 214,
-                "OPEN_PULL_REQUESTS": 0,
-                "CLOSED_PULL_REQUESTS": 109,
-                "CONTRIBUTORS": 82,
-                "FORKS": 150,
-                "STARS": 521,
-                "SUBSCRIBERS": 42,
-                "LAST_UPDATED_TS": 1690372966,
-                "LAST_PUSH_TS": 1682296247,
-                "CODE_SIZE_IN_BYTES": 1217,
-                "IS_FORK": true,
-                "LANGUAGE": "JavaScript",
-                "FORKED_ASSET_DATA": null
-            },
-            {
-                "URL": "https://github.com/binance-exchange/python-binance",
-                "OPEN_ISSUES": 4,
-                "CLOSED_ISSUES": 0,
-                "OPEN_PULL_REQUESTS": 4,
-                "CLOSED_PULL_REQUESTS": 0,
-                "CONTRIBUTORS": 21,
-                "FORKS": 182,
-                "STARS": 452,
-                "SUBSCRIBERS": 68,
-                "LAST_UPDATED_TS": 1690372967,
-                "LAST_PUSH_TS": 1531807102,
-                "CODE_SIZE_IN_BYTES": 347,
-                "IS_FORK": true,
-                "LANGUAGE": "Python",
-                "FORKED_ASSET_DATA": null
-            },
-            {
-                "URL": "https://github.com/binance-exchange/binance-java-api",
-                "OPEN_ISSUES": 154,
-                "CLOSED_ISSUES": 281,
-                "OPEN_PULL_REQUESTS": 41,
-                "CLOSED_PULL_REQUESTS": 97,
-                "CONTRIBUTORS": 43,
-                "FORKS": 640,
-                "STARS": 826,
-                "SUBSCRIBERS": 97,
-                "LAST_UPDATED_TS": 1690375177,
-                "LAST_PUSH_TS": 1686147366,
-                "CODE_SIZE_IN_BYTES": 440,
-                "IS_FORK": true,
-                "LANGUAGE": "Java",
-                "FORKED_ASSET_DATA": null
-            }
-        ],
-        "SUBREDDITS": [
-            {
-                "URL": "https://reddit.com/r/binance",
-                "MAKE_3RD_PARTY_REQUEST": true,
-                "NAME": "binance",
-                "CURRENT_ACTIVE_USERS": 43,
-                "AVERAGE_POSTS_PER_DAY": 0.22,
-                "AVERAGE_POSTS_PER_HOUR": 0.01,
-                "AVERAGE_COMMENTS_PER_DAY": 18.05,
-                "AVERAGE_COMMENTS_PER_HOUR": 0.75,
-                "SUBSCRIBERS": 904498,
-                "COMMUNITY_CREATED_AT": 1498873536,
-                "LAST_UPDATED_TS": 1718367530
-            }
-        ],
-        "TWITTER_ACCOUNTS": [
-            {
-                "URL": "https://twitter.com/binance",
-                "MAKE_3RD_PARTY_REQUEST": true,
-                "NAME": "Binance",
-                "USERNAME": "binance",
-                "VERIFIED": true,
-                "VERIFIED_TYPE": "business",
-                "FOLLOWING": 0,
-                "FOLLOWERS": 10858205,
-                "FAVOURITES": 0,
-                "LISTS": 29826,
-                "STATUSES": 29219,
-                "ACCOUNT_CREATED_AT": 1498120695,
-                "LAST_UPDATED_TS": 1698775503
-            }
-        ],
-        "DISCORD_SERVERS": [
-            {
-                "URL": "https://discord.com/invite/jE4wt8g2H2",
-                "MAKE_3RD_PARTY_REQUEST": false,
-                "NAME": "Binance",
-                "TOTAL_MEMBERS": 91925,
-                "CURRENT_ACTIVE_USERS": 7886,
-                "PREMIUM_SUBSCRIBERS": 25,
-                "LAST_UPDATED_TS": 1691075162
-            }
-        ],
-        "TELEGRAM_GROUPS": [
-            {
-                "URL": "https://t.me/BNBchaincommunity",
-                "MAKE_3RD_PARTY_REQUEST": false,
-                "NAME": "BNB Chain Community",
-                "USERNAME": "BNBchaincommunity",
-                "MEMBERS": 118577,
-                "LAST_UPDATED_TS": 1695279793
-            }
-        ],
-        "OTHER_SOCIAL_NETWORKS": [
-            {
-                "NAME": "FACEBOOK",
-                "URL": "https://www.facebook.com/binance"
-            },
-            {
-                "NAME": "INSTAGRAM",
-                "URL": "https://www.instagram.com/Binance/"
-            },
-            {
-                "NAME": "YOUTUBE",
-                "URL": "https://www.youtube.com/binanceyoutube"
-            },
-            {
-                "NAME": "LINKEDIN",
-                "URL": "https://www.linkedin.com/company/binance"
-            }
-        ],
-        "HELD_TOKEN_SALE": true,
-        "TOKEN_SALES": [
-            {
-                "TOKEN_SALE_TYPE": "ICO",
-                "TOKEN_SALE_DATE_START": 1498435200,
-                "TOKEN_SALE_DATE_END": 1499040000,
-                "TOKEN_SALE_DESCRIPTION": "Binance held its ICO on the 26th of June, with the ICO supply representing half of the total 200M token supply, thus selling 100,000,000 BNB tokens and raising $15,000,000. The ICO ended on the 3rd of July. The issue price was 1 ETH for 2,700 BNB or 1 BTC for 20,000 BNB (around 0.11 USD back then).",
-                "TOKEN_SALE_TEAM_MEMBERS": [
-                    {
-                        "TYPE": "EMPLOYEE",
-                        "FULL_NAME": "Changpeng Zhao"
-                    },
-                    {
-                        "TYPE": "EMPLOYEE",
-                        "FULL_NAME": "Roger Wang"
-                    },
-                    {
-                        "TYPE": "EMPLOYEE",
-                        "FULL_NAME": "James Hofbauer"
-                    },
-                    {
-                        "TYPE": "EMPLOYEE",
-                        "FULL_NAME": "Paul Jankunas"
-                    }
-                ],
-                "TOKEN_SALE_WEBSITE_URL": "https://www.binance.com/",
-                "TOKEN_SALE_SUPPLY": 100000000,
-                "TOKEN_SUPPLY_POST_SALE": "DECREASES",
-                "TOKEN_SALE_PAYMENT_METHOD_TYPE": "CRYPTO",
-                "TOKEN_SALE_START_PRICE": 0.15,
-                "TOKEN_SALE_START_PRICE_CURRENCY": "USD",
-                "TOKEN_SALE_FUNDS_RAISED": [
-                    {
-                        "CURRENCY": "BNB",
-                        "TOTAL_VALUE": 100000000,
-                        "DESCRIPTION": "Out of the total 200 million BNB token supply, 100 million BNB coins were sold during the ICO at a Binance ICO price of 2,700 BNB for 1 ETH"
-                    }
-                ],
-                "TOKEN_SALE_FUNDS_RAISED_USD": 15000000,
-                "TOKEN_SALE_INVESTORS_SPLIT": [
-                    {
-                        "CATEGORY": "PUBLIC",
-                        "TOTAL_TOKENS": 50000000
-                    }
-                ],
-                "TOKEN_SALE_RESERVE_SPLIT": [
-                    {
-                        "CATEGORY": "TEAM",
-                        "TOTAL_TOKENS": 80000000
-                    },
-                    {
-                        "CATEGORY": "TEAM",
-                        "TOTAL_TOKENS": 20000000
-                    }
-                ],
-                "TOKEN_SALE_JURISDICTIONS": [
-                    {
-                        "NAME": "KY"
-                    }
-                ],
-                "TOKEN_SALE_LEGAL_FORMS": [
-                    {
-                        "NAME": "CORPORATION"
-                    }
-                ]
-            }
-        ],
-        "HELD_EQUITY_SALE": false,
-        "WEBSITE_URL": "https://www.binance.com/",
-        "BLOG_URL": "https://www.binance.com/en/blog",
-        "WHITE_PAPER_URL": "https://resources.cryptocompare.com/asset-management/8/1661250302524.pdf",
-        "OTHER_DOCUMENT_URLS": [
-            {
-                "TYPE": "TOKEN_ECONOMICS",
-                "VERSION": 1,
-                "URL": "https://resources.cryptocompare.com/asset-management/8/1664528054778.pdf",
-                "COMMENT": "\"Tokenomics Deep Dive\""
-            },
-            {
-                "TYPE": "AUDIT",
-                "URL": "https://resources.cryptocompare.com/asset-management/8/1702984453160.pdf",
-                "COMMENT": "Certik audit"
-            },
-            {
-                "TYPE": "AUDIT",
-                "URL": "https://resources.cryptocompare.com/asset-management/8/1702984652594.pdf",
-                "COMMENT": "Hacken audit"
-            },
-            {
-                "TYPE": "ROADMAP",
-                "URL": "https://resources.cryptocompare.com/asset-management/8/1703172228924.pdf",
-                "COMMENT": "Uploaded 21/12/2023"
-            },
-            {
-                "TYPE": "ROADMAP",
-                "URL": "https://resources.cryptocompare.com/asset-management/8/1703172721792.pdf",
-                "COMMENT": "BNB Chain Fusion Roadmap"
-            },
-            {
-                "TYPE": "ESG_REPORT",
-                "URL": "https://resources.cryptocompare.com/asset-management/8/1703174584333.pdf",
-                "COMMENT": "Governance of BSC"
-            },
-            {
-                "TYPE": "ESG_REPORT",
-                "URL": "https://resources.cryptocompare.com/asset-management/8/1703174851909.pdf",
-                "COMMENT": "Governance of BNB Beacon Chain"
-            }
-        ],
-        "EXPLORER_ADDRESSES": [
-            {
-                "URL": "https://explorer.bnbchain.org/"
-            },
-            {
-                "URL": "https://bscscan.com/"
-            },
-            {
-                "URL": "https://binance.mintscan.io/"
-            },
-            {
-                "URL": "https://explorer.bitquery.io/bsc"
-            },
-            {
-                "URL": "https://bsctrace.com/"
-            },
-            {
-                "URL": "https://www.oklink.com/bsc"
-            },
-            {
-                "URL": "https://getblock.io/explorers/bsc/"
-            },
-            {
-                "URL": "https://blockchair.com/bnb"
-            },
-            {
-                "URL": "https://bsc.tokenview.io/"
-            },
-            {
-                "URL": "https://bnb.dex.guru/"
-            }
-        ],
-        "ASSET_INDUSTRIES": [
-            {
-                "ASSET_INDUSTRY": "PLATFORM"
-            },
-            {
-                "ASSET_INDUSTRY": "BRIDGE_OPERATOR"
-            },
-            {
-                "ASSET_INDUSTRY": "PAYMENTS"
-            }
-        ],
-        "CONSENSUS_MECHANISMS": [
-            {
-                "NAME": "POS"
-            },
-            {
-                "NAME": "POA"
-            }
-        ],
-        "CONSENSUS_ALGORITHM_TYPES": [
-            {
-                "NAME": "Tendermint BFT",
-                "DESCRIPTION": "Binance Chain relies on the Tendermint BFT consensus and Delegated Proof of Stake (DPoS) with a dedicated application layer that runs upon it. As of today, the vote delegation feature is not open for users.\n\nBinance Chain is built on forks of Tendermint and Cosmos SDK. Specifically, Binance Chain is made on a revised edition of the Tendermint consensus while leveraging its P2P networking logic.\n\nAt any given state, the following information is being recorded:\n\n- Account and balances: the balance (i.e., the number of tokens) of each asset is composed of 3 different parts: locked, frozen, and available. It is recorded on all accounts.\n- Fees: it represents how much fees were paid in the previous block.\n- Token information: what are the tokens, along with their respective information (frozen supply, etc.)\n- Trading pairs: the list of all the trading pairs on Binance DEX.\n- Tick size and lot size: various information related to the trading engine of Binance DEX.\n- Governance information: information about the validators, and governance mechanism of the Binance Chain."
-            },
-            {
-                "NAME": "Proof-of-Staked Authority (PoSA)",
-                "DESCRIPTION": "The Binance Smart Chain combines several features from Delegated Proof-of-Stake (DPoS) and Proof-of-Authority (PoA) for consensus finding.\n\nAt its core, this consensus algorithm called Parlia is built on a network of 21 validators and delegators, who vote for validators.\n\nIn Binance Smart Chain, any party can attempt to become a validator, assuming they meet (1) the hardware requirement to run the network, (2) manage their own keys, (3) guarantee a high uptime and (4) own and stake at least 20,000 BNB. Similar to other DPoS blockchains, BSC validators receive network fees for validating blocks.\n\nAs of writing, the top 21 validators with the largest number of BNBs staked are selected every day and eligible to validate and produce blocks on the Binance Smart Chain. \n\nDelegators can vote for validators, allocating BNB to increase the validator\u2019s capacity to produce blocks, while receiving some of the rewards from network fees. This election is repeated every 24 hours. At the end of the period, the staking status decides what the top 21 highest nodes with votes to become the validator set for the next period are.\n\nIn addition, Binance Smart Chain introduces a slashing mechanism to prevent bad behaviour and other issues: e.g., prevent double signatures, penalties for being offline, etc.\n\nFor instance, BSC\u2019s slashing mechanism allows any individual to submit a \u201cslashing request\u201d on Binance Chain, assuming they are able to bring the proof that the malicious validator produced two block headers with the same height and parent block."
-            }
-        ],
-        "HASHING_ALGORITHM_TYPES": [
-            {
-                "NAME": "SHA_256"
-            }
-        ],
-        "PRICE_USD": 605.351392574596,
-        "PRICE_USD_SOURCE": "cadli",
-        "PRICE_USD_LAST_UPDATE_TS": 1718377245,
-        "MKT_CAP_PENALTY": 0,
-        "CIRCULATING_MKT_CAP_USD": 89340306138.43848,
-        "TOTAL_MKT_CAP_USD": 89340306138.43828,
-        "SPOT_MOVING_24_HOUR_QUOTE_VOLUME_TOP_TIER_DIRECT_USD": 5717.5208552127,
-        "SPOT_MOVING_24_HOUR_QUOTE_VOLUME_DIRECT_USD": 681557.540655213,
-        "SPOT_MOVING_24_HOUR_QUOTE_VOLUME_TOP_TIER_USD": 337790572.034656,
-        "SPOT_MOVING_24_HOUR_QUOTE_VOLUME_USD": 507742353.298955,
-        "SPOT_MOVING_24_HOUR_CHANGE_USD": 8.158248751072,
-        "SPOT_MOVING_24_HOUR_CHANGE_PERCENTAGE_USD": 1.3660988635667999,
-        "TOPLIST_BASE_RANK": {
-            "LAUNCH_DATE": 4601,
-            "CIRCULATING_MKT_CAP_USD": 4,
-            "TOTAL_MKT_CAP_USD": 4,
-            "SPOT_MOVING_24_HOUR_QUOTE_VOLUME_USD": 8,
-            "SPOT_MOVING_24_HOUR_CHANGE_PERCENTAGE_USD": 1037
-        },
-        "ASSET_DESCRIPTION": "## What is Binance Coin (BNB)?\nBinance Coin, known as BNB, is the cryptocurrency token that powers the BNB Chain ecosystem. Initially referred to as Binance Coin, it plays a central role in the functionalities of the Binance ecosystem, which includes the world's largest cryptocurrency exchange by volume. BNB operates as a utility token, allowing users to trade it like any other cryptocurrency. Furthermore, BNB is used in a variety of applications and use cases, extending its utility beyond mere trading. The Binance Chain, supporting BNB, uses a Byzantine Fault Tolerance (BFT) consensus mechanism for fast and efficient transaction processing.\n\nBinance is a cryptocurrency exchange platform. It offers a variety of services including cryptocurrency trading, digital asset management, and various blockchain-based offerings. Binance is known for supporting a wide array of cryptocurrencies for trading. The platform also provides features like crypto-to-crypto trading, staking, and an option for users to participate in initial coin offerings (ICOs) and token sales. The exchange is designed to cater to both beginner and experienced traders, offering a range of tools and functionalities tailored to different levels of expertise in the cryptocurrency market.\n\n## What is Binance Coin (BNB) used for?\nBNB was initially used mainly to pay for trading and transaction fees on the Binance cryptocurrency exchange. Over time, its utility has expanded significantly. Today, BNB is used in various applications within and outside the Binance ecosystem, showcasing its versatility as a utility token. The token's role in facilitating various transactions and activities within the Binance ecosystem has been a key factor in its widespread adoption and use. Additionally, the introduction of the Binance Smart Chain, a parallel blockchain, has enabled smart contract functionality and interoperability, further expanding BNB's use cases.\n\n## Who created Binance Coin (BNB)?\nBinance Coin (BNB) was founded in 2017 by Changpeng Zhao, a tech entrepreneur commonly referred to as CZ. He established BNB as the native utility token of the Binance exchange. The BNB Chain employs a unique combination of Proof of Stake (PoS) and Proof of Authority (PoA) consensus mechanisms to validate network transactions, contributing to the security and efficiency of the ecosystem.",
-        "ASSET_DESCRIPTION_SUMMARY": "Binance Coin (BNB) is a cryptocurrency token central to the Binance ecosystem, including the world's largest cryptocurrency exchange. Created by Changpeng Zhao in 2017, BNB has evolved from a tool for paying transaction fees on the Binance exchange to a widely used utility token in various applications. Its importance in the Binance ecosystem is underscored by its role in facilitating transactions and activities across the platform.",
-        "PROJECT_LEADERS": [
-            {
-                "LEADER_TYPE": "FOUNDER",
-                "FULL_NAME": "Changpeng Zhao"
-            },
-            {
-                "LEADER_TYPE": "FOUNDER",
-                "FULL_NAME": "Yi He"
-            }
-        ],
-        "ASSOCIATED_CONTACT_DETAILS": [
-            {
-                "CONTACT_MEDIUM": "EMAIL_ADDRESS"
-            }
-        ],
-        "SEO_TITLE": "Binance Coin (BNB)",
-        "SEO_DESCRIPTION": "Live Binance Coin price movements from all markets and BNB market cap, use our charts and see when there is an opportunity to buy or sell."
-    },
-    "Err": {}
-}

src/knowledge_bases/data/coin_data/pepe.json

@@ -1,158 +0,0 @@
-{
-    "Data": {
-        "ID": 3010,
-        "TYPE": "162",
-        "ID_LEGACY": 953245,
-        "ID_PARENT_ASSET": null,
-        "SYMBOL": "PEPE",
-        "URI": "pepe",
-        "ASSET_TYPE": "TOKEN",
-        "PARENT_ASSET_SYMBOL": null,
-        "CREATED_ON": 1683799924,
-        "UPDATED_ON": 1709050507,
-        "PUBLIC_NOTICE": null,
-        "NAME": "Pepe",
-        "LOGO_URL": "https://resources.cryptocompare.com/asset-management/3010/1709049889749.png",
-        "LAUNCH_DATE": 1681430400,
-        "PREVIOUS_ASSET_SYMBOLS": null,
-        "ASSET_ALTERNATIVE_IDS": [
-            {
-                "NAME": "CG",
-                "ID": "pepe"
-            },
-            {
-                "NAME": "CMC",
-                "ID": 24478
-            }
-        ],
-        "ASSET_DESCRIPTION_SNIPPET": "$PEPE is an ERC-20 token, known as the most memeable memecoin, embracing the internet meme Pepe's popularity. Unlike other meme-based cryptocurrencies, it launched stealthily without presale or taxes, aiming to be a coin for the people.",
-        "SUPPORTED_PLATFORMS": [
-            {
-                "BLOCKCHAIN": "ETH",
-                "TOKEN_STANDARD": "ERC20",
-                "BRIDGE_OPERATOR": "NATIVE_ASSET",
-                "IS_ASSET_ISSUER": true,
-                "EXPLORER_URL": "https://etherscan.io/token/0x6982508145454Ce325dDbE47a25d4ec3d2311933",
-                "SMART_CONTRACT_ADDRESS": "0x6982508145454Ce325dDbE47a25d4ec3d2311933",
-                "LAUNCH_DATE": 1681430400,
-                "TRADING_AS": "PEPE",
-                "DECIMALS": 18
-            }
-        ],
-        "ASSET_CUSTODIANS": null,
-        "CONTROLLED_ADDRESSES": null,
-        "ASSET_SECURITY_METRICS": [
-            {
-                "NAME": "CERTIK",
-                "OVERALL_SCORE": 86.87,
-                "OVERALL_RANK": 211,
-                "UPDATED_AT": 1718409600
-            }
-        ],
-        "SUPPLY_MAX": -1,
-        "SUPPLY_ISSUED": 420689899999995,
-        "SUPPLY_TOTAL": 420689899999995,
-        "SUPPLY_CIRCULATING": 420689899999995,
-        "SUPPLY_FUTURE": -1,
-        "SUPPLY_LOCKED": 0,
-        "SUPPLY_BURNT": 0,
-        "SUPPLY_STAKED": null,
-        "BURN_ADDRESSES": null,
-        "LOCKED_ADDRESSES": null,
-        "CODE_REPOSITORIES": null,
-        "SUBREDDITS": null,
-        "TWITTER_ACCOUNTS": [
-            {
-                "URL": "https://twitter.com/pepecoineth",
-                "NAME": "Pepe",
-                "USERNAME": "pepecoineth",
-                "FOLLOWING": 1,
-                "FOLLOWERS": 413507,
-                "FAVOURITES": 254,
-                "LISTS": 458,
-                "STATUSES": 407,
-                "ACCOUNT_CREATED_AT": 1680625027,
-                "LAST_UPDATED_TS": 1689344201
-            }
-        ],
-        "DISCORD_SERVERS": [
-            {
-                "URL": "https://discord.com/invite/pepe-palace",
-                "MAKE_3RD_PARTY_REQUEST": false,
-                "NAME": "\ud83d\udc38Pepe Palace\ud83d\udc38",
-                "TOTAL_MEMBERS": 16821,
-                "CURRENT_ACTIVE_USERS": 1262,
-                "PREMIUM_SUBSCRIBERS": 14,
-                "LAST_UPDATED_TS": 1688912388
-            }
-        ],
-        "TELEGRAM_GROUPS": [
-            {
-                "URL": "https://t.me/pepecoineth"
-            }
-        ],
-        "OTHER_SOCIAL_NETWORKS": null,
-        "HELD_TOKEN_SALE": false,
-        "HELD_EQUITY_SALE": false,
-        "WEBSITE_URL": "https://www.pepe.vip/",
-        "BLOG_URL": null,
-        "WHITE_PAPER_URL": null,
-        "OTHER_DOCUMENT_URLS": [
-            {
-                "TYPE": "ROADMAP",
-                "URL": "https://resources.cryptocompare.com/asset-management/3010/1683800472809.png"
-            },
-            {
-                "TYPE": "TOKEN_ECONOMICS",
-                "URL": "https://resources.cryptocompare.com/asset-management/3010/1683800517243.png"
-            },
-            {
-                "TYPE": "AUDIT",
-                "URL": "https://resources.cryptocompare.com/asset-management/3010/1699438872469.pdf",
-                "COMMENT": "Pepe-Token-Audit-Report 2023-08-21"
-            },
-            {
-                "TYPE": "AUDIT",
-                "URL": "https://resources.cryptocompare.com/asset-management/3010/1709050208378.pdf",
-                "COMMENT": "Certik audit"
-            }
-        ],
-        "ASSET_INDUSTRIES": [
-            {
-                "ASSET_INDUSTRY": "PLATFORM"
-            },
-            {
-                "ASSET_INDUSTRY": "PAYMENT"
-            },
-            {
-                "ASSET_INDUSTRY": "BRIDGE_OPERATOR"
-            }
-        ],
-        "PRICE_USD": 1.23046108057731e-05,
-        "PRICE_USD_SOURCE": "cadli",
-        "PRICE_USD_LAST_UPDATE_TS": 1718570876,
-        "MKT_CAP_PENALTY": null,
-        "CIRCULATING_MKT_CAP_USD": 5176425489.419543,
-        "TOTAL_MKT_CAP_USD": 5176425489.419543,
-        "SPOT_MOVING_24_HOUR_QUOTE_VOLUME_TOP_TIER_DIRECT_USD": 4732463.87575718,
-        "SPOT_MOVING_24_HOUR_QUOTE_VOLUME_DIRECT_USD": 15781091.0909223,
-        "SPOT_MOVING_24_HOUR_QUOTE_VOLUME_TOP_TIER_USD": 335538908.468033,
-        "SPOT_MOVING_24_HOUR_QUOTE_VOLUME_USD": 425993603.001758,
-        "SPOT_MOVING_24_HOUR_CHANGE_USD": 4.714449013124e-07,
-        "SPOT_MOVING_24_HOUR_CHANGE_PERCENTAGE_USD": 3.9840977902175903,
-        "TOPLIST_BASE_RANK": {
-            "LAUNCH_DATE": 10706,
-            "CIRCULATING_MKT_CAP_USD": 23,
-            "TOTAL_MKT_CAP_USD": 38,
-            "SPOT_MOVING_24_HOUR_QUOTE_VOLUME_USD": 7,
-            "SPOT_MOVING_24_HOUR_CHANGE_PERCENTAGE_USD": 614
-        },
-        "ASSET_DESCRIPTION": "## What is Pepe (PEPE)?\n\nPepe (PEPE) is a digital currency positioned as the next step in the evolution of meme coins, leveraging the memetic power of Pepe, a recognizable internet meme. Launched without any presale or taxes, PEPE aims to be a coin for the people, with its liquidity provider tokens burned and contract renounced.\n\n## Who created Pepe?\n\nThe specific creator of Pepe (PEPE) is not mentioned, but it is clarified that it is not associated with Matt Furie or his creation Pepe the Frog. Instead, it pays homage to the meme character.\n\n## What is $PEPE used for?\n\nThe PEPE token serves as a meme coin with no intrinsic value or financial return expectation. It follows a roadmap with phases like getting listed on popular crypto platforms, community partnerships, launching Pepe Times newsletter, token-gated Discord group, and plans for Pepe merchandise and academy. PEPE is intended to be entirely useless and solely for entertainment purposes.",
-        "ASSET_DESCRIPTION_SUMMARY": "Pepe (PEPE) is a digital currency that positions itself as the next step in meme coin evolution, leveraging the memetic power of the internet meme Pepe. Launched without presale or taxes, it aims to be a coin for the people, with liquidity provider tokens burned and contract renounced. The token has a roadmap with different phases, including community partnerships, launching a digital newsletter, and getting listed on exchanges. However, it's important to note that PEPE is not associated with Pepe the Frog and is intended to be entirely useless and for entertainment purposes only.",
-        "PROJECT_LEADERS": null,
-        "ASSOCIATED_CONTACT_DETAILS": null,
-        "SEO_TITLE": "Pepe (PEPE)",
-        "SEO_DESCRIPTION": "Live Pepe price movements from all markets and PEPE market cap, use our charts and see when there is an opportunity to buy or sell."
-    },
-    "Err": {}
-}

src/knowledge_bases/json.py

@@ -0,0 +1,61 @@
+import json
+from fs_s3fs import S3FS
+from src.libs.logger import logger
+from src.libs.s3fs import get_s3_credentials
+from phi.vectordb.pgvector import PgVector2
+from phi.knowledge.json import JSONKnowledgeBase
+from src.databases.postgres import sqlalchemy_engine
+
+class JSONKnowledgeBaseExtended(JSONKnowledgeBase):
+    s3fs: S3FS = None  # Explicitly declare the s3fs attribute
+
+    def __init__(
+        self, 
+        s3_bucket_name, 
+        vector_db, 
+        s3_access_key_id,
+        s3_secret_access_key,
+        s3_endpoint_url,
+        s3_region,
+    ):
+        super().__init__(path=s3_bucket_name, vector_db=vector_db, bucket_name = s3_bucket_name)
+
+        # Initialize the S3 filesystem
+        self.s3fs = S3FS(
+            bucket_name=s3_bucket_name,
+            aws_access_key_id=s3_access_key_id,
+            aws_secret_access_key=s3_secret_access_key,
+            endpoint_url = s3_endpoint_url,
+            region = s3_region,
+        )
+
+    def load_knowledge_base(self, recreate: bool = False):
+        json_knowledge_base.load(recreate=recreate)
+
+    def store_json_data_in_s3(self, json_data, file_path):
+        if file_path[0] == '/':
+            file_path = f"/json-data/{file_path[1:]}"
+        else:
+            file_path = f"/json-data/{file_path}"
+
+        logger.info(f"Storing JSON data in S3 bucket: {self.s3fs._bucket_name} at path: {file_path}")
+        
+        # Open the file in write mode and write the JSON data
+        self.s3fs.open(path = f"/{file_path}", mode = 'w').write(json.dumps(json_data, indent=2))
+        return True
+
+
+# S3 credentials
+_s3_credendtials = get_s3_credentials()
+_json_knowledge_base_arguments = {
+    "vector_db": PgVector2(
+        collection="json_documents",
+        db_engine=sqlalchemy_engine
+    ),
+    **_s3_credendtials
+}
+
+# Initialize the extended JSONKnowledgeBase with the S3 bucket name and S3 credentials
+json_knowledge_base = JSONKnowledgeBaseExtended(
+    **_json_knowledge_base_arguments
+)

src/knowledge_bases/json_knowledge_base.py

@@ -1,12 +0,0 @@
-from phi.knowledge.json import JSONKnowledgeBase
-from phi.vectordb.pgvector import PgVector2
-from src.databases.postgres import sqlalchemy_engine
-
-json_knowledge_base = JSONKnowledgeBase(
-    path="data/json",
-    # Table name: ai.json_documents
-    vector_db=PgVector2(
-        collection="json_documents",
-        db_engine=sqlalchemy_engine
-    ),
-)

src/knowledge_bases/pipeline.py

@@ -0,0 +1,45 @@
+from chainlit import make_async
+from src.data_sources.cryptocompare import CryptoCompare
+from src.data_sources.coin_gecko import CoinGecko
+from src.knowledge_bases.json import json_knowledge_base
+
+coin_gecko = CoinGecko()
+crypto_compare = CryptoCompare()
+
+store_json_data_in_s3 = make_async(json_knowledge_base.store_json_data_in_s3)
+
+async def fetch_and_load_crypto_compare_json_data():
+    get_all_coins = make_async(crypto_compare.get_all_coins)
+    get_news_categories = make_async(crypto_compare.get_news_categories)
+    get_top_tier_exchanges_list = make_async(crypto_compare.get_top_tier_exchanges_list)
+    get_all_wallets_general_info = make_async(crypto_compare.get_all_wallets_general_info)
+    get_all_exchanges_general_info = make_async(crypto_compare.get_all_exchanges_general_info)
+
+    all_coins = await get_all_coins()
+    news_categories = await get_news_categories()
+    top_tier_exchanges_list = await get_top_tier_exchanges_list()
+    all_wallets_general_info = await get_all_wallets_general_info()
+    all_exchanges_general_info = await get_all_exchanges_general_info()
+
+    await store_json_data_in_s3(all_coins, "crypto_compare_all_coins.json")
+    await store_json_data_in_s3(news_categories, "crypto_compare_news_categories.json")
+    await store_json_data_in_s3(top_tier_exchanges_list, "crypto_compare_top_tier_exchanges_list.json")
+    await store_json_data_in_s3(all_wallets_general_info, "crypto_compare_all_wallets_general_info.json")
+    await store_json_data_in_s3(all_exchanges_general_info, "crypto_compare_all_exchanges_general_info.json")
+
+async def fetch_and_load_coin_gecko_json_data():
+    get_all_coins = make_async(coin_gecko.get_exchanges_list)
+
+    all_exchange_data = await get_all_coins()
+
+    await store_json_data_in_s3(all_exchange_data, "coin_gecko_all_exchange_data")
+
+async def load_json_data_into_knowledge_base():
+    load_knowledge_base = make_async(json_knowledge_base.load_knowledge_base)
+
+    await load_knowledge_base(recreate=True)
+
+async def main():
+    await fetch_and_load_coin_gecko_json_data()
+    await fetch_and_load_crypto_compare_json_data()
+    await load_json_data_into_knowledge_base()

src/libs/logger.py

@@ -7,7 +7,7 @@ load_dotenv()
 
 lf.configure(
     token=os.getenv('LOGFIRE_TOKEN'),
-    pydantic_plugin=lf.PydanticPlugin(record='all'),
+    # pydantic_plugin=lf.PydanticPlugin(record='all'),
     console=lf.ConsoleOptions(min_log_level= os.getenv('LOG_LEVEL'))
 )
 lf.instrument_redis()

src/libs/s3fs.py

@@ -0,0 +1,26 @@
+import os
+import s3fs
+from fs_s3fs import S3FS
+from dotenv import load_dotenv
+
+load_dotenv()
+
+exchanges_data_s3fs = S3FS(
+    bucket_name = 'exchanges-data',
+    aws_access_key_id = os.getenv('DO_SPACES_ACCESS_KEY'),
+    aws_secret_access_key = os.getenv('DO_SPACES_SECRET_KEY'),
+    endpoint_url = os.getenv('DO_SPACES_ENDPOINT'),
+    region = os.getenv('DO_SPACES_REGION'),
+)
+
+# AWS credentials
+s3_credentials = {
+    "s3_access_key_id": os.getenv("DO_SPACES_ACCESS_KEY"),
+    "s3_secret_access_key": os.getenv("DO_SPACES_SECRET_KEY"),
+    "s3_endpoint_url": os.getenv("DO_SPACES_ENDPOINT_URL"),
+    "s3_bucket_name": os.getenv("DO_SPACES_BUCKET_NAME"),
+    "s3_region": os.getenv("DO_SPACES_REGION_NAME"),
+}
+
+def get_s3_credentials():
+    return s3_credentials