updating scripts to be compatible with a separate roi analysis

scripts/get_mech_info.py

@@ -64,6 +64,42 @@ def fetch_block_number(timestamp_from: int, timestamp_to: int) -> dict:
     return blocks[0]
 
 
+def get_mech_info_2024() -> dict[str, Any]:
+    """Query the subgraph to get the 2024 information from mech."""
+
+    date = "2024-01-01"
+    datetime_jan_2024 = datetime.strptime(date, "%Y-%m-%d")
+    timestamp_jan_2024 = int(datetime_jan_2024.timestamp())
+    margin = timedelta(seconds=5)
+    timestamp_jan_2024_plus_margin = int((datetime_jan_2024 + margin).timestamp())
+
+    jan_block_number = fetch_block_number(
+        timestamp_jan_2024, timestamp_jan_2024_plus_margin
+    )
+    # expecting only one block
+    jan_block_number = jan_block_number.get("number", "")
+    if jan_block_number.isdigit():
+        jan_block_number = int(jan_block_number)
+
+    if jan_block_number == "":
+        raise ValueError(
+            "Could not find a valid block number for the first of January 2024"
+        )
+    MECH_TO_INFO = {
+        # this block number is when the creator had its first tx ever, and after this mech's creation
+        "0xff82123dfb52ab75c417195c5fdb87630145ae81": (
+            "old_mech_abi.json",
+            jan_block_number,
+        ),
+        # this block number is when this mech was created
+        "0x77af31de935740567cf4ff1986d04b2c964a786a": (
+            "new_mech_abi.json",
+            jan_block_number,
+        ),
+    }
+    return MECH_TO_INFO
+
+
 def get_mech_info_last_60_days() -> dict[str, Any]:
     """Query the subgraph to get the last 60 days of information from mech."""
 

scripts/profitability.py

@@ -22,14 +22,14 @@ import requests
 import datetime
 import pandas as pd
 from collections import defaultdict
-from typing import Any, Union
+from typing import Any
 from string import Template
 from enum import Enum
 from tqdm import tqdm
 import numpy as np
 from pathlib import Path
 from get_mech_info import DATETIME_60_DAYS_AGO
-from utils import SUBGRAPH_API_KEY
+from utils import SUBGRAPH_API_KEY, wei_to_unit, convert_hex_to_int
 
 IRRELEVANT_TOOLS = [
     "openai-text-davinci-002",
@@ -45,14 +45,12 @@ IRRELEVANT_TOOLS = [
 ]
 QUERY_BATCH_SIZE = 1000
 DUST_THRESHOLD = 10000000000000
-INVALID_ANSWER_HEX = (
-    "0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"
-)
 INVALID_ANSWER = -1
 FPMM_CREATOR = "0x89c5cc945dd550bcffb72fe42bff002429f46fec"
 DEFAULT_FROM_DATE = "1970-01-01T00:00:00"
 DEFAULT_TO_DATE = "2038-01-19T03:14:07"
 DEFAULT_FROM_TIMESTAMP = 0
+DEFAULT_60_DAYS_AGO_TIMESTAMP = (DATETIME_60_DAYS_AGO).timestamp()
 DEFAULT_TO_TIMESTAMP = 2147483647
 WXDAI_CONTRACT_ADDRESS = "0xe91D153E0b41518A2Ce8Dd3D7944Fa863463a97d"
 DEFAULT_MECH_FEE = 0.01
@@ -251,7 +249,6 @@ def _query_omen_xdai_subgraph(
     fpmm_to_timestamp: float,
 ) -> dict[str, Any]:
     """Query the subgraph."""
-    # url = "https://api.thegraph.com/subgraphs/name/protofire/omen-xdai"
     OMEN_SUBGRAPH_URL = Template(
         """https://gateway-arbitrum.network.thegraph.com/api/${subgraph_api_key}/subgraphs/id/9fUVQpFwzpdWS9bq5WkAnmKbNNcoBwatMR4yZq81pbbz"""
     )
@@ -301,7 +298,6 @@ def _query_omen_xdai_subgraph(
 
 def _query_conditional_tokens_gc_subgraph(creator: str) -> dict[str, Any]:
     """Query the subgraph."""
-    # url = "https://api.thegraph.com/subgraphs/name/gnosis/conditional-tokens-gc"
     SUBGRAPH_URL = Template(
         """https://gateway-arbitrum.network.thegraph.com/api/${subgraph_api_key}/subgraphs/id/7s9rGBffUTL8kDZuxvvpuc46v44iuDarbrADBFw5uVp2"""
     )
@@ -338,22 +334,6 @@ def _query_conditional_tokens_gc_subgraph(creator: str) -> dict[str, Any]:
     return all_results
 
 
-def convert_hex_to_int(x: Union[str, float]) -> Union[int, float]:
-    """Convert hex to int"""
-    if isinstance(x, float):
-        return np.nan
-    elif isinstance(x, str):
-        if x == INVALID_ANSWER_HEX:
-            return -1
-        else:
-            return int(x, 16)
-
-
-def wei_to_unit(wei: int) -> float:
-    """Converts wei to currency unit."""
-    return wei / 10**18
-
-
 def _is_redeemed(user_json: dict[str, Any], fpmmTrade: dict[str, Any]) -> bool:
     """Returns whether the user has redeemed the position."""
     user_positions = user_json["data"]["user"]["userPositions"]
@@ -404,12 +384,17 @@ def create_fpmmTrades(rpc: str, from_timestamp: float = DEFAULT_FROM_TIMESTAMP):
     return df
 
 
-def prepare_profitalibity_data(rpc: str):
+def prepare_profitalibity_data(
+    rpc: str,
+    tools_filename: str = "tools.parquet",
+    trades_filename: str = "fpmmTrades.parquet",
+    from_timestamp: float = DEFAULT_60_DAYS_AGO_TIMESTAMP,
+):
     """Prepare data for profitalibity analysis."""
 
     # Check if tools.parquet is in the same directory
     try:
-        tools = pd.read_parquet(DATA_DIR / "tools.parquet")
+        tools = pd.read_parquet(DATA_DIR / tools_filename)
 
         # make sure creator_address is in the columns
         assert "trader_address" in tools.columns, "trader_address column not found"
@@ -420,20 +405,18 @@ def prepare_profitalibity_data(rpc: str):
         # drop duplicates
         tools.drop_duplicates(inplace=True)
 
-        print("tools.parquet loaded")
+        print(f"{tools_filename} loaded")
     except FileNotFoundError:
         print("tools.parquet not found. Please run tools.py first.")
         return
 
     # Check if fpmmTrades.parquet is in the same directory
     try:
-        fpmmTrades = pd.read_parquet(DATA_DIR / "fpmmTrades.parquet")
-        print("fpmmTrades.parquet loaded")
+        fpmmTrades = pd.read_parquet(DATA_DIR / trades_filename)
+        print(f"{trades_filename} loaded")
     except FileNotFoundError:
         print("fpmmTrades.parquet not found. Creating fpmmTrades.parquet...")
-        # Prepare the same time window as used for the tools
-        timestamp_60_days_ago = (DATETIME_60_DAYS_AGO).timestamp()
-        fpmmTrades = create_fpmmTrades(rpc, from_timestamp=timestamp_60_days_ago)
+        fpmmTrades = create_fpmmTrades(rpc, from_timestamp=from_timestamp)
         fpmmTrades.to_parquet(DATA_DIR / "fpmmTrades.parquet", index=False)
 
     # make sure trader_address is in the columns
@@ -621,19 +604,28 @@ def summary_analyse(df):
     return summary_df
 
 
-def run_profitability_analysis(rpc):
+def run_profitability_analysis(
+    rpc: str,
+    tools_filename: str = "tools.parquet",
+    trades_filename: str = "fpmmTrades.parquet",
+    from_timestamp: float = DEFAULT_60_DAYS_AGO_TIMESTAMP,
+):
     """Create all trades analysis."""
 
     # load dfs from data folder for analysis
-    print("Preparing data...")
-    fpmmTrades, tools = prepare_profitalibity_data(rpc)
+    print(f"Preparing data with {tools_filename} and {trades_filename}")
+    fpmmTrades, tools = prepare_profitalibity_data(
+        rpc, tools_filename, trades_filename, from_timestamp
+    )
     tools["trader_address"] = tools["trader_address"].str.lower()
 
     # all trades profitability df
     print("Analysing trades...")
     all_trades_df = analyse_all_traders(fpmmTrades, tools)
 
-    # filter invalid markets. Condition: "is_invalid" are True
+    # filter invalid markets. Condition: "is_invalid" is True
+    # TODO fix this mask
+    print(all_trades_df.head())
     invalid_market_mask = all_trades_df["is_invalid"]
     all_trades_df = all_trades_df[~invalid_market_mask]
 

scripts/pull_data.py

@@ -19,7 +19,7 @@ from tools import (
 )
 from profitability import run_profitability_analysis
 from utils import get_question, current_answer
-
+from get_mech_info import get_mech_info_last_60_days
 import gc
 
 logging.basicConfig(level=logging.INFO)
@@ -46,35 +46,8 @@ def parallelize_timestamp_conversion(df: pd.DataFrame, function: callable) -> li
     return results
 
 
-def weekly_analysis():
-    """Run weekly analysis for the FPMMS project."""
-    rpc = "https://lb.nodies.app/v1/406d8dcc043f4cb3959ed7d6673d311a"
+def updating_timestamps(rpc: str):
     web3 = Web3(Web3.HTTPProvider(rpc))
-
-    # Run markets ETL
-    logging.info("Running markets ETL")
-    mkt_etl(MARKETS_FILENAME)
-    logging.info("Markets ETL completed")
-
-    # Run tools ETL
-    logging.info("Running tools ETL")
-
-    # This etl is saving already the tools parquet file
-    tools_etl(
-        rpcs=[rpc],
-        filename=TOOLS_FILENAME,
-    )
-    logging.info("Tools ETL completed")
-
-    # Run profitability analysis
-    logging.info("Running profitability analysis")
-    if os.path.exists(DATA_DIR / "fpmmTrades.parquet"):
-        os.remove(DATA_DIR / "fpmmTrades.parquet")
-    run_profitability_analysis(
-        rpc=rpc,
-    )
-    logging.info("Profitability analysis completed")
-
     # Get currentAnswer from FPMMS
     fpmms = pd.read_parquet(DATA_DIR / MARKETS_FILENAME)
     tools = pd.read_parquet(DATA_DIR / TOOLS_FILENAME)
@@ -134,8 +107,42 @@ def weekly_analysis():
     del t_map
     gc.collect()
 
+
+def weekly_analysis():
+    """Run weekly analysis for the FPMMS project."""
+    rpc = "https://lb.nodies.app/v1/406d8dcc043f4cb3959ed7d6673d311a"
+
+    # Run markets ETL
+    logging.info("Running markets ETL")
+    mkt_etl(MARKETS_FILENAME)
+    logging.info("Markets ETL completed")
+
+    # Run tools ETL
+    logging.info("Running tools ETL")
+
+    # This etl is saving already the tools parquet file
+    tools_etl(
+        rpcs=[rpc],
+        mech_info=get_mech_info_last_60_days(),
+        filename=TOOLS_FILENAME,
+    )
+    logging.info("Tools ETL completed")
+
+    # Run profitability analysis
+    logging.info("Running profitability analysis")
+    if os.path.exists(DATA_DIR / "fpmmTrades.parquet"):
+        os.remove(DATA_DIR / "fpmmTrades.parquet")
+    run_profitability_analysis(
+        rpc=rpc,
+    )
+    logging.info("Profitability analysis completed")
+
+    updating_timestamps(rpc)
+
     logging.info("Weekly analysis files generated and saved")
 
 
 if __name__ == "__main__":
-    weekly_analysis()
+    # weekly_analysis()
+    rpc = "https://lb.nodies.app/v1/406d8dcc043f4cb3959ed7d6673d311a"
+    updating_timestamps(rpc)

scripts/roi_analysis.py

@@ -0,0 +1,130 @@
+import logging
+import os
+import pickle
+from web3 import Web3
+import pandas as pd
+from functools import partial
+from datetime import datetime
+from markets import (
+    etl as mkt_etl,
+    DEFAULT_FILENAME as MARKETS_FILENAME,
+)
+
+TOOLS_FILENAME = "tools_2024.parquet"
+from tools import (
+    etl as tools_etl,
+    update_tools_accuracy,
+)
+from pull_data import (
+    DATA_DIR,
+    parallelize_timestamp_conversion,
+    block_number_to_timestamp,
+)
+from profitability import run_profitability_analysis
+from get_mech_info import get_mech_info_2024
+from utils import get_question, current_answer
+import gc
+
+logging.basicConfig(level=logging.INFO)
+
+
+def roi_analysis():
+    """Run ROI analysis for the trades done in 2024."""
+    rpc = "https://lb.nodies.app/v1/406d8dcc043f4cb3959ed7d6673d311a"
+    web3 = Web3(Web3.HTTPProvider(rpc))
+
+    # Run markets ETL
+    logging.info("Running markets ETL")
+    mkt_etl(MARKETS_FILENAME)
+    logging.info("Markets ETL completed")
+
+    # Run tools ETL
+    logging.info("Running tools ETL")
+
+    # This etl is saving already the tools parquet file
+    tools_etl(
+        rpcs=[rpc],
+        mech_info=get_mech_info_2024(),
+        filename=TOOLS_FILENAME,
+    )
+    logging.info("Tools ETL completed")
+
+    # Run profitability analysis
+    if os.path.exists(DATA_DIR / "fpmmTrades.parquet"):
+        os.remove(DATA_DIR / "fpmmTrades.parquet")
+    logging.info("Running profitability analysis")
+    date = "2024-01-01"
+    datetime_jan_2024 = datetime.strptime(date, "%Y-%m-%d")
+    timestamp_jan_2024 = int(datetime_jan_2024.timestamp())
+    run_profitability_analysis(
+        rpc=rpc,
+        tools_filename=TOOLS_FILENAME,
+        trades_filename="fpmmTrades.parquet",
+        from_timestamp=timestamp_jan_2024,
+    )
+    logging.info("Profitability analysis completed")
+
+    # Get currentAnswer from FPMMS
+    fpmms = pd.read_parquet(DATA_DIR / MARKETS_FILENAME)
+    tools = pd.read_parquet(DATA_DIR / TOOLS_FILENAME)
+
+    # Get the question from the tools
+    logging.info("Getting the question and current answer for the tools")
+    tools["title"] = tools["prompt_request"].apply(lambda x: get_question(x))
+    tools["currentAnswer"] = tools["title"].apply(lambda x: current_answer(x, fpmms))
+
+    tools["currentAnswer"] = tools["currentAnswer"].str.replace("yes", "Yes")
+    tools["currentAnswer"] = tools["currentAnswer"].str.replace("no", "No")
+
+    # Convert block number to timestamp
+    logging.info("Converting block number to timestamp")
+    t_map = pickle.load(open(DATA_DIR / "t_map.pkl", "rb"))
+    tools["request_time"] = tools["request_block"].map(t_map)
+
+    # Identify tools with missing request_time and fill them
+    missing_time_indices = tools[tools["request_time"].isna()].index
+    if not missing_time_indices.empty:
+        partial_block_number_to_timestamp = partial(
+            block_number_to_timestamp, web3=web3
+        )
+        missing_timestamps = parallelize_timestamp_conversion(
+            tools.loc[missing_time_indices], partial_block_number_to_timestamp
+        )
+
+        # Update the original DataFrame with the missing timestamps
+        for i, timestamp in zip(missing_time_indices, missing_timestamps):
+            tools.at[i, "request_time"] = timestamp
+
+    tools["request_month_year"] = pd.to_datetime(tools["request_time"]).dt.strftime(
+        "%Y-%m"
+    )
+    tools["request_month_year_week"] = (
+        pd.to_datetime(tools["request_time"]).dt.to_period("W").astype(str)
+    )
+
+    # Save the tools data after the updates on the content
+    tools.to_parquet(DATA_DIR / TOOLS_FILENAME, index=False)
+
+    # Update t_map with new timestamps
+    new_timestamps = (
+        tools[["request_block", "request_time"]]
+        .dropna()
+        .set_index("request_block")
+        .to_dict()["request_time"]
+    )
+    t_map.update(new_timestamps)
+
+    with open(DATA_DIR / "t_map_2024.pkl", "wb") as f:
+        pickle.dump(t_map, f)
+
+    # clean and release all memory
+    del tools
+    del fpmms
+    del t_map
+    gc.collect()
+
+    logging.info("ROI analysis files generated and saved")
+
+
+if __name__ == "__main__":
+    roi_analysis()

scripts/tools.py

@@ -26,6 +26,7 @@ from typing import (
     List,
     Dict,
     Union,
+    Any,
 )
 import pandas as pd
 import requests
@@ -46,7 +47,6 @@ from web3 import Web3, HTTPProvider
 from web3.exceptions import MismatchedABI
 from web3.types import BlockParams
 from concurrent.futures import ThreadPoolExecutor, as_completed
-from get_mech_info import get_mech_info_last_60_days
 from utils import (
     clean,
     BLOCK_FIELD,
@@ -376,6 +376,7 @@ def store_progress(
 
 def etl(
     rpcs: List[str],
+    mech_info: dict[str, Any],
     filename: Optional[str] = None,
 ) -> pd.DataFrame:
     """Fetch from on-chain events, process, store and return the tools' results on
@@ -392,7 +393,7 @@ def etl(
             os.path.join(CONTRACTS_PATH, filename),
             earliest_block,
         )
-        for address, (filename, earliest_block) in get_mech_info_last_60_days().items()
+        for address, (filename, earliest_block) in mech_info.items()
     }
 
     event_to_contents = {}
@@ -526,6 +527,8 @@ def update_tools_accuracy(
     # update the old information
     print("Updating accuracy information")
     tools_to_update = list(acc_info["tool"].values)
+    print("tools to update")
+    print(tools_to_update)
     existing_tools = list(tools_acc["tool"].values)
     for tool in tools_to_update:
         if tool in existing_tools:

scripts/update_tools_accuracy.py

@@ -20,9 +20,11 @@ def compute_tools_accuracy():
     if os.path.exists(DATA_DIR / ACCURACY_FILENAME):
         acc_data = pd.read_csv(DATA_DIR / ACCURACY_FILENAME)
     acc_data = update_tools_accuracy(acc_data, tools, INC_TOOLS)
+
     # save acc_data into a CSV file
     print("Saving into a csv file")
     acc_data.to_csv(DATA_DIR / ACCURACY_FILENAME, index=False)
+    print(acc_data.head())
 
     # save the data into IPFS
     client = ipfshttpclient.connect(IPFS_SERVER)

scripts/utils.py

@@ -3,7 +3,7 @@ import json
 import os
 import time
 from tqdm import tqdm
-from typing import List, Any, Optional
+from typing import List, Any, Optional, Union
 import pandas as pd
 import gc
 import re
@@ -27,6 +27,9 @@ HTTP = "http://"
 HTTPS = HTTP[:4] + "s" + HTTP[4:]
 IPFS_ADDRESS = f"{HTTPS}gateway.autonolas.tech/ipfs/"
 FORMAT_UPDATE_BLOCK_NUMBER = 30411638
+INVALID_ANSWER_HEX = (
+    "0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"
+)
 
 INC_TOOLS = [
     "prediction-online",
@@ -330,3 +333,18 @@ def current_answer(text: str, fpmms: pd.DataFrame) -> Optional[str]:
     if row.shape[0] == 0:
         return None
     return row["currentAnswer"].values[0]
+
+
+def convert_hex_to_int(x: Union[str, float]) -> Union[int, float]:
+    """Convert hex to int"""
+    if isinstance(x, float):
+        return np.nan
+    if isinstance(x, str):
+        if x == INVALID_ANSWER_HEX:
+            return -1
+        return int(x, 16)
+
+
+def wei_to_unit(wei: int) -> float:
+    """Converts wei to currency unit."""
+    return wei / 10**18