adding new unknown trader category

app.py

@@ -70,43 +70,19 @@ def get_logger():
 logger = get_logger()
 
 
-def get_last_one_month_data():
-    """
-    Get the last one month data from the tools.parquet file
-    """
-    logger.info("Getting last one month data")
-    con = duckdb.connect(":memory:")
-    one_months_ago = (datetime.now() - timedelta(days=60)).strftime("%Y-%m-%d")
-
-    # Query to fetch data from all_trades_profitability.parquet
-    query2 = f"""
-    SELECT *
-    FROM read_parquet('./data/all_trades_profitability.parquet')
-    WHERE creation_timestamp >= '{one_months_ago}'
-    """
-    df2 = con.execute(query2).fetchdf()
-    logger.info("Got last one month data from all_trades_profitability.parquet")
-
-    query1 = f"""
-    SELECT *
-    FROM read_parquet('./data/tools.parquet')
-    WHERE request_time >= '{one_months_ago}'
-    """
-    df1 = con.execute(query1).fetchdf()
-    logger.info("Got last one month data from tools.parquet")
-
-    con.close()
-
-    return df1, df2
-
-
 def get_all_data():
     """
-    Get all data from the tools.parquet, tools_accuracy and trades parquet files
+    Get all data from the parquet files
     """
     logger.info("Getting all data")
 
     con = duckdb.connect(":memory:")
+    query5 = f"""
+    SELECT *
+    FROM read_parquet('./data/unknown_traders.parquet')
+    """
+    df5 = con.execute(query5).fetchdf()
+
     # Query to fetch invalid trades data
     query4 = f"""
     SELECT *
@@ -138,18 +114,21 @@ def get_all_data():
 
     con.close()
 
-    return df1, df2, df3, df4
+    return df1, df2, df3, df4, df5
 
 
 def prepare_data():
     """
     Prepare the data for the dashboard
     """
-    tools_df, trades_df, tools_accuracy_info, invalid_trades = get_all_data()
+    tools_df, trades_df, tools_accuracy_info, invalid_trades, unknown_trades = (
+        get_all_data()
+    )
     print(trades_df.info())
 
     tools_df = prepare_tools(tools_df)
     trades_df = prepare_trades(trades_df)
+    unknown_trades = prepare_trades(unknown_trades)
 
     tools_accuracy_info = compute_weighted_accuracy(tools_accuracy_info)
     print("weighted accuracy info")
@@ -166,11 +145,14 @@ def prepare_data():
         outliers.to_parquet("./data/outliers.parquet")
         trades_df = trades_df.loc[trades_df["roi"] < 1000]
 
-    return tools_df, trades_df, tools_accuracy_info, invalid_trades
+    return tools_df, trades_df, tools_accuracy_info, invalid_trades, unknown_trades
 
 
-tools_df, trades_df, tools_accuracy_info, invalid_trades = prepare_data()
+tools_df, trades_df, tools_accuracy_info, invalid_trades, unknown_trades = (
+    prepare_data()
+)
 trades_df = trades_df.sort_values(by="creation_timestamp", ascending=True)
+unknown_trades = unknown_trades.sort_values(by="creation_timestamp", ascending=True)
 
 demo = gr.Blocks()
 
@@ -255,7 +237,7 @@ with demo:
                     "# Weekly trading metrics for trades coming from 🌊 Olas traders"
                 )
             with gr.Row():
-                trade_a_details_selector = gr.Dropdown(
+                trade_o_details_selector = gr.Dropdown(
                     label="Select a trade metric",
                     choices=trade_metric_choices,
                     value=default_trade_metric,
@@ -263,15 +245,15 @@ with demo:
 
             with gr.Row():
                 with gr.Column(scale=3):
-                    a_trade_details_plot = plot_trade_metrics(
+                    trade_o_details_plot = plot_trade_metrics(
                         metric_name=default_trade_metric,
                         trades_df=trades_df,
-                        trader_filter="agent",
+                        trader_filter="Olas",
                     )
                 with gr.Column(scale=1):
                     trade_details_text = get_trade_metrics_text()
 
-            def update_a_trade_details(trade_detail, trade_details_plot):
+            def update_a_trade_details(trade_detail, trade_o_details_plot):
                 new_a_plot = plot_trade_metrics(
                     metric_name=trade_detail,
                     trades_df=trades_df,
@@ -279,10 +261,10 @@ with demo:
                 )
                 return new_a_plot
 
-            trade_a_details_selector.change(
+            trade_o_details_selector.change(
                 update_a_trade_details,
-                inputs=[trade_a_details_selector, a_trade_details_plot],
-                outputs=[a_trade_details_plot],
+                inputs=[trade_o_details_selector, trade_o_details_plot],
+                outputs=[trade_o_details_plot],
             )
 
             # Non-Olasic traders graph
@@ -291,7 +273,7 @@ with demo:
                     "# Weekly trading metrics for trades coming from Non-Olas traders"
                 )
             with gr.Row():
-                trade_na_details_selector = gr.Dropdown(
+                trade_no_details_selector = gr.Dropdown(
                     label="Select a trade metric",
                     choices=trade_metric_choices,
                     value=default_trade_metric,
@@ -299,7 +281,7 @@ with demo:
 
             with gr.Row():
                 with gr.Column(scale=3):
-                    na_trade_details_plot = plot_trade_metrics(
+                    trade_no_details_plot = plot_trade_metrics(
                         metric_name=default_trade_metric,
                         trades_df=trades_df,
                         trader_filter="non_Olas",
@@ -308,23 +290,59 @@ with demo:
                     trade_details_text = get_trade_metrics_text()
 
             def update_na_trade_details(trade_detail, trade_details_plot):
-                new_a_plot = plot_trade_metrics(
+                new_no_plot = plot_trade_metrics(
                     metric_name=trade_detail,
                     trades_df=trades_df,
                     trader_filter="non_Olas",
                 )
-                return new_a_plot
+                return new_no_plot
+
+            trade_no_details_selector.change(
+                update_na_trade_details,
+                inputs=[trade_no_details_selector, trade_no_details_plot],
+                outputs=[trade_no_details_plot],
+            )
+            # Unknown traders graph
+            with gr.Row():
+                gr.Markdown(
+                    "# Weekly trading metrics for trades coming from unknown traders"
+                )
+            with gr.Row():
+                trade_u_details_selector = gr.Dropdown(
+                    label="Select a trade metric",
+                    choices=trade_metric_choices,
+                    value=default_trade_metric,
+                )
+
+            with gr.Row():
+                with gr.Column(scale=3):
+                    trade_u_details_plot = plot_trade_metrics(
+                        metric_name=default_trade_metric,
+                        trades_df=unknown_trades,
+                        trader_filter="all",
+                    )
+                with gr.Column(scale=1):
+                    trade_details_text = get_trade_metrics_text()
+
+            def update_na_trade_details(trade_detail, trade_u_details_plot):
+                new_u_plot = plot_trade_metrics(
+                    metric_name=trade_detail,
+                    trades_df=unknown_trades,
+                    trader_filter="all",
+                )
+                return new_u_plot
 
-            trade_na_details_selector.change(
+            trade_u_details_selector.change(
                 update_na_trade_details,
-                inputs=[trade_na_details_selector, na_trade_details_plot],
-                outputs=[na_trade_details_plot],
+                inputs=[trade_u_details_selector, trade_u_details_plot],
+                outputs=[trade_u_details_plot],
             )
 
         with gr.TabItem("🔒 Staking traders"):
             with gr.Row():
                 gr.Markdown("# Trades conducted at the Pearl markets")
             with gr.Row():
+                print("Calling plot staking with pearl")
                 staking_pearl_trades_by_week = plot_staking_trades_per_market_by_week(
                     trades_df=trades_df, market_creator="pearl"
                 )

data/all_trades_profitability.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9ff7a9001dceeac25cf151e9b8cff55beafe610387c42e093d004e2712206e6b
-size 3884891
+oid sha256:6d097ed0d81bd0fa9f2e301a1db66da1a5cb122f1ad8626327715dcaff127b83
+size 3558217

data/unknown_traders.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0ab41a7a35d8bf5c588b95849ec650e048578ddcbb18bc62df0e7a3c96902ea5
+size 368142

scripts/cleaning_old_info.py

@@ -49,9 +49,6 @@ def clean_old_data_from_parquet_files(cutoff_date: str):
 
     # generate again summary_profitability.parquet
     try:
-        print("Summarising trades...")
-        summary_df = summary_analyse(all_trades)
-
         # add staking labels
         label_trades_by_staking(trades_df=all_trades, update=False)
 
@@ -59,6 +56,8 @@ def clean_old_data_from_parquet_files(cutoff_date: str):
         all_trades.to_parquet(
             DATA_DIR / "all_trades_profitability.parquet", index=False
         )
+        print("Summarising trades...")
+        summary_df = summary_analyse(all_trades)
         summary_df.to_parquet(DATA_DIR / "summary_profitability.parquet", index=False)
     except Exception as e:
         print(f"Error generating summary and saving all trades profitability file {e}")

scripts/daily_data.py

@@ -5,6 +5,7 @@ from profitability import (
     label_trades_by_staking,
 )
 import pandas as pd
+from nr_mech_calls import create_unknown_traders_df
 
 logging.basicConfig(level=logging.INFO)
 
@@ -21,6 +22,14 @@ def prepare_live_metrics(
     # staking label
     label_trades_by_staking(all_trades_df)
 
+    # create the unknown traders dataset
+    unknown_traders_df, all_trades_df = create_unknown_traders_df(
+        trades_df=all_trades_df
+    )
+    unknown_traders_df.to_parquet(
+        TMP_DIR / "unknown_daily_traders.parquet", index=False
+    )
+
     # save into a separate file
     all_trades_df.to_parquet(DATA_DIR / "daily_info.parquet", index=False)
 

scripts/{update_nr_mech_calls.py → nr_mech_calls.py}

@@ -1,5 +1,5 @@
 import pandas as pd
-from profitability import DATA_DIR, DEFAULT_MECH_FEE, summary_analyse
+from utils import DATA_DIR, DEFAULT_MECH_FEE
 from tqdm import tqdm
 
 
@@ -12,6 +12,16 @@ def update_roi(row: pd.DataFrame) -> float:
     return new_value
 
 
+def create_unknown_traders_df(trades_df: pd.DataFrame) -> pd.DataFrame:
+    """filter trades coming from non-Olas traders that are placing no mech calls"""
+    no_mech_calls_mask = (trades_df["staking"] == "non_Olas") & (
+        trades_df["num_mech_calls"] == 0
+    )
+    no_mech_calls_df = trades_df.loc[no_mech_calls_mask]
+    trades_df = trades_df.loc[~no_mech_calls_mask]
+    return no_mech_calls_df, trades_df
+
+
 def update_trade_nr_mech_calls(non_agents: bool = False):
     try:
         all_trades_df = pd.read_parquet(DATA_DIR / "all_trades_profitability.parquet")
@@ -51,10 +61,19 @@ def update_trade_nr_mech_calls(non_agents: bool = False):
 
     # saving
     all_trades_df.to_parquet(DATA_DIR / "all_trades_profitability.parquet", index=False)
-    print("Summarising trades...")
-    summary_df = summary_analyse(all_trades_df)
-    summary_df.to_parquet(DATA_DIR / "summary_profitability.parquet", index=False)
+    # print("Summarising trades...")
+    # summary_df = summary_analyse(all_trades_df)
+    # summary_df.to_parquet(DATA_DIR / "summary_profitability.parquet", index=False)
 
 
 if __name__ == "__main__":
-    update_trade_nr_mech_calls(non_agents=True)
+    # update_trade_nr_mech_calls(non_agents=True)
+    trades_df = pd.read_parquet(DATA_DIR / "all_trades_profitability.parquet")
+    print("before filtering")
+    print(trades_df.staking.value_counts())
+    unknown_df, trades_df = create_unknown_traders_df(trades_df=trades_df)
+    print("after filtering")
+    print(trades_df.staking.value_counts())
+    print("saving unknown traders")
+    unknown_df.to_parquet(DATA_DIR / "unknown_traders.parquet", index=False)
+    trades_df.to_parquet(DATA_DIR / "all_trades_profitability.parquet", index=False)

scripts/profitability.py

@@ -32,18 +32,20 @@ from get_mech_info import (
     update_tools_parquet,
     update_all_trades_parquet,
 )
-from utils import wei_to_unit, convert_hex_to_int, JSON_DATA_DIR, DATA_DIR, TMP_DIR
+from utils import (
+    wei_to_unit,
+    convert_hex_to_int,
+    JSON_DATA_DIR,
+    DATA_DIR,
+    DEFAULT_MECH_FEE,
+)
 from staking import label_trades_by_staking
+from nr_mech_calls import create_unknown_traders_df
 
 DUST_THRESHOLD = 10000000000000
 INVALID_ANSWER = -1
-DEFAULT_FROM_DATE = "1970-01-01T00:00:00"
-DEFAULT_TO_DATE = "2038-01-19T03:14:07"
-
 DEFAULT_60_DAYS_AGO_TIMESTAMP = (DATETIME_60_DAYS_AGO).timestamp()
-
 WXDAI_CONTRACT_ADDRESS = "0xe91D153E0b41518A2Ce8Dd3D7944Fa863463a97d"
-DEFAULT_MECH_FEE = 0.01
 DUST_THRESHOLD = 10000000000000
 
 
@@ -423,14 +425,6 @@ def run_profitability_analysis(
     # debugging purposes
     all_trades_df.to_parquet(JSON_DATA_DIR / "all_trades_df.parquet", index=False)
 
-    # filter trades coming from non-Olas traders that are placing no mech calls
-    no_mech_calls_mask = (all_trades_df["staking"] == "non_Olas") & (
-        all_trades_df.loc["num_mech_calls"] == 0
-    )
-    no_mech_calls_df = all_trades_df.loc[no_mech_calls_mask]
-    no_mech_calls_df.to_parquet(TMP_DIR / "no_mech_calls_trades.parquet", index=False)
-    all_trades_df = all_trades_df.loc[~no_mech_calls_mask]
-
     # filter invalid markets. Condition: "is_invalid" is True
     invalid_trades = all_trades_df.loc[all_trades_df["is_invalid"] == True]
     if len(invalid_trades) == 0:
@@ -452,15 +446,21 @@ def run_profitability_analysis(
 
     all_trades_df = all_trades_df.loc[all_trades_df["is_invalid"] == False]
 
-    # summarize profitability df
-    print("Summarising trades...")
-    summary_df = summary_analyse(all_trades_df)
-
     # add staking labels
     label_trades_by_staking(trades_df=all_trades_df)
 
+    # create the unknown traders dataset
+    unknown_traders_df, all_trades_df = create_unknown_traders_df(
+        trades_df=all_trades_df
+    )
+    unknown_traders_df.to_parquet(DATA_DIR / "unknown_traders.parquet", index=False)
+
     # save to parquet
     all_trades_df.to_parquet(DATA_DIR / "all_trades_profitability.parquet", index=False)
+
+    # summarize profitability df
+    print("Summarising trades...")
+    summary_df = summary_analyse(all_trades_df)
     summary_df.to_parquet(DATA_DIR / "summary_profitability.parquet", index=False)
 
     print("Done!")

scripts/pull_data.py

@@ -128,7 +128,7 @@ def only_new_weekly_analysis():
 
     save_historical_data()
 
-    clean_old_data_from_parquet_files("2024-10-13")
+    clean_old_data_from_parquet_files("2024-10-14")
 
     compute_tools_accuracy()
 

scripts/staking.py

@@ -173,7 +173,7 @@ def get_trader_address_staking(trader_address: str, service_map: dict) -> str:
     return check_owner_staking_contract(owner_address=owner)
 
 
-def label_trades_by_staking(trades_df: pd.DataFrame, start: int = None) -> pd.DataFrame:
+def label_trades_by_staking(trades_df: pd.DataFrame, start: int = None) -> None:
     with open(DATA_DIR / "service_map.pkl", "rb") as f:
         service_map = pickle.load(f)
     # get the last service id

scripts/utils.py

@@ -11,6 +11,7 @@ from pathlib import Path
 from enum import Enum
 from json.decoder import JSONDecodeError
 
+DEFAULT_MECH_FEE = 0.01
 REDUCE_FACTOR = 0.25
 SLEEP = 0.5
 REQUEST_ID_FIELD = "request_id"

tabs/staking.py

@@ -39,8 +39,6 @@ def plot_staking_trades_per_market_by_week(
     all_filtered_trades = all_filtered_trades.loc[
         all_filtered_trades["market_creator"] == market_creator
     ]
-    print(f"Checking values for market creator={market_creator}")
-    print(all_filtered_trades.staking.value_counts())
     if market_creator != "all":
         if market_creator == "pearl":
             # remove the staking data from quickstart

tabs/trades.py

@@ -197,7 +197,13 @@ def integrated_plot_trades_per_market_by_week_v2(trades_df: pd.DataFrame) -> gr.
     # Process both Olas and non-Olas traces for each market together
     for market in ["pearl", "quickstart", "all"]:
         market_data = trades[trades["market_creator"] == market]
-
+        # Create a dictionary to store the Olas values for each week
+        olas_values = dict(
+            zip(
+                market_data[market_data["staking_type"] == "Olas"]["month_year_week"],
+                market_data[market_data["staking_type"] == "Olas"]["trades"],
+            )
+        )
         # First add 'Olas' trace
         olas_data = market_data[market_data["staking_type"] == "Olas"]
         olas_trace = go.Bar(
@@ -217,7 +223,7 @@ def integrated_plot_trades_per_market_by_week_v2(trades_df: pd.DataFrame) -> gr.
             name=f"{market}-non_Olas",
             marker_color=market_darker_colors[market],
             offsetgroup=market,  # Keep the market grouping
-            base=olas_data["trades"],  # Stack on top of olas trace
+            base=[olas_values.get(x, 0) for x in non_Olas_data["month_year_week"]],
             showlegend=True,
         )