updated live data including Friday

data/closed_markets_div.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:99b7ff8fafd742e4c7b4601adb95bd42cf560d6a81ac97819ea5748a6ba4b900
-size 50378
+oid sha256:d715e676a0779f18b69e9f549175793e6581cb9e87a456f3e8b0bc7db26190d6
+size 48884

data/daily_info.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:23cc8b305b1f2f5b6cc7b4a6017c1aeb2a003ec5cc921be1ad66f380d29a6102
-size 658465
+oid sha256:e0bf740abcffe1facea0fed126c61a19406ef32474c8dd63d85ea9448f96f701
+size 423117

data/unknown_daily_traders.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c74aad2a1150163d78233e9a54ffa262d195e43b68b9aba1e33536671075ef57
-size 163247
+oid sha256:1db631b6cc5b6ff1aadd6ce3285dc032fe79c83cd14bb2c1cb1fa7b7917e61b0
+size 25139

data/weekly_mech_calls.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a91eccf4392fc1cbdbb911fa2163a5aff3f4483692ab059083492047c3a4f55a
-size 50574
+oid sha256:6e321e63d58f312fe2769880d9ec5ec9fba24229e427a514a3a9567936edbab5
+size 50009

scripts/wow_retentions.py

@@ -4,83 +4,64 @@ from utils import DATA_DIR
 
 
 # Basic Week over Week Retention
-def calculate_wow_retention(
-    traders_df: pd.DataFrame, trader_filter: str
-) -> pd.DataFrame:
-    """Function to compute the wow retention at the week level"""
-    if trader_filter == "Olas":
-        df = traders_df.loc[traders_df["staking"] != "non_Olas"]
-    elif trader_filter == "non_Olas":
-        df = traders_df.loc[traders_df["staking"] == "non_Olas"]
-    else:
-        # unknown traders
-        print("Not implemented yet")
-    # Get unique traders per week
+def calculate_wow_retention_by_type(df: pd.DataFrame) -> pd.DataFrame:
+    # Get unique traders per week and type
     weekly_traders = (
-        df.groupby("month_year_week")["trader_address"].nunique().reset_index()
+        df.groupby(["month_year_week", "trader_type"])["trader_address"]
+        .nunique()
+        .reset_index()
     )
+    weekly_traders = weekly_traders.sort_values(["trader_type", "month_year_week"])
 
     # Calculate retention
     retention = []
-    for i in range(1, len(weekly_traders)):
-        current_week = weekly_traders.iloc[i]["month_year_week"]
-        previous_week = weekly_traders.iloc[i - 1]["month_year_week"]
-
-        # Get traders in both weeks
-        current_traders = set(
-            df[df["month_year_week"] == current_week]["trader_address"]
-        )
-        previous_traders = set(
-            df[df["month_year_week"] == previous_week]["trader_address"]
-        )
-
-        retained = len(current_traders.intersection(previous_traders))
-        retention_rate = (retained / len(previous_traders)) * 100
-
-        retention.append(
-            {
-                "month_year_week": current_week,
-                "retained_traders": retained,
-                "previous_traders": len(previous_traders),
-                "retention_rate": retention_rate,
-            }
-        )
+    # Iterate through each trader type
+    for trader_type in weekly_traders["trader_type"].unique():
+        type_data = weekly_traders[weekly_traders["trader_type"] == trader_type]
+
+        # Calculate retention for each week within this trader type
+        for i in range(1, len(type_data)):
+            current_week = type_data.iloc[i]["month_year_week"]
+            previous_week = type_data.iloc[i - 1]["month_year_week"]
+
+            # Get traders in both weeks for this type
+            current_traders = set(
+                df[
+                    (df["month_year_week"] == current_week)
+                    & (df["trader_type"] == trader_type)
+                ]["trader_address"]
+            )
+
+            previous_traders = set(
+                df[
+                    (df["month_year_week"] == previous_week)
+                    & (df["trader_type"] == trader_type)
+                ]["trader_address"]
+            )
+
+            retained = len(current_traders.intersection(previous_traders))
+            retention_rate = (
+                (retained / len(previous_traders)) * 100
+                if len(previous_traders) > 0
+                else 0
+            )
+
+            retention.append(
+                {
+                    "trader_type": trader_type,
+                    "week": current_week,
+                    "retained_traders": retained,
+                    "previous_traders": len(previous_traders),
+                    "retention_rate": round(retention_rate, 2),
+                }
+            )
 
     return pd.DataFrame(retention)
 
 
-# N-Week Rolling Retention
-def calculate_nweek_retention(df: pd.DataFrame, n_weeks=4):
-    # Get first and last trade for each trader
-    trader_activity = (
-        df.groupby("trader_address")
-        .agg({"creation_timestamp": ["min", "max"]})
-        .reset_index()
-    )
-
-    trader_activity.columns = ["trader_address", "first_trade", "last_trade"]
-    trader_activity["weeks_active"] = (
-        pd.to_datetime(trader_activity["last_trade"])
-        - pd.to_datetime(trader_activity["first_trade"])
-    ).dt.days / 7
-
-    return {
-        "total_traders": len(trader_activity),
-        f"{n_weeks}_week_retained": len(
-            trader_activity[trader_activity["weeks_active"] >= n_weeks]
-        ),
-        "retention_rate": (
-            len(trader_activity[trader_activity["weeks_active"] >= n_weeks])
-            / len(trader_activity)
-        )
-        * 100,
-    }
-
-
 # Cohort Retention
-def calculate_cohort_retention(df, max_weeks=12):
+def calculate_cohort_retention(df, max_weeks=12) -> pd.DataFrame:
     # Get first week for each trader
-    # TODO check if first will retrieve the first week of the data or not
     first_trades = (
         df.groupby("trader_address")
         .agg({"creation_timestamp": "min", "month_year_week": "first"})
@@ -119,6 +100,10 @@ def calculate_cohort_retention(df, max_weeks=12):
     # Convert to percentages
     retention_matrix = retention_matrix.div(cohort_sizes, axis=0) * 100
 
+    # Sort index (cohort_week) chronologically
+    retention_matrix.index = pd.to_datetime(retention_matrix.index)
+    retention_matrix = retention_matrix.sort_index()
+
     # Limit to max_weeks if specified
     if max_weeks is not None and max_weeks < retention_matrix.shape[1]:
         retention_matrix = retention_matrix.iloc[:, :max_weeks]
@@ -126,16 +111,28 @@ def calculate_cohort_retention(df, max_weeks=12):
     return retention_matrix.round(2)
 
 
-if __name__ == "__main__":
-    # read trades dataset
-    trades_df = pd.read_parquet(DATA_DIR / "all_trades_profitability.parquet")
-    # First, create week numbers from timestamps
-    trades_df["creation_timestamp"] = pd.to_datetime(trades_df["creation_timestamp"])
-    trades_df = trades_df.sort_values(by="creation_timestamp", ascending=True)
-    trades_df["month_year_week"] = (
-        trades_df["creation_timestamp"].dt.to_period("W").dt.strftime("%b-%d-%Y")
+def prepare_retention_dataset() -> pd.DataFrame:
+    # read all datasets
+    traders_df = pd.read_parquet(DATA_DIR / "all_trades_profitability.parquet")
+    unknown_df = pd.read_parquet(DATA_DIR / "unknown_traders.parquet")
+    traders_df["trader_type"] = traders_df["staking"].apply(
+        lambda x: "non_Olas" if x == "non_Olas" else "Olas"
+    )
+    unknown_df["trader_type"] = "unclassified"
+    all_traders = pd.concat([traders_df, unknown_df], ignore_index=True)
+
+    all_traders["creation_timestamp"] = pd.to_datetime(
+        all_traders["creation_timestamp"]
+    )
+    all_traders = all_traders.sort_values(by="creation_timestamp", ascending=True)
+    all_traders["month_year_week"] = (
+        all_traders["creation_timestamp"].dt.to_period("W").dt.strftime("%b-%d-%Y")
     )
+    return all_traders
+
+
+if __name__ == "__main__":
+    all_traders = prepare_retention_dataset()
     # Usage example:
-    wow_retention = calculate_wow_retention(trades_df, trader_filter="Olas")
-    rolling_retention = calculate_nweek_retention(trades_df, n_weeks=4)
-    cohort_retention = calculate_cohort_retention(trades_df)
+    wow_retention = calculate_wow_retention_by_type(all_traders)
+    cohort_retention = calculate_cohort_retention(all_traders)