new weekly data and fixes on the pipeline due to new year format

data/all_trades_profitability.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:533530b73aa7075ddb221e0820df23f77a87db90da0bbf9404ea1a98b80d9bc5
-size 6389356
+oid sha256:08d3f27f13f20022a8aa176e3f84559a38c001d03753fd13101eae15ab4a1123
+size 6500230

data/daily_info.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d9f224f954dd108e164b12763dd628e05a5f17a94fd2422d9853f60f470a690d
-size 697569
+oid sha256:689d0f6abf1886fb0b089db0294e07c551485005fabbaf73ea36083d6388ce5f
+size 78026

data/error_by_markets.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:13112e4809f1c2451419991c7737171fad6b3537f5d43d9e9e72d350b98f7083
-size 12552
+oid sha256:7832acf3c67797c32225d4c253d651ffcbd4aba4bb19354ea00ad0c1d7200cc1
+size 13489

data/invalid_trades.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9b7c4c831e583b8632a6a45079df9e400fea4e40287bbed594624ad9f9437907
-size 196588
+oid sha256:6156d839803a0c09606450ead26fb3b31ef6ef30bf3507ccbd0dc0399ad366f7
+size 152273

data/tools_accuracy.csv

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:09cf501daa10343c3e3a9a93fa81290e8399db2ec2b0550e722730bcd13a423e
-size 1101
+oid sha256:ab13e55237b5f9bee0012eab8976b0ea3ef1518f0071a081d9052f97061315a7
+size 1100

data/unknown_traders.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cd75418327e20282ad0793d5f092a362d6572d5d823b87da39ba874ea2938154
-size 184739
+oid sha256:3620eaba76778060f41059fb2b6ff6e92a6000eedfd9a9119b703f84cdda11ff
+size 194084

data/winning_df.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e93e4e91ce125aa92dcfd206ad366c86b758aed598b2ce40403c22acd05f5e5c
-size 13042
+oid sha256:1ad71600af38478d3dc275bf7ad0f3cd3cd46d44d1bf54f06bb6f9d9540cc041
+size 13714

notebooks/invalid_markets.ipynb

@@ -115,11 +115,88 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": 3,
    "metadata": {},
    "outputs": [],
    "source": [
-    "old_invalid_trades = pd.read_parquet(\"../json_data/invalid_trades.parquet\")"
+    "invalid_trades = pd.read_parquet(\"../data/invalid_trades.parquet\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "<class 'pandas.core.frame.DataFrame'>\n",
+      "RangeIndex: 2589 entries, 0 to 2588\n",
+      "Data columns (total 22 columns):\n",
+      " #   Column                  Non-Null Count  Dtype              \n",
+      "---  ------                  --------------  -----              \n",
+      " 0   trader_address          2589 non-null   object             \n",
+      " 1   market_creator          2589 non-null   object             \n",
+      " 2   trade_id                2589 non-null   object             \n",
+      " 3   creation_timestamp      2589 non-null   datetime64[ns, UTC]\n",
+      " 4   title                   2589 non-null   object             \n",
+      " 5   market_status           2589 non-null   object             \n",
+      " 6   collateral_amount       2589 non-null   float64            \n",
+      " 7   outcome_index           2589 non-null   object             \n",
+      " 8   trade_fee_amount        2589 non-null   float64            \n",
+      " 9   outcomes_tokens_traded  2589 non-null   float64            \n",
+      " 10  current_answer          2589 non-null   int64              \n",
+      " 11  is_invalid              2589 non-null   bool               \n",
+      " 12  winning_trade           2589 non-null   bool               \n",
+      " 13  earnings                2589 non-null   float64            \n",
+      " 14  redeemed                2589 non-null   bool               \n",
+      " 15  redeemed_amount         2589 non-null   float64            \n",
+      " 16  num_mech_calls          2589 non-null   int64              \n",
+      " 17  mech_fee_amount         2589 non-null   float64            \n",
+      " 18  net_earnings            2589 non-null   float64            \n",
+      " 19  roi                     2573 non-null   float64            \n",
+      " 20  staking                 0 non-null      object             \n",
+      " 21  nr_mech_calls           0 non-null      float64            \n",
+      "dtypes: bool(3), datetime64[ns, UTC](1), float64(9), int64(2), object(7)\n",
+      "memory usage: 392.0+ KB\n"
+     ]
+    }
+   ],
+   "source": [
+    "invalid_trades.info()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Timestamp('2024-09-09 02:55:15+0000', tz='UTC')"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "min(invalid_trades.creation_timestamp)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "cutoff_date = \"2024-10-30\"\n",
+    "min_date_utc = pd.to_datetime(cutoff_date, format=\"%Y-%m-%d\", utc=True)\n",
+    "invalid_trades = invalid_trades.loc[invalid_trades[\"creation_timestamp\"]> min_date_utc]\n",
+    "invalid_trades.to_parquet(\"../data/invalid_trades.parquet\", index=False)"
    ]
   },
   {
@@ -139,7 +216,7 @@
     }
    ],
    "source": [
-    "len(old_invalid_trades)"
+    "len(invalid_trades)"
    ]
   },
   {

scripts/cleaning_old_info.py

@@ -79,6 +79,24 @@ def clean_old_data_from_parquet_files(cutoff_date: str):
     except Exception as e:
         print(f"Error cleaning fpmmTrades file {e}")
 
+    # clean invalid trades parquet
+    try:
+        invalid_trades = pd.read_parquet(DATA_DIR / "invalid_trades.parquet")
+
+        invalid_trades["creation_timestamp"] = pd.to_datetime(
+            invalid_trades["creation_timestamp"], utc=True
+        )
+
+        print(f"length before filtering {len(invalid_trades)}")
+        invalid_trades = invalid_trades.loc[
+            invalid_trades["creation_timestamp"] > min_date_utc
+        ]
+        print(f"length after filtering {len(invalid_trades)}")
+        invalid_trades.to_parquet(DATA_DIR / "invalid_trades.parquet", index=False)
+
+    except Exception as e:
+        print(f"Error cleaning fpmmTrades file {e}")
+
 
 if __name__ == "__main__":
     clean_old_data_from_parquet_files("2024-10-25")

scripts/get_mech_info.py

@@ -133,11 +133,16 @@ def update_fpmmTrades_parquet(trades_filename: str) -> pd.DataFrame:
         print(f"Error reading new trades parquet file {e}")
         return None
 
+    # lowercase and strip creator_address
+    new_trades_df["trader_address"] = (
+        new_trades_df["trader_address"].str.lower().str.strip()
+    )
     # ensure creationTimestamp compatibility
     try:
         new_trades_df["creationTimestamp"] = new_trades_df["creationTimestamp"].apply(
             lambda x: transform_to_datetime(x)
         )
+
     except Exception as e:
         print(f"Transformation not needed")
     try:

scripts/markets.py

@@ -309,6 +309,7 @@ def add_market_creator(tools: pd.DataFrame) -> None:
         return
     tools["market_creator"] = ""
     # traverse the list of traders
+    tools_no_market_creator = 0
     traders_list = list(tools.trader_address.unique())
     for trader_address in traders_list:
         market_creator = ""
@@ -317,6 +318,7 @@ def add_market_creator(tools: pd.DataFrame) -> None:
             market_creator = trades.iloc[0]["market_creator"]  # first value is enough
         except Exception:
             print(f"ERROR getting the market creator of {trader_address}")
+            tools_no_market_creator += 1
             continue
         # update
         tools.loc[tools["trader_address"] == trader_address, "market_creator"] = (
@@ -324,6 +326,7 @@ def add_market_creator(tools: pd.DataFrame) -> None:
         )
     # filter those tools where we don't have market creator info
     tools = tools.loc[tools["market_creator"] != ""]
+    print(f"Number of tools with no market creator info = {tools_no_market_creator}")
     return tools
 
 

scripts/mech_request_utils.py

@@ -244,6 +244,7 @@ def collect_missing_delivers(request_id: int, block_number: int) -> Dict[str, An
             return items[0]
     except Exception as e:
         print(f"Error while getting the response: {e}")
+        # TODO count how many mech requests without a deliver do we have
 
     return mech_delivers
 

scripts/profitability.py

@@ -27,7 +27,6 @@ import os
 from web3_utils import query_conditional_tokens_gc_subgraph
 from get_mech_info import (
     DATETIME_60_DAYS_AGO,
-    update_fpmmTrades_parquet,
     update_tools_parquet,
     update_all_trades_parquet,
 )
@@ -221,13 +220,16 @@ def analyse_trader(
         return trades_df
 
     # Iterate over the trades
+    trades_answer_nan = 0
+    trades_no_closed_market = 0
     for i, trade in tqdm(trades.iterrows(), total=len(trades), desc="Analysing trades"):
         try:
             market_answer = trade["fpmm.currentAnswer"]
             trading_day = trade["creation_date"]
             trade_id = trade["id"]
             if not daily_info and not market_answer:
-                print(f"Skipping trade {i} because currentAnswer is NaN")
+                # print(f"Skipping trade {i} because currentAnswer is NaN")
+                trades_answer_nan += 1
                 continue
             # Parsing and computing shared values
             collateral_amount = wei_to_unit(float(trade["collateralAmount"]))
@@ -243,9 +245,10 @@ def analyse_trader(
 
             # Skip non-closed markets
             if not daily_info and market_status != MarketState.CLOSED:
-                print(
-                    f"Skipping trade {i} because market is not closed. Market Status: {market_status}"
-                )
+                # print(
+                #     f"Skipping trade {i} because market is not closed. Market Status: {market_status}"
+                # )
+                trades_no_closed_market += 1
                 continue
             if current_answer is not None:
                 current_answer = convert_hex_to_int(current_answer)
@@ -316,6 +319,10 @@ def analyse_trader(
             print(trade)
             continue
 
+    print(f"Number of trades where currentAnswer is NaN = {trades_answer_nan}")
+    print(
+        f"Number of trades where the market is not closed = {trades_no_closed_market}"
+    )
     return trades_df
 
 
@@ -380,7 +387,6 @@ def run_profitability_analysis(
 
     # # merge previous files if requested
     if merge:
-        update_fpmmTrades_parquet(trades_filename)
         all_trades_df = update_all_trades_parquet(all_trades_df)
 
     # debugging purposes

scripts/pull_data.py

@@ -15,6 +15,7 @@ from utils import (
 )
 from get_mech_info import (
     get_mech_events_since_last_run,
+    update_fpmmTrades_parquet,
     update_json_files,
 )
 from update_tools_accuracy import compute_tools_accuracy
@@ -99,6 +100,10 @@ def only_new_weekly_analysis():
         trades_filename="new_fpmmTrades.parquet",
         from_timestamp=int(latest_timestamp.timestamp()),
     )
+    # merge with previous file
+    print("Merging with previous fpmmTrades file")
+    update_fpmmTrades_parquet(trades_filename="new_fpmmTrades.parquet")
+
     # Run tools ETL
     logging.info("Generate and parse the tools content")
     # generate only new file
@@ -122,7 +127,7 @@ def only_new_weekly_analysis():
 
     save_historical_data()
     try:
-        clean_old_data_from_parquet_files("2024-10-29")
+        clean_old_data_from_parquet_files("2024-11-06")
     except Exception as e:
         print("Error cleaning the oldest information from parquet files")
         print(f"reason = {e}")

scripts/tools_metrics.py

@@ -61,7 +61,7 @@ def prepare_tools(tools: pd.DataFrame) -> pd.DataFrame:
     tools = tools.sort_values(by="request_time", ascending=True)
 
     tools["request_month_year_week"] = (
-        pd.to_datetime(tools["request_time"]).dt.to_period("W").dt.strftime("%b-%d")
+        pd.to_datetime(tools["request_time"]).dt.to_period("W").dt.strftime("%b-%d-%Y")
     )
     # preparing the tools graph
     # adding the total

tabs/metrics.py

@@ -86,12 +86,12 @@ def plot_trade_metrics(
     # Convert string dates to datetime and sort them
     all_dates_dt = sorted(
         [
-            datetime.strptime(date, "%b-%d")
+            datetime.strptime(date, "%b-%d-%Y")
             for date in trades_filtered["month_year_week"].unique()
         ]
     )
     # Convert back to string format
-    all_dates = [date.strftime("%b-%d") for date in all_dates_dt]
+    all_dates = [date.strftime("%b-%d-%Y") for date in all_dates_dt]
     fig = px.box(
         trades_filtered,
         x="month_year_week",

tabs/staking.py

@@ -78,12 +78,12 @@ def plot_staking_trades_per_market_by_week(
     # Convert string dates to datetime and sort them
     all_dates_dt = sorted(
         [
-            datetime.strptime(date, "%b-%d")
+            datetime.strptime(date, "%b-%d-%Y")
             for date in trades["month_year_week"].unique()
         ]
     )
     # Convert back to string format
-    all_dates = [date.strftime("%b-%d") for date in all_dates_dt]
+    all_dates = [date.strftime("%b-%d-%Y") for date in all_dates_dt]
 
     fig = px.bar(
         trades,

tabs/tool_win.py

@@ -14,7 +14,7 @@ def prepare_tools(tools: pd.DataFrame) -> pd.DataFrame:
     tools = tools.sort_values(by="request_time", ascending=True)
 
     tools["request_month_year_week"] = (
-        pd.to_datetime(tools["request_time"]).dt.to_period("W").dt.strftime("%b-%d")
+        pd.to_datetime(tools["request_time"]).dt.to_period("W").dt.strftime("%b-%d-%Y")
     )
     # preparing the tools graph
     # adding the total
@@ -38,7 +38,7 @@ def get_overall_winning_rate_by_market(wins_df: pd.DataFrame) -> pd.DataFrame:
 
 
 def sort_key(date_str):
-    month, year_week = date_str.split("-")
+    month, day, year = date_str.split("-")
     month_order = [
         "Jan",
         "Feb",
@@ -54,8 +54,9 @@ def sort_key(date_str):
         "Dec",
     ]
     month_num = month_order.index(month) + 1
-    week = int(year_week)
-    return (week // 100, month_num, week % 100)  # year, month, week
+    day = int(day)
+    year = int(year)
+    return (year, month_num, day)  # year, month, day
 
 
 def integrated_plot_tool_winnings_overall_per_market_by_week(

tabs/trades.py

@@ -21,7 +21,7 @@ def prepare_trades(trades_df: pd.DataFrame) -> pd.DataFrame:
         trades_df["creation_timestamp"].dt.to_period("M").astype(str)
     )
     trades_df["month_year_week"] = (
-        trades_df["creation_timestamp"].dt.to_period("W").dt.strftime("%b-%d")
+        trades_df["creation_timestamp"].dt.to_period("W").dt.strftime("%b-%d-%Y")
     )
     trades_df["winning_trade"] = trades_df["winning_trade"].astype(int)
     return trades_df
@@ -179,12 +179,12 @@ def integrated_plot_trades_per_market_by_week_v2(trades_df: pd.DataFrame) -> gr.
     # Convert string dates to datetime and sort them
     all_dates_dt = sorted(
         [
-            datetime.strptime(date, "%b-%d")
+            datetime.strptime(date, "%b-%d-%Y")
             for date in trades["month_year_week"].unique()
         ]
     )
     # Convert back to string format
-    all_dates = [date.strftime("%b-%d") for date in all_dates_dt]
+    all_dates = [date.strftime("%b-%d-%Y") for date in all_dates_dt]
     # Combine the traces
     final_traces = []
     market_colors = {"pearl": "darkviolet", "quickstart": "goldenrod", "all": "green"}
@@ -308,12 +308,12 @@ def integrated_plot_winning_trades_per_market_by_week_v2(
     # Convert string dates to datetime and sort them
     all_dates_dt = sorted(
         [
-            datetime.strptime(date, "%b-%d")
+            datetime.strptime(date, "%b-%d-%Y")
             for date in final_df["month_year_week"].unique()
         ]
     )
     # Convert back to string format
-    all_dates = [date.strftime("%b-%d") for date in all_dates_dt]
+    all_dates = [date.strftime("%b-%d-%Y") for date in all_dates_dt]
     color_discrete_sequence = ["darkviolet", "goldenrod", "green"]
     if trader_filter == "Olas":
         final_df = final_df[final_df["staking_type"] == "Olas"]