Upload data/data_loader.py with huggingface_hub

data/data_loader.py

@@ -681,14 +681,18 @@ class OracleDataset(Dataset):
             profile['deployed_tokens_avg_peak_mc_usd'] = torch.mean(torch.tensor(peak_mcs)).item() if peak_mcs else 0.0
             profile['deployed_tokens_median_peak_mc_usd'] = torch.median(torch.tensor(peak_mcs)).item() if peak_mcs else 0.0
     
-    def _process_wallet_data(self, wallet_addresses: List[str], token_data: Dict[str, Any], pooler: EmbeddingPooler, T_cutoff: datetime.datetime, 
+    def _process_wallet_data(self, wallet_addresses: List[str], token_data: Dict[str, Any], pooler: EmbeddingPooler, T_cutoff: datetime.datetime,
                              profiles_override: Optional[Dict] = None, socials_override: Optional[Dict] = None, holdings_override: Optional[Dict] = None) -> tuple[Dict[str, Dict[str, Any]], Dict[str, Dict[str, Any]]]:
         """
         Fetches or uses cached profile, social, and holdings data.
         """
+        import time as _time
+        _wd_timings = {}
+
         if not wallet_addresses:
             return {}, token_data
 
+        _t0 = _time.perf_counter()
         if profiles_override is not None and socials_override is not None:
              profiles, socials = profiles_override, socials_override
              holdings = holdings_override if holdings_override is not None else {}
@@ -698,6 +702,7 @@ class OracleDataset(Dataset):
                  holdings = self.fetcher.fetch_wallet_holdings(wallet_addresses, T_cutoff)
              else:
                  profiles, socials, holdings = {}, {}, {}
+        _wd_timings['db_fetch'] = _time.perf_counter() - _t0
 
         valid_wallets = [addr for addr in wallet_addresses if addr in profiles]
         if not valid_wallets:
@@ -710,11 +715,19 @@ class OracleDataset(Dataset):
             for holding_item in holdings.get(wallet_addr, []):
                 if 'mint_address' in holding_item:
                     all_holding_mints.add(holding_item['mint_address'])
-        
+        _wd_timings['num_holding_tokens'] = len(all_holding_mints)
+
         # --- Process all discovered tokens with point-in-time logic ---
+        _t0 = _time.perf_counter()
         processed_new_tokens = self._process_token_data(list(all_holding_mints), pooler, T_cutoff)
+        _wd_timings['holding_token_processing'] = _time.perf_counter() - _t0
         all_token_data = {**token_data, **(processed_new_tokens or {})}
 
+        # Print wallet_data sub-timings
+        print(f"    [WALLET_DATA] db_fetch: {_wd_timings['db_fetch']*1000:.1f}ms, "
+              f"holding_tokens: {_wd_timings['num_holding_tokens']}, "
+              f"holding_token_processing: {_wd_timings['holding_token_processing']*1000:.1f}ms")
+
         # --- Calculate deployed token stats using point-in-time logic ---
         self._calculate_deployed_token_stats(profiles, T_cutoff)