Upload data/data_fetcher.py with huggingface_hub

data/data_fetcher.py

@@ -445,12 +445,13 @@ class DataFetcher:
                         _t_query_start = time.perf_counter()
 
                         # Cypher query to find direct neighbors of the current frontier
+                        # OPTIMIZED: Filter by timestamp IN Neo4j to avoid transferring 97%+ unused records
                         query = """
                         MATCH (a)-[r]-(b)
-                        WHERE a.address IN $addresses
+                        WHERE a.address IN $addresses AND r.timestamp <= $cutoff_ts
                         RETURN a.address AS source_address, type(r) AS link_type, properties(r) AS link_props, b.address AS dest_address, labels(b)[0] AS dest_type
                         """
-                        params = {'addresses': list(newly_found_entities)}
+                        params = {'addresses': list(newly_found_entities), 'cutoff_ts': cutoff_ts}
                         result = session.run(query, params)
 
                         _t_query_done = time.perf_counter()
@@ -458,23 +459,12 @@ class DataFetcher:
                         # --- TIMING: Result processing ---
                         _t_process_start = time.perf_counter()
                         records_total = 0
-                        records_filtered_by_time = 0
-                        records_kept = 0
 
                         current_degree_new_entities = set()
                         for record in result:
                             records_total += 1
                             link_type = record['link_type']
                             link_props = dict(record['link_props'])
-                            link_ts_raw = link_props.get('timestamp')
-                            try:
-                                link_ts = int(link_ts_raw)
-                            except (TypeError, ValueError):
-                                continue
-                            if link_ts > cutoff_ts:
-                                records_filtered_by_time += 1
-                                continue
-                            records_kept += 1
                             source_addr = record['source_address']
                             dest_addr = record['dest_address']
                             dest_type = record['dest_type']
@@ -493,8 +483,7 @@ class DataFetcher:
                         # --- TIMING: Print detailed stats ---
                         print(f"    [NEO4J TIMING] query_exec: {(_t_query_done - _t_query_start)*1000:.1f}ms, "
                               f"result_process: {(_t_process_done - _t_process_start)*1000:.1f}ms")
-                        print(f"    [NEO4J STATS] records_total: {records_total}, "
-                              f"filtered_by_time: {records_filtered_by_time}, kept: {records_kept}, "
+                        print(f"    [NEO4J STATS] records_returned: {records_total}, "
                               f"new_entities: {len(current_degree_new_entities)}")
 
                         newly_found_entities = current_degree_new_entities