Upload app.py with huggingface_hub
Browse files
app.py
CHANGED
|
@@ -36,13 +36,18 @@ class UserIdIndex:
|
|
| 36 |
def build(self, arrow_dataset):
|
| 37 |
logger.info("Building user_id index from parquet metadata...")
|
| 38 |
t0 = time.time()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 39 |
for frag in arrow_dataset.get_fragments():
|
| 40 |
meta = frag.metadata
|
| 41 |
path = frag.path
|
| 42 |
for rg_idx in range(meta.num_row_groups):
|
| 43 |
-
col = meta.row_group(rg_idx).column(
|
| 44 |
s = col.statistics
|
| 45 |
-
if s:
|
| 46 |
self.entries.append((int(s.min), int(s.max), path, rg_idx))
|
| 47 |
self.entries.sort(key=lambda x: x[0])
|
| 48 |
self.keys = [e[0] for e in self.entries]
|
|
@@ -93,6 +98,8 @@ def init_dataset():
|
|
| 93 |
stats["startup_time"] = time.time()
|
| 94 |
|
| 95 |
|
|
|
|
|
|
|
| 96 |
def query_user(user_id: int):
|
| 97 |
stats["queries"] += 1
|
| 98 |
t0 = time.time()
|
|
@@ -102,19 +109,15 @@ def query_user(user_id: int):
|
|
| 102 |
path, rg_idx = result
|
| 103 |
try:
|
| 104 |
pf = pq.ParquetFile(path)
|
| 105 |
-
tbl = pf.read_row_group(rg_idx, columns=
|
| 106 |
-
|
| 107 |
-
tbl = tbl.filter(mask)
|
| 108 |
if len(tbl):
|
| 109 |
elapsed = time.time() - t0
|
| 110 |
return tbl.to_pylist()[0], elapsed
|
| 111 |
except Exception as e:
|
| 112 |
logger.warning(f"Index lookup failed for {user_id}: {e}")
|
| 113 |
|
| 114 |
-
tbl = dataset.to_table(
|
| 115 |
-
filter=pc.equal(pc.field("user_id"), user_id),
|
| 116 |
-
columns=["user_id", "username", "first_name", "last_name", "phone", "email", "status", "linked_id", "linked_name", "linked_handle"]
|
| 117 |
-
)
|
| 118 |
elapsed = time.time() - t0
|
| 119 |
if len(tbl) == 0:
|
| 120 |
return None, elapsed
|
|
|
|
| 36 |
def build(self, arrow_dataset):
|
| 37 |
logger.info("Building user_id index from parquet metadata...")
|
| 38 |
t0 = time.time()
|
| 39 |
+
schema = arrow_dataset.schema
|
| 40 |
+
uid_idx = schema.get_field_index("user_id")
|
| 41 |
+
if uid_idx < 0:
|
| 42 |
+
logger.error("user_id column not found in schema!")
|
| 43 |
+
return
|
| 44 |
for frag in arrow_dataset.get_fragments():
|
| 45 |
meta = frag.metadata
|
| 46 |
path = frag.path
|
| 47 |
for rg_idx in range(meta.num_row_groups):
|
| 48 |
+
col = meta.row_group(rg_idx).column(uid_idx)
|
| 49 |
s = col.statistics
|
| 50 |
+
if s and s.min is not None and s.max is not None:
|
| 51 |
self.entries.append((int(s.min), int(s.max), path, rg_idx))
|
| 52 |
self.entries.sort(key=lambda x: x[0])
|
| 53 |
self.keys = [e[0] for e in self.entries]
|
|
|
|
| 98 |
stats["startup_time"] = time.time()
|
| 99 |
|
| 100 |
|
| 101 |
+
ALL_COLS = ["user_id", "username", "first_name", "last_name", "phone", "email", "status", "linked_id", "linked_name", "linked_handle"]
|
| 102 |
+
|
| 103 |
def query_user(user_id: int):
|
| 104 |
stats["queries"] += 1
|
| 105 |
t0 = time.time()
|
|
|
|
| 109 |
path, rg_idx = result
|
| 110 |
try:
|
| 111 |
pf = pq.ParquetFile(path)
|
| 112 |
+
tbl = pf.read_row_group(rg_idx, columns=ALL_COLS)
|
| 113 |
+
tbl = tbl.filter(pc.equal(pc.field("user_id"), user_id))
|
|
|
|
| 114 |
if len(tbl):
|
| 115 |
elapsed = time.time() - t0
|
| 116 |
return tbl.to_pylist()[0], elapsed
|
| 117 |
except Exception as e:
|
| 118 |
logger.warning(f"Index lookup failed for {user_id}: {e}")
|
| 119 |
|
| 120 |
+
tbl = dataset.to_table(filter=pc.equal(pc.field("user_id"), user_id), columns=ALL_COLS)
|
|
|
|
|
|
|
|
|
|
| 121 |
elapsed = time.time() - t0
|
| 122 |
if len(tbl) == 0:
|
| 123 |
return None, elapsed
|