Bruhletme commited on
Commit
60a9efd
·
verified ·
1 Parent(s): 77e5169

Upload app.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. app.py +12 -9
app.py CHANGED
@@ -36,13 +36,18 @@ class UserIdIndex:
36
  def build(self, arrow_dataset):
37
  logger.info("Building user_id index from parquet metadata...")
38
  t0 = time.time()
 
 
 
 
 
39
  for frag in arrow_dataset.get_fragments():
40
  meta = frag.metadata
41
  path = frag.path
42
  for rg_idx in range(meta.num_row_groups):
43
- col = meta.row_group(rg_idx).column(0)
44
  s = col.statistics
45
- if s:
46
  self.entries.append((int(s.min), int(s.max), path, rg_idx))
47
  self.entries.sort(key=lambda x: x[0])
48
  self.keys = [e[0] for e in self.entries]
@@ -93,6 +98,8 @@ def init_dataset():
93
  stats["startup_time"] = time.time()
94
 
95
 
 
 
96
  def query_user(user_id: int):
97
  stats["queries"] += 1
98
  t0 = time.time()
@@ -102,19 +109,15 @@ def query_user(user_id: int):
102
  path, rg_idx = result
103
  try:
104
  pf = pq.ParquetFile(path)
105
- tbl = pf.read_row_group(rg_idx, columns=[f.name for f in pf.schema_arrow])
106
- mask = pc.equal(pc.field("user_id"), user_id)
107
- tbl = tbl.filter(mask)
108
  if len(tbl):
109
  elapsed = time.time() - t0
110
  return tbl.to_pylist()[0], elapsed
111
  except Exception as e:
112
  logger.warning(f"Index lookup failed for {user_id}: {e}")
113
 
114
- tbl = dataset.to_table(
115
- filter=pc.equal(pc.field("user_id"), user_id),
116
- columns=["user_id", "username", "first_name", "last_name", "phone", "email", "status", "linked_id", "linked_name", "linked_handle"]
117
- )
118
  elapsed = time.time() - t0
119
  if len(tbl) == 0:
120
  return None, elapsed
 
36
  def build(self, arrow_dataset):
37
  logger.info("Building user_id index from parquet metadata...")
38
  t0 = time.time()
39
+ schema = arrow_dataset.schema
40
+ uid_idx = schema.get_field_index("user_id")
41
+ if uid_idx < 0:
42
+ logger.error("user_id column not found in schema!")
43
+ return
44
  for frag in arrow_dataset.get_fragments():
45
  meta = frag.metadata
46
  path = frag.path
47
  for rg_idx in range(meta.num_row_groups):
48
+ col = meta.row_group(rg_idx).column(uid_idx)
49
  s = col.statistics
50
+ if s and s.min is not None and s.max is not None:
51
  self.entries.append((int(s.min), int(s.max), path, rg_idx))
52
  self.entries.sort(key=lambda x: x[0])
53
  self.keys = [e[0] for e in self.entries]
 
98
  stats["startup_time"] = time.time()
99
 
100
 
101
+ ALL_COLS = ["user_id", "username", "first_name", "last_name", "phone", "email", "status", "linked_id", "linked_name", "linked_handle"]
102
+
103
  def query_user(user_id: int):
104
  stats["queries"] += 1
105
  t0 = time.time()
 
109
  path, rg_idx = result
110
  try:
111
  pf = pq.ParquetFile(path)
112
+ tbl = pf.read_row_group(rg_idx, columns=ALL_COLS)
113
+ tbl = tbl.filter(pc.equal(pc.field("user_id"), user_id))
 
114
  if len(tbl):
115
  elapsed = time.time() - t0
116
  return tbl.to_pylist()[0], elapsed
117
  except Exception as e:
118
  logger.warning(f"Index lookup failed for {user_id}: {e}")
119
 
120
+ tbl = dataset.to_table(filter=pc.equal(pc.field("user_id"), user_id), columns=ALL_COLS)
 
 
 
121
  elapsed = time.time() - t0
122
  if len(tbl) == 0:
123
  return None, elapsed