davanstrien HF Staff commited on
Commit
04d67e9
·
verified ·
1 Parent(s): a10a183

Dedup ids at load; prep v3 corpus swap

Browse files
Files changed (1) hide show
  1. app.py +1 -0
app.py CHANGED
@@ -111,6 +111,7 @@ def load_table(cfg: str, table: str, has_param: bool):
111
  SELECT *, embedding[1:{EMB_DIM}] AS s
112
  FROM read_parquet({plist})
113
  WHERE embedding IS NOT NULL
 
114
  )
115
  )
116
  """)
 
111
  SELECT *, embedding[1:{EMB_DIM}] AS s
112
  FROM read_parquet({plist})
113
  WHERE embedding IS NOT NULL
114
+ QUALIFY row_number() OVER (PARTITION BY id ORDER BY last_modified DESC NULLS LAST) = 1
115
  )
116
  )
117
  """)