backup

app.py

@@ -32,8 +32,12 @@ dataset_post_ids = list(
 photoexp = pd.read_csv("./photoexp_filtered.csv")
 valid_post_ids = set(photoexp.post_id.tolist())
 
-# filter RESULTS_BACKUP_REPO to include only valid_post_ids
-dataset = dataset.filter(lambda x: x["post_id"] in valid_post_ids)
+# filter RESULTS_BACKUP_REPO to include only valid_post_ids using batched processing
+dataset = dataset.filter(
+    lambda xs: [x in valid_post_ids for x in xs["post_id"]],
+    batched=True,
+    batch_size=256,
+)
 
 
 # Download existing data from hub