Spaces:
Running
on
CPU Upgrade
Running
on
CPU Upgrade
Commit
·
ff54ca8
1
Parent(s):
a75901c
refactor: enhance database setup logging and retrieve all metadata for accurate timestamp
Browse files
main.py
CHANGED
|
@@ -250,7 +250,8 @@ def setup_database():
|
|
| 250 |
|
| 251 |
# Verify the update worked by checking latest records
|
| 252 |
if final_count > 0:
|
| 253 |
-
|
|
|
|
| 254 |
final_timestamps = [
|
| 255 |
dateutil.parser.parse(m.get("last_modified"))
|
| 256 |
for m in final_metadata.get("metadatas")
|
|
@@ -266,6 +267,8 @@ def setup_database():
|
|
| 266 |
logger.info(
|
| 267 |
f"Successfully added records from {latest_update} to {latest_after_update}"
|
| 268 |
)
|
|
|
|
|
|
|
| 269 |
|
| 270 |
# Load model data
|
| 271 |
model_lazy_df = pl.scan_parquet(
|
|
|
|
| 250 |
|
| 251 |
# Verify the update worked by checking latest records
|
| 252 |
if final_count > 0:
|
| 253 |
+
# Get ALL metadata to find the true latest timestamp (not just 5 records)
|
| 254 |
+
final_metadata = dataset_collection.get(include=["metadatas"])
|
| 255 |
final_timestamps = [
|
| 256 |
dateutil.parser.parse(m.get("last_modified"))
|
| 257 |
for m in final_metadata.get("metadatas")
|
|
|
|
| 267 |
logger.info(
|
| 268 |
f"Successfully added records from {latest_update} to {latest_after_update}"
|
| 269 |
)
|
| 270 |
+
else:
|
| 271 |
+
logger.info(f"Initial database setup completed. Latest record: {latest_after_update}")
|
| 272 |
|
| 273 |
# Load model data
|
| 274 |
model_lazy_df = pl.scan_parquet(
|