Add huggingface_hub library and pre-populate cache

main.py

@@ -23,6 +23,7 @@ from starlette.responses import RedirectResponse
 from card_processing import parse_markdown, try_load_text, is_empty_template
 import logging
 import logging.config
+from huggingface_hub import list_datasets
 import yaml
 
 disable_progress_bars()
@@ -73,6 +74,14 @@ async def lifespan(app: FastAPI):
             local_dir_use_symlinks=False,
         )
         logger.info(f"Downloaded votes.jsonl to {path}")
+        # pre-populate the cache
+        datasets = list(list_datasets(sort="likes7d", direction=-1, limit=1000))
+        dataset_ids = [dataset.id for dataset in datasets]
+        # batch into 100s
+        for i in range(0, len(dataset_ids), 10):
+            # sleep for 1 second to avoid rate limiting
+            await asyncio.sleep(1)
+            await get_summaries(SummariesRequest(dataset_ids=dataset_ids[i : i + 10]))
     else:
         logger.info("Votes file already exists")
     yield
@@ -88,7 +97,7 @@ app = FastAPI(lifespan=lifespan)
 
 # # Configure CORS settings
 # app.add_middleware(
-#     CORSMiddleware,
+#     CORSMiddleware,``
 #     allow_origins=["https://huggingface.co/*"],  # Update with your frontend URL
 #     allow_credentials=True,
 #     allow_methods=["*"],