add dotenv support for environment variables and Hugging Face login

main.py

@@ -17,7 +17,12 @@ from fastapi import FastAPI, HTTPException, Query
 from fastapi.middleware.cors import CORSMiddleware
 from pydantic import BaseModel
 from transformers import AutoTokenizer
+from dotenv import load_dotenv
+from huggingface_hub import login
 
+load_dotenv(override=True)
+HF_TOKEN = os.getenv("HF_TOKEN")
+login(token=HF_TOKEN)
 # Configuration constants
 MODEL_NAME = "davanstrien/SmolLM2-360M-tldr-sft-2025-02-12_15-13"
 EMBEDDING_MODEL = "nomic-ai/modernbert-embed-base"
@@ -73,7 +78,7 @@ app.add_middleware(
     allow_origins=[
         "https://*.hf.space",  # Allow all Hugging Face Spaces
         "https://*.huggingface.co",  # Allow all Hugging Face domains
-        #      "http://localhost:5500",  # Allow localhost:5500 # TODO remove before prod
+        # "http://localhost:5500",  # Allow localhost:5500 # TODO remove before prod
     ],
     allow_credentials=True,
     allow_methods=["*"],
@@ -142,6 +147,8 @@ def setup_database():
 
         if latest_update:
             logger.info(f"Filtering records newer than {latest_update}")
+            # Ensure last_modified is datetime before comparison
+            df = df.with_columns(pl.col("last_modified").str.to_datetime())
             df = df.filter(pl.col("last_modified") > latest_update)
             filtered_count = df.select(pl.len()).collect().item()
             logger.info(f"Found {filtered_count} records to update after filtering")