Skip existing bond details during refresh

App/routers/bonds/utils.py

@@ -230,7 +230,11 @@ class BondDataScraper:
             isin=json_data.get("ISIN")
         )
 
-    async def scrape_all_bond_data(self) -> AsyncGenerator[BondCreate, None]:
+    async def scrape_all_bond_data(
+        self, existing_auction_numbers: Optional[set[int]] = None
+    ) -> AsyncGenerator[BondCreate, None]:
+        existing_auction_numbers = existing_auction_numbers or set()
+
         async with AsyncSession() as session:
             # First GET request to establish session cookies if necessary
             await session.get(self.TBONDS_URL, headers=self.headers, impersonate=self.IMPERSONATE_PROFILE,timeout=60*5)
@@ -245,6 +249,10 @@ class BondDataScraper:
             print(f"Found {len(initial_bond_rows)} initial bond rows from main table.")
 
             for row_data in initial_bond_rows:
+                if row_data["au_no"] in existing_auction_numbers:
+                    print(f"Skipping existing bond details for au_no: {row_data['au_no']}")
+                    continue
+
                 print(f"Fetching details for au_no: {row_data['au_no']}...")
                 await asyncio.sleep(0.5) # Small delay to be polite
                 

App/scheduler.py

@@ -117,13 +117,19 @@ async def refresh_bonds() -> None:
     from App.routers.bonds.models import Bond
     from App.routers.bonds.utils import BondDataScraper
 
-    logger.info("[scheduler] Bond refresh: scraping bot.go.tz/TBonds")
+    existing_auction_numbers = set(await Bond.all().values_list("auction_number", flat=True))
+    logger.info(
+        "[scheduler] Bond refresh: checking bot.go.tz/TBonds, %s auction(s) already stored",
+        len(existing_auction_numbers),
+    )
     scraper = BondDataScraper()
     created = updated = failed = 0
     processed_isins: set[str] = set()
 
     try:
-        async for bond_data in scraper.scrape_all_bond_data():
+        async for bond_data in scraper.scrape_all_bond_data(
+            existing_auction_numbers=existing_auction_numbers
+        ):
             if not bond_data:
                 failed += 1
                 continue
@@ -150,6 +156,7 @@ async def refresh_bonds() -> None:
                     else:
                         await Bond.create(**bond_data.dict())
                         created += 1
+                        existing_auction_numbers.add(bond_data.auction_number)
 
                     if bond_data.isin:
                         processed_isins.add(bond_data.isin)