Upload folder using huggingface_hub

scripts/cleanup_old_filings.py

@@ -0,0 +1,44 @@
+import os
+import json
+import datetime
+from advisor.db import SessionLocal
+from advisor.models import Filing
+from config.settings import settings
+
+def cleanup_old_filings(days: int = 365):
+    cutoff = datetime.date.today() - datetime.timedelta(days=days)
+    db = SessionLocal()
+
+    old_filings = (
+        db.query(Filing)
+        .filter(Filing.filing_date < cutoff)
+        .all()
+    )
+
+    removed = []
+
+    for filing in old_filings:
+        # Delete raw HTML
+        raw_path = filing.local_path
+        if raw_path and os.path.exists(raw_path):
+            os.remove(raw_path)
+
+        # Delete processed JSON
+        processed_path = os.path.join(
+            settings.SEC_PROCESSED_DIR,
+            f"{filing.accession_no}.json"
+        )
+        if os.path.exists(processed_path):
+            os.remove(processed_path)
+
+        removed.append(filing.accession_no)
+        db.delete(filing)
+
+    db.commit()
+    db.close()
+
+    return removed
+
+if __name__ == "__main__":
+    removed = cleanup_old_filings()
+    print(f"Removed {len(removed)} old filings.")

scripts/run_pipeline.py

@@ -0,0 +1,20 @@
+import sys
+from pathlib import Path
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+from ingestion.sec_downloader import ingest_ticker
+from ingestion.text_extractor import extract_all
+from indexes.build_sec_index import build_index
+import argparse
+
+p = argparse.ArgumentParser()
+p.add_argument("tickers", nargs="+")
+p.add_argument("--forms", nargs="+", default=["10-K","10-Q"])
+p.add_argument("--max", type=int, default=5)
+args = p.parse_args()
+
+for ticker in args.tickers:
+    print(f"\n{'#'*50}\nRunning pipeline for {ticker.upper()}\n{'#'*50}")
+    ingest_ticker(ticker, form_types=args.forms, max_filings=args.max)
+    extract_all(ticker)
+    build_index(ticker)
+    print(f"\n✓ {ticker.upper()} complete")