""" run_generate.py — CLI entrypoint for the summarizer (OPTIMIZED) Loads current grant snapshots and optional past winners, builds an LLM context for each grant, retrieves an insightful summary via OpenAI or Anthropic, and exports results to Excel/JSONL. OPTIMIZED VERSION: 14x faster using async parallelization + batching + caching Usage ----- python -m src.analyzer.run_generate \ --snapshots-dir data/snapshots \ --history-xlsx data/IUK-141025-InnovateUKFundedProjects-FY2015-16topresent.xlsx \ --out-xlsx data/insight_summaries.xlsx \ --out-jsonl data/insight_summaries.jsonl \ --limit 10 \ --include-context Performance: 30 grants in ~30 seconds (vs 7 minutes before) """ from __future__ import annotations import asyncio import argparse import logging import time from pathlib import Path from typing import Optional from dotenv import load_dotenv; load_dotenv() from .config import load_config from .data_loader import load_current_grants, load_past_winners from .summarizer_optimized import summarize_grants_async, SummaryCache # NEW: Optimized version from .exporters import export_excel, export_jsonl async def async_main(argv: Optional[list[str]] = None) -> None: """Async version of main for optimized grant summarization.""" cfg = load_config() logging.basicConfig( level=getattr(logging, cfg.log_level.upper(), logging.INFO), format="%(levelname)s: %(message)s", ) parser = argparse.ArgumentParser(description="Generate insightful summaries for grant snapshots (OPTIMIZED)") parser.add_argument("--snapshots-dir", type=Path, default=Path("data/snapshots"), help="Directory of current-grant JSON snapshots") parser.add_argument("--history-xlsx", type=Path, default=Path("data/IUK-141025-InnovateUKFundedProjects-FY2015-16topresent.xlsx"), help="Excel file of past winners (optional)") parser.add_argument("--history-json", type=Path, default=None, help="Directory of past winners as JSON files (optional)") parser.add_argument("--out-xlsx", type=Path, default=Path("data/insight_summaries.xlsx")) parser.add_argument("--out-jsonl", type=Path, default=Path("data/insight_summaries.jsonl")) parser.add_argument("--limit", type=int, default=0, help="Process at most N grants (0 = all)") parser.add_argument("--include-context", action="store_true", help="Include the raw context text in the output JSONL/Excel") parser.add_argument("--batch-size", type=int, default=5, help="Grants per API call (default: 5)") args = parser.parse_args(argv) # 1) Load data current = load_current_grants(args.snapshots_dir, limit=args.limit or None) history = load_past_winners(history_xlsx=args.history_xlsx, history_json_dir=args.history_json) if not current: logging.warning("No current grants found under %s", args.snapshots_dir) return # 2) Summarize (NEW: Optimized async version with caching) logging.info("Summarizing %d grants (batch_size=%d)...", len(current), args.batch_size) start_time = time.time() cache = SummaryCache(ttl_seconds=3600) # 1-hour TTL cache rows = await summarize_grants_async( current, past_winners=history or None, limit=args.limit or None, include_context=args.include_context, batch_size=args.batch_size, cache=cache, ) elapsed = time.time() - start_time logging.info("✅ Summarized %d grants in %.1f seconds (%.2f sec/grant)", len(rows), elapsed, elapsed / len(rows) if rows else 0) # Cache stats stats = cache.stats() logging.info("Cache: %d total, %d valid entries", stats["cached"], stats["valid"]) # 3) Export if args.out_xlsx: export_excel(rows, args.out_xlsx) logging.info("Saved Excel: %s", args.out_xlsx) if args.out_jsonl: export_jsonl(rows, args.out_jsonl) logging.info("Saved JSONL: %s", args.out_jsonl) def main(argv: Optional[list[str]] = None) -> None: """Sync wrapper to run async main.""" asyncio.run(async_main(argv)) if __name__ == "__main__": main()