vectorstore.py

"""
vectorstore.py
----------------------
FAISS vector store operations: filtering, chunking, indexing, deduplication.
"""
import re
import uuid
import gc
from datetime import datetime, timezone, timedelta

from langchain_community.vectorstores import FAISS
from langchain_text_splitters import RecursiveCharacterTextSplitter
from langchain_core.documents import Document

from config import EXPIRY_DAYS
from model_setup import embedding_model
from logging_config import get_logger

logger = get_logger(__name__)

# Module-level state
vectorstore: FAISS | None = None
seen_urls: dict = {}   # {url: {"ids": [...], "last_crawled": datetime}}

# Content filtering
_GARBAGE_LINE = re.compile(r'^\*?\s*\[.*?\]\(https?://.*?\)\s*$')
_GARBAGE_PHRASES = [
    'follow us', 'sign up', 'log in', 'log out', 'subscribe',
    'skip to main content', 'cookie', 'privacy policy',
    'terms of service', 'all rights reserved', 'advertisement',
]


def filter_content(text: str) -> str:
  """
  Strip navigation, footer, and boilerplate lines from scraped
  web content before it enters FAISS.
  """
  cleaned = []
  for line in text.split('\n'):
    stripped = line.strip()
    if not stripped:
      continue
    if _GARBAGE_LINE.match(stripped):
      continue
    if any(phrase in stripped.lower() for phrase in _GARBAGE_PHRASES):
      continue
    cleaned.append(line)
  return '\n'.join(cleaned)


def chunk_and_index(text: str, source_url: str = None) -> None:
  """
  Filter -> chunk -> upsert into the FAISS index.

  chunk_size=1000/overlap=200: sized for MiniLM's effective encoding
  range. Deduplication: re-indexing the same URL first removes old
  entries so stale chunks don't accumulate.
  """
  global vectorstore, seen_urls

  text = filter_content(text)
  if len(text.strip()) < 100:
      logger.debug(f"Skipping {source_url!r} — too sparse after filtering")
      return

  splitter = RecursiveCharacterTextSplitter(
      chunk_size=1000,
      chunk_overlap=200,
  )
  chunks = splitter.split_text(text)

  docs = [
      Document(
          page_content=chunk,
          metadata={"source": source_url or "unknown"},
      )
      for chunk in chunks
  ]
  ids = [str(uuid.uuid4()) for _ in docs]

  if vectorstore is None:
      vectorstore = FAISS.from_documents(docs, embedding_model, ids=ids)
  else:
      if source_url in seen_urls:
          vectorstore.delete(ids=seen_urls[source_url]["ids"])
      vectorstore.add_documents(docs, ids=ids)

  seen_urls[source_url] = {
      "ids":          ids,
      "last_crawled": datetime.now(timezone.utc),
  }

  del docs, chunks
  gc.collect()


def prune_vectorstore() -> None:
  """Evict FAISS entries whose source URLs are older than EXPIRY_DAYS."""
  global vectorstore
  if vectorstore is None:
      return

  now = datetime.now(timezone.utc)
  cutoff = timedelta(days=EXPIRY_DAYS)

  to_delete = []
  for url, entry in list(seen_urls.items()):
      if now - entry["last_crawled"] > cutoff:
          to_delete.extend(entry["ids"])
          del seen_urls[url]

  if to_delete:
      vectorstore.delete(to_delete)