Spaces:
Running on Zero
Running on Zero
File size: 3,085 Bytes
b1aba72 3f3f531 b1aba72 51e9502 b1aba72 51e9502 b1aba72 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 | """
vectorstore.py
----------------------
FAISS vector store operations: filtering, chunking, indexing, deduplication.
"""
import re
import uuid
import gc
from datetime import datetime, timezone, timedelta
from langchain_community.vectorstores import FAISS
from langchain_text_splitters import RecursiveCharacterTextSplitter
from langchain_core.documents import Document
from config import EXPIRY_DAYS
from model_setup import embedding_model
from logging_config import get_logger
logger = get_logger(__name__)
# Module-level state
vectorstore: FAISS | None = None
seen_urls: dict = {} # {url: {"ids": [...], "last_crawled": datetime}}
# Content filtering
_GARBAGE_LINE = re.compile(r'^\*?\s*\[.*?\]\(https?://.*?\)\s*$')
_GARBAGE_PHRASES = [
'follow us', 'sign up', 'log in', 'log out', 'subscribe',
'skip to main content', 'cookie', 'privacy policy',
'terms of service', 'all rights reserved', 'advertisement',
]
def filter_content(text: str) -> str:
"""
Strip navigation, footer, and boilerplate lines from scraped
web content before it enters FAISS.
"""
cleaned = []
for line in text.split('\n'):
stripped = line.strip()
if not stripped:
continue
if _GARBAGE_LINE.match(stripped):
continue
if any(phrase in stripped.lower() for phrase in _GARBAGE_PHRASES):
continue
cleaned.append(line)
return '\n'.join(cleaned)
def chunk_and_index(text: str, source_url: str = None) -> None:
"""
Filter -> chunk -> upsert into the FAISS index.
chunk_size=1000/overlap=200: sized for MiniLM's effective encoding
range. Deduplication: re-indexing the same URL first removes old
entries so stale chunks don't accumulate.
"""
global vectorstore, seen_urls
text = filter_content(text)
if len(text.strip()) < 100:
logger.debug(f"Skipping {source_url!r} — too sparse after filtering")
return
splitter = RecursiveCharacterTextSplitter(
chunk_size=1000,
chunk_overlap=200,
)
chunks = splitter.split_text(text)
docs = [
Document(
page_content=chunk,
metadata={"source": source_url or "unknown"},
)
for chunk in chunks
]
ids = [str(uuid.uuid4()) for _ in docs]
if vectorstore is None:
vectorstore = FAISS.from_documents(docs, embedding_model, ids=ids)
else:
if source_url in seen_urls:
vectorstore.delete(ids=seen_urls[source_url]["ids"])
vectorstore.add_documents(docs, ids=ids)
seen_urls[source_url] = {
"ids": ids,
"last_crawled": datetime.now(timezone.utc),
}
del docs, chunks
gc.collect()
def prune_vectorstore() -> None:
"""Evict FAISS entries whose source URLs are older than EXPIRY_DAYS."""
global vectorstore
if vectorstore is None:
return
now = datetime.now(timezone.utc)
cutoff = timedelta(days=EXPIRY_DAYS)
to_delete = []
for url, entry in list(seen_urls.items()):
if now - entry["last_crawled"] > cutoff:
to_delete.extend(entry["ids"])
del seen_urls[url]
if to_delete:
vectorstore.delete(to_delete)
|