File size: 1,515 Bytes
780c162 cbd62a3 780c162 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 | """Computes which papers need (re)embedding since the last sync."""
import hashlib
from dataclasses import dataclass
@dataclass
class DeltaResult:
new_or_changed: list[dict]
removed_ids: list[str]
def compute_content_hash(paper: dict) -> str:
# NOTE: `bibtex` and `pdf_url` are deliberately excluded. This hash drives
# re-embedding, and `add_vector` only ever appends to the FAISS index — it
# never overwrites — so a hash change here would orphan the paper's old
# vector. Both fields are kept fresh instead by the metadata-only
# `update_metadata_fields` in `common.db`, which is decoupled from embedding.
payload = "|".join([
paper["title"], paper["abstract"], paper["authors"],
paper["venue"], str(paper["year"]), paper["url"],
])
return hashlib.sha256(payload.encode("utf-8")).hexdigest()
def compute_delta(current_papers: list[dict], last_state: dict[str, str]) -> DeltaResult:
current_ids = set()
new_or_changed = []
for paper in current_papers:
current_ids.add(paper["id"])
content_hash = compute_content_hash(paper)
if last_state.get(paper["id"]) != content_hash:
new_or_changed.append(paper)
removed_ids = [pid for pid in last_state if pid not in current_ids]
return DeltaResult(new_or_changed=new_or_changed, removed_ids=removed_ids)
def build_state(current_papers: list[dict]) -> dict[str, str]:
return {paper["id"]: compute_content_hash(paper) for paper in current_papers}
|