"""Computes which papers need (re)embedding since the last sync.""" import hashlib from dataclasses import dataclass @dataclass class DeltaResult: new_or_changed: list[dict] removed_ids: list[str] def compute_content_hash(paper: dict) -> str: # NOTE: `bibtex` and `pdf_url` are deliberately excluded. This hash drives # re-embedding, and `add_vector` only ever appends to the FAISS index — it # never overwrites — so a hash change here would orphan the paper's old # vector. Both fields are kept fresh instead by the metadata-only # `update_metadata_fields` in `common.db`, which is decoupled from embedding. payload = "|".join([ paper["title"], paper["abstract"], paper["authors"], paper["venue"], str(paper["year"]), paper["url"], ]) return hashlib.sha256(payload.encode("utf-8")).hexdigest() def compute_delta(current_papers: list[dict], last_state: dict[str, str]) -> DeltaResult: current_ids = set() new_or_changed = [] for paper in current_papers: current_ids.add(paper["id"]) content_hash = compute_content_hash(paper) if last_state.get(paper["id"]) != content_hash: new_or_changed.append(paper) removed_ids = [pid for pid in last_state if pid not in current_ids] return DeltaResult(new_or_changed=new_or_changed, removed_ids=removed_ids) def build_state(current_papers: list[dict]) -> dict[str, str]: return {paper["id"]: compute_content_hash(paper) for paper in current_papers}