sync/delta.py

"""Computes which papers need (re)embedding since the last sync."""
import hashlib
from dataclasses import dataclass


@dataclass
class DeltaResult:
    new_or_changed: list[dict]
    removed_ids: list[str]


def compute_content_hash(paper: dict) -> str:
    # NOTE: `bibtex` and `pdf_url` are deliberately excluded. This hash drives
    # re-embedding, and `add_vector` only ever appends to the FAISS index — it
    # never overwrites — so a hash change here would orphan the paper's old
    # vector. Both fields are kept fresh instead by the metadata-only
    # `update_metadata_fields` in `common.db`, which is decoupled from embedding.
    payload = "|".join([
        paper["title"], paper["abstract"], paper["authors"],
        paper["venue"], str(paper["year"]), paper["url"],
    ])
    return hashlib.sha256(payload.encode("utf-8")).hexdigest()


def compute_delta(current_papers: list[dict], last_state: dict[str, str]) -> DeltaResult:
    current_ids = set()
    new_or_changed = []
    for paper in current_papers:
        current_ids.add(paper["id"])
        content_hash = compute_content_hash(paper)
        if last_state.get(paper["id"]) != content_hash:
            new_or_changed.append(paper)
    removed_ids = [pid for pid in last_state if pid not in current_ids]
    return DeltaResult(new_or_changed=new_or_changed, removed_ids=removed_ids)


def build_state(current_papers: list[dict]) -> dict[str, str]:
    return {paper["id"]: compute_content_hash(paper) for paper in current_papers}