import json import os import httpx import pytest from huggingface_hub.errors import RemoteEntryNotFoundError from common.vector_index import load_index from common.db import init_db from sync.hf_dataset_store import download_snapshot, upload_snapshot def _remote_entry_not_found(): request = httpx.Request("GET", "http://fake/resolve/main/missing") response = httpx.Response(404, request=request) return RemoteEntryNotFoundError("404 Entry Not Found", response=response) def test_download_snapshot_returns_parsed_state(tmp_path, monkeypatch): def fake_hf_hub_download(repo_id, repo_type, filename, local_dir, token): target = os.path.join(local_dir, filename) if filename == "state.json": with open(target, "w") as f: json.dump({"papers": {"p1": "hash1"}, "last_synced_at": "2026-07-01T00:00:00+00:00"}, f) else: open(target, "wb").close() return target monkeypatch.setattr("sync.hf_dataset_store.hf_hub_download", fake_hf_hub_download) state = download_snapshot("org/repo", str(tmp_path), token="tok") assert state == {"papers": {"p1": "hash1"}, "last_synced_at": "2026-07-01T00:00:00+00:00"} assert os.path.exists(tmp_path / "index.faiss") assert os.path.exists(tmp_path / "papers.db") def test_download_snapshot_missing_state_returns_empty_state(tmp_path, monkeypatch): def fake_hf_hub_download(repo_id, repo_type, filename, local_dir, token): if filename == "state.json": raise FileNotFoundError("no state.json yet") target = os.path.join(local_dir, filename) open(target, "wb").close() return target monkeypatch.setattr("sync.hf_dataset_store.hf_hub_download", fake_hf_hub_download) state = download_snapshot("org/repo", str(tmp_path), token="tok") assert state == {"papers": {}, "last_synced_at": None} assert not os.path.exists(tmp_path / "state.json") def test_download_snapshot_handles_empty_remote_dataset_repo(tmp_path, monkeypatch): """Reproduces a real first-time sync against a brand-new, empty dataset repo: huggingface_hub raises RemoteEntryNotFoundError (an HTTP 404), not a plain FileNotFoundError, when a file doesn't exist yet on the Hub. Missing files must be left absent (not stubbed with an empty placeholder), since a 0-byte file is not a valid "missing" signal to faiss.read_index / sqlite3.connect.""" def fake_hf_hub_download(repo_id, repo_type, filename, local_dir, token): raise _remote_entry_not_found() monkeypatch.setattr("sync.hf_dataset_store.hf_hub_download", fake_hf_hub_download) state = download_snapshot("org/repo", str(tmp_path), token="tok") assert state == {"papers": {}, "last_synced_at": None} assert not os.path.exists(tmp_path / "index.faiss") assert not os.path.exists(tmp_path / "papers.db") # Downstream loaders must fall back cleanly, exactly as run_sync relies on. with pytest.raises(FileNotFoundError): load_index(str(tmp_path / "index.faiss")) conn = init_db(str(tmp_path / "papers.db")) assert conn.execute("SELECT COUNT(*) FROM papers").fetchone()[0] == 0 def test_upload_snapshot_uploads_folder_as_single_atomic_commit(tmp_path, monkeypatch): for name in ["index.faiss", "papers.db", "state.json"]: (tmp_path / name).write_text("data") calls = [] def fake_upload_folder(**kwargs): calls.append(kwargs) monkeypatch.setattr("sync.hf_dataset_store.upload_folder", fake_upload_folder) upload_snapshot("org/repo", str(tmp_path), token="tok") assert len(calls) == 1 assert calls[0]["repo_id"] == "org/repo" assert calls[0]["repo_type"] == "dataset" assert calls[0]["folder_path"] == str(tmp_path) assert calls[0]["token"] == "tok"