| import json |
| import os |
| import httpx |
| import pytest |
| from huggingface_hub.errors import RemoteEntryNotFoundError |
| from common.vector_index import load_index |
| from common.db import init_db |
| from sync.hf_dataset_store import download_snapshot, upload_snapshot |
|
|
|
|
| def _remote_entry_not_found(): |
| request = httpx.Request("GET", "http://fake/resolve/main/missing") |
| response = httpx.Response(404, request=request) |
| return RemoteEntryNotFoundError("404 Entry Not Found", response=response) |
|
|
| def test_download_snapshot_returns_parsed_state(tmp_path, monkeypatch): |
| def fake_hf_hub_download(repo_id, repo_type, filename, local_dir, token): |
| target = os.path.join(local_dir, filename) |
| if filename == "state.json": |
| with open(target, "w") as f: |
| json.dump({"papers": {"p1": "hash1"}, "last_synced_at": "2026-07-01T00:00:00+00:00"}, f) |
| else: |
| open(target, "wb").close() |
| return target |
|
|
| monkeypatch.setattr("sync.hf_dataset_store.hf_hub_download", fake_hf_hub_download) |
| state = download_snapshot("org/repo", str(tmp_path), token="tok") |
| assert state == {"papers": {"p1": "hash1"}, "last_synced_at": "2026-07-01T00:00:00+00:00"} |
| assert os.path.exists(tmp_path / "index.faiss") |
| assert os.path.exists(tmp_path / "papers.db") |
|
|
| def test_download_snapshot_missing_state_returns_empty_state(tmp_path, monkeypatch): |
| def fake_hf_hub_download(repo_id, repo_type, filename, local_dir, token): |
| if filename == "state.json": |
| raise FileNotFoundError("no state.json yet") |
| target = os.path.join(local_dir, filename) |
| open(target, "wb").close() |
| return target |
|
|
| monkeypatch.setattr("sync.hf_dataset_store.hf_hub_download", fake_hf_hub_download) |
| state = download_snapshot("org/repo", str(tmp_path), token="tok") |
| assert state == {"papers": {}, "last_synced_at": None} |
| assert not os.path.exists(tmp_path / "state.json") |
|
|
| def test_download_snapshot_handles_empty_remote_dataset_repo(tmp_path, monkeypatch): |
| """Reproduces a real first-time sync against a brand-new, empty dataset repo: |
| huggingface_hub raises RemoteEntryNotFoundError (an HTTP 404), not a plain |
| FileNotFoundError, when a file doesn't exist yet on the Hub. Missing files |
| must be left absent (not stubbed with an empty placeholder), since a 0-byte |
| file is not a valid "missing" signal to faiss.read_index / sqlite3.connect.""" |
| def fake_hf_hub_download(repo_id, repo_type, filename, local_dir, token): |
| raise _remote_entry_not_found() |
|
|
| monkeypatch.setattr("sync.hf_dataset_store.hf_hub_download", fake_hf_hub_download) |
| state = download_snapshot("org/repo", str(tmp_path), token="tok") |
| assert state == {"papers": {}, "last_synced_at": None} |
| assert not os.path.exists(tmp_path / "index.faiss") |
| assert not os.path.exists(tmp_path / "papers.db") |
|
|
| |
| with pytest.raises(FileNotFoundError): |
| load_index(str(tmp_path / "index.faiss")) |
| conn = init_db(str(tmp_path / "papers.db")) |
| assert conn.execute("SELECT COUNT(*) FROM papers").fetchone()[0] == 0 |
|
|
| def test_upload_snapshot_uploads_folder_as_single_atomic_commit(tmp_path, monkeypatch): |
| for name in ["index.faiss", "papers.db", "state.json"]: |
| (tmp_path / name).write_text("data") |
|
|
| calls = [] |
|
|
| def fake_upload_folder(**kwargs): |
| calls.append(kwargs) |
|
|
| monkeypatch.setattr("sync.hf_dataset_store.upload_folder", fake_upload_folder) |
| upload_snapshot("org/repo", str(tmp_path), token="tok") |
| assert len(calls) == 1 |
| assert calls[0]["repo_id"] == "org/repo" |
| assert calls[0]["repo_type"] == "dataset" |
| assert calls[0]["folder_path"] == str(tmp_path) |
| assert calls[0]["token"] == "tok" |
|
|
