Spaces:
Running on Zero
Running on Zero
File size: 5,197 Bytes
ca0e7d6 818b129 ca0e7d6 2f63460 ca0e7d6 818b129 c149547 818b129 ca0e7d6 818b129 9235827 ca0e7d6 2f63460 ca0e7d6 2f63460 ca0e7d6 9235827 2f63460 9235827 2f63460 c149547 2f63460 ca0e7d6 c149547 2f63460 818b129 c149547 2f63460 ca0e7d6 c149547 2f63460 ca0e7d6 2f63460 ca0e7d6 2f63460 ca0e7d6 2f63460 ca0e7d6 2f63460 c149547 2f63460 0597d39 2f63460 0597d39 2f63460 0597d39 2f63460 0597d39 2f63460 0597d39 2f63460 0597d39 2f63460 0597d39 2f63460 0597d39 2f63460 0597d39 2f63460 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 | from datetime import datetime, timezone, timedelta
from types import SimpleNamespace
from unittest.mock import MagicMock
from huggingface_hub import CommitOperationAdd, CommitOperationDelete
from logging_utils import (
_build_add_ops, _build_delete_ops, _make_path, _squash_if_needed,
)
NOW = datetime(2026, 5, 17, 12, 0, 0, tzinfo=timezone.utc)
KEEP_COUNT = 10
def _file(days_ago):
return _make_path(NOW - timedelta(days=days_ago), "abcd1234")
# ── _build_add_ops ────────────────────────────────────────────────────────────
def test_build_add_ops_returns_one_op_per_entry(tmp_path):
files = [tmp_path / "a.parquet", tmp_path / "b.parquet"]
for f in files:
f.write_bytes(b"")
batch = [("data/a.parquet", str(files[0])), ("data/b.parquet", str(files[1]))]
ops = _build_add_ops(batch)
assert len(ops) == 2
assert all(isinstance(op, CommitOperationAdd) for op in ops)
assert [op.path_in_repo for op in ops] == ["data/a.parquet", "data/b.parquet"]
def test_build_add_ops_empty_batch():
assert _build_add_ops([]) == []
# ── _build_delete_ops ─────────────────────────────────────────────────────────
def test_oldest_files_deleted_when_over_limit():
existing = sorted([_file(i) for i in range(12)]) # 12 files
ops = _build_delete_ops(existing, n_new=0, max_files=KEEP_COUNT)
deleted = {op.path_in_repo for op in ops}
assert len(deleted) == 2
assert deleted == {_file(11), _file(10)}
def test_files_kept_when_under_limit():
existing = [_file(i) for i in range(5)]
assert _build_delete_ops(existing, n_new=0, max_files=KEEP_COUNT) == []
def test_exactly_at_limit_nothing_deleted():
existing = [_file(i) for i in range(KEEP_COUNT)]
assert _build_delete_ops(existing, n_new=0, max_files=KEEP_COUNT) == []
def test_one_over_limit_oldest_deleted():
existing = sorted([_file(i) for i in range(KEEP_COUNT + 1)])
ops = _build_delete_ops(existing, n_new=0, max_files=KEEP_COUNT)
assert {op.path_in_repo for op in ops} == {_file(KEEP_COUNT)}
def test_empty_existing_does_nothing():
assert _build_delete_ops([], n_new=0, max_files=KEEP_COUNT) == []
def test_max_files_zero_skips_pruning():
existing = [_file(0)]
assert _build_delete_ops(existing, n_new=0, max_files=0) == []
def test_n_new_counted_toward_total():
# 8 existing + 4 new = 12 total, need to delete 2
existing = sorted([_file(i) for i in range(8)])
ops = _build_delete_ops(existing, n_new=4, max_files=KEEP_COUNT)
assert len(ops) == 2
def test_oldest_files_deleted_regardless_of_input_order():
files = [_file(3), _file(11), _file(0), _file(10), _file(1),
_file(2), _file(4), _file(5), _file(6), _file(7), _file(8)]
existing = sorted(files) # caller sorts before passing
ops = _build_delete_ops(existing, n_new=0, max_files=KEEP_COUNT)
assert {op.path_in_repo for op in ops} == {_file(11)}
def test_all_ops_are_delete_type():
existing = sorted([_file(i) for i in range(12)])
ops = _build_delete_ops(existing, n_new=0, max_files=KEEP_COUNT)
assert all(isinstance(op, CommitOperationDelete) for op in ops)
# ── _squash_if_needed ─────────────────────────────────────────────────────────
def _squash_api():
api = MagicMock()
api.token = "tok"
return api
def test_squash_runs_when_no_marker(monkeypatch):
monkeypatch.setattr(
"logging_utils.hf_hub_download",
MagicMock(side_effect=FileNotFoundError("no marker")),
)
api = _squash_api()
_squash_if_needed(api, "org/repo")
api.super_squash_history.assert_called_once()
api.upload_file.assert_called_once()
def test_squash_skipped_when_marker_is_today(monkeypatch, tmp_path):
today = datetime.now(timezone.utc).strftime("%Y-%m-%d")
marker = tmp_path / "last_squash.txt"
marker.write_text(today)
monkeypatch.setattr("logging_utils.hf_hub_download", MagicMock(return_value=str(marker)))
api = _squash_api()
_squash_if_needed(api, "org/repo")
api.super_squash_history.assert_not_called()
def test_squash_runs_when_marker_is_yesterday(monkeypatch, tmp_path):
yesterday = (datetime.now(timezone.utc) - timedelta(days=1)).strftime("%Y-%m-%d")
marker = tmp_path / "last_squash.txt"
marker.write_text(yesterday)
monkeypatch.setattr("logging_utils.hf_hub_download", MagicMock(return_value=str(marker)))
api = _squash_api()
_squash_if_needed(api, "org/repo")
api.super_squash_history.assert_called_once()
def test_squash_error_does_not_raise(monkeypatch):
monkeypatch.setattr(
"logging_utils.hf_hub_download",
MagicMock(side_effect=FileNotFoundError("no marker")),
)
api = _squash_api()
api.super_squash_history.side_effect = RuntimeError("squash failed")
_squash_if_needed(api, "org/repo") # must not raise
|