Spaces:
Running on Zero
Running on Zero
Commit ·
818b129
1
Parent(s): ca0e7d6
Pass now to _prune_old_files for deterministic boundary testing
Browse filesMirrors the same pattern used by _build_table. Tests now use a fixed
NOW constant and relative offsets, enabling precise boundary checks
(exactly keep_days old is kept, keep_days+1 is deleted).
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
- logging_utils.py +3 -3
- tests/test_logging_utils.py +36 -26
logging_utils.py
CHANGED
|
@@ -96,10 +96,10 @@ def _upload_parquet(api, repo_id, table, path_in_repo):
|
|
| 96 |
pass
|
| 97 |
|
| 98 |
|
| 99 |
-
def _prune_old_files(api, repo_id, keep_days):
|
| 100 |
if keep_days <= 0:
|
| 101 |
return
|
| 102 |
-
cutoff = (
|
| 103 |
try:
|
| 104 |
to_delete = [
|
| 105 |
f.path
|
|
@@ -133,7 +133,7 @@ def log_inference(pil_inputs, output_pil, prompt, seed, steps, guidance_scale,
|
|
| 133 |
api = HfApi(token=HF_TOKEN)
|
| 134 |
api.create_repo(repo_id=DATASET_REPO, repo_type="dataset", private=True, exist_ok=True)
|
| 135 |
_upload_parquet(api, DATASET_REPO, table, path_in_repo)
|
| 136 |
-
_prune_old_files(api, DATASET_REPO, MAX_LOG_DAYS)
|
| 137 |
except Exception as log_err:
|
| 138 |
import traceback as _tb
|
| 139 |
print(f"[log] WARNING: {log_err}\n{_tb.format_exc()}")
|
|
|
|
| 96 |
pass
|
| 97 |
|
| 98 |
|
| 99 |
+
def _prune_old_files(api, repo_id, keep_days, now):
|
| 100 |
if keep_days <= 0:
|
| 101 |
return
|
| 102 |
+
cutoff = (now - timedelta(days=keep_days)).strftime("%Y-%m-%d")
|
| 103 |
try:
|
| 104 |
to_delete = [
|
| 105 |
f.path
|
|
|
|
| 133 |
api = HfApi(token=HF_TOKEN)
|
| 134 |
api.create_repo(repo_id=DATASET_REPO, repo_type="dataset", private=True, exist_ok=True)
|
| 135 |
_upload_parquet(api, DATASET_REPO, table, path_in_repo)
|
| 136 |
+
_prune_old_files(api, DATASET_REPO, MAX_LOG_DAYS, now)
|
| 137 |
except Exception as log_err:
|
| 138 |
import traceback as _tb
|
| 139 |
print(f"[log] WARNING: {log_err}\n{_tb.format_exc()}")
|
tests/test_logging_utils.py
CHANGED
|
@@ -1,68 +1,78 @@
|
|
| 1 |
from datetime import datetime, timezone, timedelta
|
| 2 |
from types import SimpleNamespace
|
| 3 |
-
from unittest.mock import MagicMock
|
| 4 |
|
| 5 |
from logging_utils import _prune_old_files
|
| 6 |
|
|
|
|
|
|
|
|
|
|
| 7 |
|
| 8 |
def _make_api(paths):
|
| 9 |
-
"""Return a mock HfApi whose list_repo_tree yields RepoFile-like objects for `paths`."""
|
| 10 |
api = MagicMock()
|
| 11 |
api.list_repo_tree.return_value = [SimpleNamespace(path=p) for p in paths]
|
| 12 |
return api
|
| 13 |
|
| 14 |
|
| 15 |
-
def
|
| 16 |
-
|
|
|
|
| 17 |
|
| 18 |
|
| 19 |
-
def
|
| 20 |
-
|
| 21 |
-
return f"data/{today}-000000-{name}.parquet"
|
| 22 |
|
| 23 |
|
| 24 |
def test_old_files_are_deleted():
|
| 25 |
-
api = _make_api([
|
| 26 |
-
_prune_old_files(api, "org/repo",
|
| 27 |
-
assert api
|
| 28 |
-
deleted = {c.kwargs["path_in_repo"] for c in api.delete_file.call_args_list}
|
| 29 |
-
assert deleted == {_old("aaa"), _old("bbb")}
|
| 30 |
|
| 31 |
|
| 32 |
def test_recent_files_are_kept():
|
| 33 |
-
api = _make_api([
|
| 34 |
-
_prune_old_files(api, "org/repo",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 35 |
api.delete_file.assert_not_called()
|
| 36 |
|
| 37 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 38 |
def test_only_old_files_deleted_in_mixed_set():
|
| 39 |
-
api = _make_api([
|
| 40 |
-
_prune_old_files(api, "org/repo",
|
| 41 |
-
|
| 42 |
-
assert deleted == {_old("aaa"), _old("ccc")}
|
| 43 |
|
| 44 |
|
| 45 |
def test_empty_directory_does_nothing():
|
| 46 |
api = _make_api([])
|
| 47 |
-
_prune_old_files(api, "org/repo",
|
| 48 |
api.delete_file.assert_not_called()
|
| 49 |
|
| 50 |
|
| 51 |
def test_keep_days_zero_skips_pruning():
|
| 52 |
-
api = _make_api([
|
| 53 |
-
_prune_old_files(api, "org/repo", keep_days=0)
|
| 54 |
api.list_repo_tree.assert_not_called()
|
| 55 |
api.delete_file.assert_not_called()
|
| 56 |
|
| 57 |
|
| 58 |
def test_non_parquet_files_are_ignored():
|
| 59 |
-
api = _make_api(["data/2020-01-01-README.md",
|
| 60 |
-
_prune_old_files(api, "org/repo",
|
| 61 |
-
|
| 62 |
-
assert deleted == {_old("aaa")}
|
| 63 |
|
| 64 |
|
| 65 |
def test_api_error_does_not_raise():
|
| 66 |
api = MagicMock()
|
| 67 |
api.list_repo_tree.side_effect = RuntimeError("network error")
|
| 68 |
-
_prune_old_files(api, "org/repo",
|
|
|
|
| 1 |
from datetime import datetime, timezone, timedelta
|
| 2 |
from types import SimpleNamespace
|
| 3 |
+
from unittest.mock import MagicMock
|
| 4 |
|
| 5 |
from logging_utils import _prune_old_files
|
| 6 |
|
| 7 |
+
NOW = datetime(2026, 5, 17, 12, 0, 0, tzinfo=timezone.utc)
|
| 8 |
+
KEEP_DAYS = 7
|
| 9 |
+
|
| 10 |
|
| 11 |
def _make_api(paths):
|
|
|
|
| 12 |
api = MagicMock()
|
| 13 |
api.list_repo_tree.return_value = [SimpleNamespace(path=p) for p in paths]
|
| 14 |
return api
|
| 15 |
|
| 16 |
|
| 17 |
+
def _file(days_ago):
|
| 18 |
+
date = (NOW - timedelta(days=days_ago)).strftime("%Y-%m-%d")
|
| 19 |
+
return f"data/{date}-000000-abcd1234.parquet"
|
| 20 |
|
| 21 |
|
| 22 |
+
def _deleted_paths(api):
|
| 23 |
+
return {c.kwargs["path_in_repo"] for c in api.delete_file.call_args_list}
|
|
|
|
| 24 |
|
| 25 |
|
| 26 |
def test_old_files_are_deleted():
|
| 27 |
+
api = _make_api([_file(8), _file(10)])
|
| 28 |
+
_prune_old_files(api, "org/repo", KEEP_DAYS, NOW)
|
| 29 |
+
assert _deleted_paths(api) == {_file(8), _file(10)}
|
|
|
|
|
|
|
| 30 |
|
| 31 |
|
| 32 |
def test_recent_files_are_kept():
|
| 33 |
+
api = _make_api([_file(6), _file(1), _file(0)])
|
| 34 |
+
_prune_old_files(api, "org/repo", KEEP_DAYS, NOW)
|
| 35 |
+
api.delete_file.assert_not_called()
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def test_boundary_exactly_keep_days_old_is_kept():
|
| 39 |
+
api = _make_api([_file(KEEP_DAYS)])
|
| 40 |
+
_prune_old_files(api, "org/repo", KEEP_DAYS, NOW)
|
| 41 |
api.delete_file.assert_not_called()
|
| 42 |
|
| 43 |
|
| 44 |
+
def test_one_day_past_boundary_is_deleted():
|
| 45 |
+
api = _make_api([_file(KEEP_DAYS + 1)])
|
| 46 |
+
_prune_old_files(api, "org/repo", KEEP_DAYS, NOW)
|
| 47 |
+
assert _deleted_paths(api) == {_file(KEEP_DAYS + 1)}
|
| 48 |
+
|
| 49 |
+
|
| 50 |
def test_only_old_files_deleted_in_mixed_set():
|
| 51 |
+
api = _make_api([_file(8), _file(6), _file(10), _file(0)])
|
| 52 |
+
_prune_old_files(api, "org/repo", KEEP_DAYS, NOW)
|
| 53 |
+
assert _deleted_paths(api) == {_file(8), _file(10)}
|
|
|
|
| 54 |
|
| 55 |
|
| 56 |
def test_empty_directory_does_nothing():
|
| 57 |
api = _make_api([])
|
| 58 |
+
_prune_old_files(api, "org/repo", KEEP_DAYS, NOW)
|
| 59 |
api.delete_file.assert_not_called()
|
| 60 |
|
| 61 |
|
| 62 |
def test_keep_days_zero_skips_pruning():
|
| 63 |
+
api = _make_api([_file(8)])
|
| 64 |
+
_prune_old_files(api, "org/repo", keep_days=0, now=NOW)
|
| 65 |
api.list_repo_tree.assert_not_called()
|
| 66 |
api.delete_file.assert_not_called()
|
| 67 |
|
| 68 |
|
| 69 |
def test_non_parquet_files_are_ignored():
|
| 70 |
+
api = _make_api(["data/2020-01-01-README.md", _file(8)])
|
| 71 |
+
_prune_old_files(api, "org/repo", KEEP_DAYS, NOW)
|
| 72 |
+
assert _deleted_paths(api) == {_file(8)}
|
|
|
|
| 73 |
|
| 74 |
|
| 75 |
def test_api_error_does_not_raise():
|
| 76 |
api = MagicMock()
|
| 77 |
api.list_repo_tree.side_effect = RuntimeError("network error")
|
| 78 |
+
_prune_old_files(api, "org/repo", KEEP_DAYS, NOW) # must not raise
|