someone-in-the-world Claude Sonnet 4.6 commited on
Commit
bc828ab
·
1 Parent(s): c149547

Prune log files before uploading, not after

Browse files

The upload was failing with the 10k-files-per-directory error because
pruning ran after the upload attempt. Moving prune first clears space
so the new file can always be written.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

Files changed (1) hide show
  1. logging_utils.py +4 -4
logging_utils.py CHANGED
@@ -176,12 +176,12 @@ def log_inference(pil_inputs, output_pil, prompt, seed, steps, guidance_scale,
176
  print(f"[log] create_repo: {_time.perf_counter() - _t2:.3f}s")
177
 
178
  _t3 = _time.perf_counter()
179
- _upload_parquet(api, DATASET_REPO, table, path_in_repo)
180
- print(f"[log] upload_parquet: {_time.perf_counter() - _t3:.3f}s")
181
 
182
  _t4 = _time.perf_counter()
183
- _prune_old_files(api, DATASET_REPO, MAX_LOG_FILES)
184
- print(f"[log] prune_old_files: {_time.perf_counter() - _t4:.3f}s")
185
 
186
  _t5 = _time.perf_counter()
187
  _maybe_squash_history(api, DATASET_REPO, now)
 
176
  print(f"[log] create_repo: {_time.perf_counter() - _t2:.3f}s")
177
 
178
  _t3 = _time.perf_counter()
179
+ _prune_old_files(api, DATASET_REPO, MAX_LOG_FILES)
180
+ print(f"[log] prune_old_files: {_time.perf_counter() - _t3:.3f}s")
181
 
182
  _t4 = _time.perf_counter()
183
+ _upload_parquet(api, DATASET_REPO, table, path_in_repo)
184
+ print(f"[log] upload_parquet: {_time.perf_counter() - _t4:.3f}s")
185
 
186
  _t5 = _time.perf_counter()
187
  _maybe_squash_history(api, DATASET_REPO, now)