Write one parquet file per inference to avoid Git LFS version bloat

Previously the logger downloaded and rewrote the same daily parquet file
on every inference, creating a new LFS object each time and inflating
private storage far beyond the actual data size. Now each inference writes
a uniquely-named file (data/YYYY-MM-DD-HHMMSS-{uid}.parquet) that is
committed once and never overwritten.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

logging_utils.py

@@ -1,4 +1,5 @@
 import os
+import uuid
 from io import BytesIO
 from datetime import datetime, timezone
 
@@ -6,35 +7,19 @@ HF_TOKEN     = os.environ.get("HF_TOKEN")
 DATASET_REPO = os.environ.get("LOG_DATASET_REPO")
 
 
-def _readme_from_features(feats):
-    lines = ["---", "configs:", "- config_name: default",
-             "  data_files:", "  - split: train",
-             "    path: data/*.parquet", "  features:"]
-    for name, f in feats.items():
-        lines.append(f"  - name: {name}")
-        if f.get("_type") == "Image":
-            lines.append("    dtype: image")
-        elif f.get("_type") == "Sequence" and f.get("feature", {}).get("_type") == "Image":
-            lines.append("    sequence: image")
-        else:
-            lines.append(f"    dtype: {f.get('dtype', 'string')}")
-    lines.append("---")
-    return "\n".join(lines) + "\n"
-
-
 def log_inference(pil_inputs, output_pil, prompt, seed, steps, guidance_scale,
                   input_width, input_height, duration_seconds, success, error_message=""):
     if not HF_TOKEN or not DATASET_REPO:
         print(f"[log] skipped — HF_TOKEN={'set' if HF_TOKEN else 'missing'}, DATASET_REPO={'set' if DATASET_REPO else 'missing'}")
         return
+    tmp_path = None
     try:
-        import tempfile, json as _json
+        import tempfile
+        import json as _json
         import pyarrow as pa
         import pyarrow.parquet as pq
-        from huggingface_hub import HfApi, hf_hub_download
+        from huggingface_hub import HfApi
 
-        # Image columns need Arrow struct {bytes: binary, path: utf8} plus
-        # a 'huggingface' schema metadata key for the HF viewer to render them.
         img_struct = pa.struct([("bytes", pa.binary()), ("path", pa.string())])
         hf_meta = _json.dumps({"info": {"features": {
             "timestamp":        {"dtype": "float64", "_type": "Value"},
@@ -75,57 +60,35 @@ def log_inference(pil_inputs, output_pil, prompt, seed, steps, guidance_scale,
         def _img(b):
             return {"bytes": b, "path": None}
 
+        now = datetime.now(timezone.utc)
         input_jpegs = [_to_jpeg(img) for img in pil_inputs]
         output_jpeg = _to_jpeg(output_pil)
 
-        new_table = pa.table({
-            "timestamp":        pa.array([datetime.now(timezone.utc).timestamp()], type=pa.float64()),
-            "prompt":           pa.array([prompt],              type=pa.string()),
-            "seed":             pa.array([int(seed)],           type=pa.int32()),
-            "steps":            pa.array([int(steps)],          type=pa.int32()),
-            "guidance_scale":   pa.array([float(guidance_scale)], type=pa.float32()),
-            "input_images":     pa.array([[_img(b) for b in input_jpegs]], type=pa.list_(img_struct)),
+        table = pa.table({
+            "timestamp":        pa.array([now.timestamp()],                    type=pa.float64()),
+            "prompt":           pa.array([prompt],                             type=pa.string()),
+            "seed":             pa.array([int(seed)],                          type=pa.int32()),
+            "steps":            pa.array([int(steps)],                         type=pa.int32()),
+            "guidance_scale":   pa.array([float(guidance_scale)],              type=pa.float32()),
+            "input_images":     pa.array([[_img(b) for b in input_jpegs]],     type=pa.list_(img_struct)),
             "output_image":     pa.array([_img(output_jpeg) if output_jpeg else None], type=img_struct),
-            "duration_seconds": pa.array([float(duration_seconds)], type=pa.float32()),
-            "input_width":      pa.array([int(input_width)],   type=pa.int32()),
-            "input_height":     pa.array([int(input_height)],  type=pa.int32()),
-            "success":          pa.array([bool(success)],      type=pa.bool_()),
-            "error_message":    pa.array([str(error_message)], type=pa.string()),
+            "duration_seconds": pa.array([float(duration_seconds)],            type=pa.float32()),
+            "input_width":      pa.array([int(input_width)],                   type=pa.int32()),
+            "input_height":     pa.array([int(input_height)],                  type=pa.int32()),
+            "success":          pa.array([bool(success)],                      type=pa.bool_()),
+            "error_message":    pa.array([str(error_message)],                 type=pa.string()),
         }, schema=schema)
-        print(f"[log] built row — success={success}, inputs={len(input_jpegs)}")
-
-        today = datetime.now(timezone.utc).strftime("%Y-%m-%d")
-        path_in_repo = f"data/{today}.parquet"
-        api = HfApi(token=HF_TOKEN)
-        api.create_repo(repo_id=DATASET_REPO, repo_type="dataset", private=True, exist_ok=True)
 
-        # Upload README.md once so the HF viewer knows the column types.
-        try:
-            hf_hub_download(repo_id=DATASET_REPO, filename="README.md",
-                            repo_type="dataset", token=HF_TOKEN)
-        except Exception:
-            readme = _readme_from_features(_json.loads(hf_meta)["info"]["features"])
-            api.upload_file(path_or_fileobj=readme.encode(), path_in_repo="README.md",
-                            repo_id=DATASET_REPO, repo_type="dataset")
-            print("[log] uploaded README.md with feature schema")
-
-        try:
-            local_path = hf_hub_download(
-                repo_id=DATASET_REPO, filename=path_in_repo,
-                repo_type="dataset", token=HF_TOKEN,
-            )
-            existing = pq.read_table(local_path)
-            combined = pa.concat_tables([existing, new_table])
-            combined = combined.replace_schema_metadata(schema.metadata)
-            print(f"[log] appending to existing {existing.num_rows} row(s)")
-        except Exception as dl_err:
-            print(f"[log] no existing file ({dl_err}), starting fresh")
-            combined = new_table
+        uid = uuid.uuid4().hex[:8]
+        path_in_repo = f"data/{now.strftime('%Y-%m-%d-%H%M%S')}-{uid}.parquet"
 
         with tempfile.NamedTemporaryFile(suffix=".parquet", delete=False) as tmp:
             tmp_path = tmp.name
-        pq.write_table(combined, tmp_path)
-        print(f"[log] uploading {path_in_repo} ({combined.num_rows} row(s), {os.path.getsize(tmp_path)//1024}KB)")
+        pq.write_table(table, tmp_path)
+
+        api = HfApi(token=HF_TOKEN)
+        api.create_repo(repo_id=DATASET_REPO, repo_type="dataset", private=True, exist_ok=True)
+        print(f"[log] uploading {path_in_repo} ({os.path.getsize(tmp_path)//1024}KB)")
         api.upload_file(
             path_or_fileobj=tmp_path, path_in_repo=path_in_repo,
             repo_id=DATASET_REPO, repo_type="dataset",
@@ -134,3 +97,9 @@ def log_inference(pil_inputs, output_pil, prompt, seed, steps, guidance_scale,
     except Exception as log_err:
         import traceback as _tb
         print(f"[log] WARNING: {log_err}\n{_tb.format_exc()}")
+    finally:
+        if tmp_path:
+            try:
+                os.unlink(tmp_path)
+            except Exception:
+                pass