Upload 11 files

gateway.py

@@ -151,8 +151,11 @@ class GatewayHandler(BaseHTTPRequestHandler):
         self.wfile.write(body)
 
     def log_message(self, format, *args):
-        """Cleaner log format."""
-        sys.stderr.write(f"[gateway] {self.address_string()} {format % args}\n")
+        """Cleaner log format. Ignores HF container ping logs for cleanliness."""
+        msg = format % args
+        if "?logs=container" in msg:
+            return
+        sys.stderr.write(f"[gateway] {self.address_string()} {msg}\n")
 
 
 def wait_for_clickhouse(timeout=60):

refresh_sources.py

@@ -107,6 +107,13 @@ def main():
             cmd = ["git", "clone", "--depth", clone_depth] + branch_flag + auth_flag + [repo_url, target_dir]
             output, success = run_subprocess(cmd)
             git_status = "cloned" if success else f"error_clone: {output}"
+            
+            # Retrieve real files for Git LFS pointers natively used by Hugging Face Datasets
+            if success:
+                cmd_lfs = ["git", "lfs", "pull"]
+                out_lfs, suc_lfs = run_subprocess(cmd_lfs, cwd=target_dir)
+                if not suc_lfs:
+                    git_status = f"error_lfs_pull: {out_lfs}"
         else:
             # We want to fetch and hard reset
             cmd_fetch = ["git"] + auth_flag + ["fetch", "--depth", "1", "origin"]