Upload folder using huggingface_hub

app.py

@@ -196,12 +196,15 @@ def build_and_push(
     tags: list = None,
     build_args: dict = None,
 ) -> bool:
-    """Build Docker image using Kaniko and push to registry."""
+    """Build Docker image using Kaniko and push to registry.
+
+    Uses tar context to avoid Kaniko's 'failed to get files used from context'
+    bug with multi-stage Dockerfiles that use FROM <previous_stage>.
+    """
 
     if tags is None:
         tags = ["latest"]
 
-    # Full image name with registry
     full_image = f"{REGISTRY_URL}/{image_name}"
 
     log(f"Building {full_image}...")
@@ -217,11 +220,29 @@ def build_and_push(
             "started_at": datetime.now(timezone.utc).isoformat(),
         }
 
-    # Build Kaniko command
-    # Note: dockerfile path should be relative to context for multi-stage builds
+    # Create tar of context - this avoids Kaniko's dir:// context resolution bug
+    # that causes "failed to get files used from context" on multi-stage builds
+    tar_path = f"{context_dir}.tar.gz"
+    log(f"Creating tar context: {tar_path}")
+    try:
+        tar_result = subprocess.run(
+            ["tar", "-czf", tar_path, "-C", context_dir, "."],
+            capture_output=True,
+            text=True,
+            timeout=120,
+        )
+        if tar_result.returncode != 0:
+            log(f"✗ Failed to create tar: {tar_result.stderr}")
+            return False
+        log(f"✓ Tar created")
+    except Exception as e:
+        log(f"✗ Tar error: {e}")
+        return False
+
+    # Build Kaniko command with tar context
     cmd = [
         "/kaniko/executor",
-        f"--context=dir://{context_dir}",
+        f"--context=tar://{tar_path}",
         f"--dockerfile={dockerfile}",
     ]
 
@@ -236,10 +257,9 @@ def build_and_push(
 
     # Kaniko options
     cmd.extend([
-        "--cache=false",  # Disabled due to HF Space network issues
+        "--cache=false",
         "--ignore-path=/product_uuid",
         "--ignore-path=/sys",
-        "--verbosity=info",
     ])
 
     log(f"Executing: {' '.join(cmd[:5])}...")
@@ -295,6 +315,120 @@ def build_and_push(
             state["current_build"] = None
 
 
+def build_and_push_git(
+    git_context: str,
+    image_name: str,
+    dockerfile: str = "Dockerfile",
+    context_subpath: str = None,
+    tags: list = None,
+    build_args: dict = None,
+) -> bool:
+    """Build Docker image using Kaniko with git context source."""
+
+    if tags is None:
+        tags = ["latest"]
+
+    full_image = f"{REGISTRY_URL}/{image_name}"
+
+    # Mask token in logs
+    safe_context = git_context
+    if "@github.com" in git_context:
+        safe_context = git_context.split("@")[0][:10] + "...@github.com" + git_context.split("@github.com")[1]
+
+    log(f"Building {full_image}...")
+    log(f"  Git context: {safe_context}")
+    log(f"  Dockerfile: {dockerfile}")
+    log(f"  Tags: {tags}")
+
+    with state_lock:
+        state["status"] = "building"
+        state["current_build"] = {
+            "image": full_image,
+            "tags": tags,
+            "started_at": datetime.now(timezone.utc).isoformat(),
+        }
+
+    # Build Kaniko command with git context
+    cmd = [
+        "/kaniko/executor",
+        f"--context={git_context}",
+        f"--dockerfile={dockerfile}",
+    ]
+
+    if context_subpath:
+        cmd.append(f"--context-sub-path={context_subpath}")
+
+    # Add destination tags
+    for tag in tags:
+        cmd.append(f"--destination={full_image}:{tag}")
+
+    # Add build args
+    if build_args:
+        for key, value in build_args.items():
+            cmd.append(f"--build-arg={key}={value}")
+
+    # Kaniko options
+    cmd.extend([
+        "--cache=false",
+        "--ignore-path=/product_uuid",
+        "--ignore-path=/sys",
+    ])
+
+    log(f"Executing kaniko with git context...")
+
+    try:
+        process = subprocess.Popen(
+            cmd,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            text=True,
+        )
+
+        for line in process.stdout:
+            line = line.strip()
+            if line:
+                # Mask any tokens in output
+                if GITHUB_TOKEN and GITHUB_TOKEN in line:
+                    line = line.replace(GITHUB_TOKEN, "***")
+                log(f"  {line[:100]}")
+
+        process.wait()
+
+        if process.returncode == 0:
+            log(f"✓ Build successful: {full_image}")
+            with state_lock:
+                state["builds_completed"] += 1
+                state["last_build"] = {
+                    "image": full_image,
+                    "tags": tags,
+                    "status": "success",
+                    "completed_at": datetime.now(timezone.utc).isoformat(),
+                }
+            return True
+        else:
+            log(f"✗ Build failed with exit code {process.returncode}")
+            with state_lock:
+                state["builds_failed"] += 1
+                state["last_build"] = {
+                    "image": full_image,
+                    "tags": tags,
+                    "status": "failed",
+                    "exit_code": process.returncode,
+                    "completed_at": datetime.now(timezone.utc).isoformat(),
+                }
+            return False
+
+    except Exception as e:
+        log(f"✗ Build error: {e}")
+        with state_lock:
+            state["builds_failed"] += 1
+        return False
+    finally:
+        with state_lock:
+            state["status"] = "idle"
+            state["current_build"] = None
+
+
 def run_build(build_config: dict) -> bool:
     """Run a build from config."""
     repo_url = build_config.get("repo_url")
@@ -305,20 +439,49 @@ def run_build(build_config: dict) -> bool:
     tags = build_config.get("tags", ["latest"])
     build_args = build_config.get("build_args", {})
     github_token = build_config.get("github_token")  # Optional per-build token
+    # Default to tar context (clone + tar) for reliability with multi-stage builds
+    use_git_context = build_config.get("use_git_context", False)
 
     if not repo_url or not image_name:
         log("✗ Missing repo_url or image_name")
         return False
 
+    # Option: Use git context source (Kaniko handles clone internally)
+    # Note: git context can have issues with multi-stage builds, prefer tar context
+    if use_git_context and "github.com" in repo_url:
+        token = github_token or GITHUB_TOKEN
+        if token:
+            git_context = repo_url.replace("https://github.com", f"git://{token}@github.com")
+        else:
+            git_context = repo_url.replace("https://github.com", "git://github.com")
+        git_context = f"{git_context}#refs/heads/{branch}"
+
+        if context_path and context_path != ".":
+            return build_and_push_git(
+                git_context=git_context,
+                image_name=image_name,
+                dockerfile=dockerfile,
+                context_subpath=context_path,
+                tags=tags,
+                build_args=build_args,
+            )
+        else:
+            return build_and_push_git(
+                git_context=git_context,
+                image_name=image_name,
+                dockerfile=dockerfile,
+                tags=tags,
+                build_args=build_args,
+            )
+
+    # Default: clone repo, create tar context (most reliable for multi-stage builds)
     tmpdir = None
+    tar_path = None
     try:
-        # Clone repo
         tmpdir = clone_repo(repo_url, branch, github_token=github_token)
-
-        # Set context directory
         context_dir = str(tmpdir / context_path) if context_path != "." else str(tmpdir)
 
-        # Build and push
+        tar_path = f"{context_dir}.tar.gz"  # Track for cleanup
         return build_and_push(
             context_dir=context_dir,
             image_name=image_name,
@@ -328,8 +491,11 @@ def run_build(build_config: dict) -> bool:
         )
 
     finally:
+        # Cleanup: remove cloned repo and tar file
         if tmpdir and tmpdir.exists():
             shutil.rmtree(tmpdir, ignore_errors=True)
+        if tar_path and os.path.exists(tar_path):
+            os.remove(tar_path)
 
 
 # =============================================================================