import os import sys import time import shutil import tempfile import traceback from pathlib import Path from typing import Generator, List, Tuple, Dict, Any import gradio as gr from huggingface_hub import HfApi, create_repo, create_bucket from huggingface_hub.utils import HfHubHTTPError, RepositoryNotFoundError, EntryNotFoundError def get_token(custom_token: str | None = None) -> str | None: """Return explicit custom token if provided, else fall back to HF_TOKEN env var.""" if custom_token and custom_token.strip(): return custom_token.strip() return os.environ.get("HF_TOKEN") or None def format_size(size_bytes: int | float | None) -> str: """Format byte count into human-readable string.""" if size_bytes is None: return "Unknown size" try: size = float(size_bytes) except (ValueError, TypeError): return "Unknown size" if size < 1024: return f"{int(size)} B" elif size < 1024 * 1024: return f"{size / 1024:.1f} KB" elif size < 1024 * 1024 * 1024: return f"{size / (1024 * 1024):.1f} MB" else: return f"{size / (1024 * 1024 * 1024):.2f} GB" def clean_identifier(uri_or_id: str, kind: str = "bucket") -> str: """Normalize user input by stripping hf:// URIs and prefixes.""" val = (uri_or_id or "").strip() if not val: return "" # Strip hf:// URI prefixes if user copied full URI if val.startswith("hf://"): val = val[len("hf://"):] if val.startswith("buckets/"): val = val[len("buckets/"):] elif val.startswith("datasets/"): val = val[len("datasets/"):] elif val.startswith("models/"): val = val[len("models/"):] return val.strip("/") def get_user_buckets(custom_token: str | None = None) -> List[str]: """Fetch existing buckets for the logged-in user or orgs.""" token = get_token(custom_token) api = HfApi(token=token) buckets = [] try: user_info = api.whoami() username = user_info.get("name") orgs = [org["name"] for org in user_info.get("orgs", []) if "name" in org] namespaces = [username] + orgs if username else [] for ns in namespaces: try: for b in api.list_buckets(namespace=ns): buckets.append(b.id) except Exception: pass except Exception: # Fallback to general list if whoami fails try: for b in api.list_buckets(): buckets.append(b.id) except Exception: pass return sorted(list(set(buckets))) def list_files_in_bucket( bucket_input: str, prefix_filter: str, only_safetensors: bool = True, custom_token: str | None = None ) -> Tuple[Any, str, Dict[str, Any]]: """List files in the selected source bucket and prepare CheckboxGroup choices.""" bucket_id = clean_identifier(bucket_input, kind="bucket") if not bucket_id: return gr.update(choices=[], value=[]), "⚠️ Please enter or select a source bucket name.", {} token = get_token(custom_token) api = HfApi(token=token) prefix = prefix_filter.strip() if prefix_filter else None try: tree = list(api.list_bucket_tree(bucket_id, prefix=prefix, recursive=True)) except Exception as e: return gr.update(choices=[], value=[]), f"❌ Error accessing bucket '{bucket_id}': {str(e)}", {} # Filter only file items and .safetensors if enabled files_map = {} choices = [] total_size = 0 for item in tree: # Item has attribute 'type', 'path', 'size', 'xet_hash' item_type = getattr(item, "type", "file") if item_type != "file": continue path = getattr(item, "path", None) or str(item) size = getattr(item, "size", 0) xet_hash = getattr(item, "xet_hash", None) # Filter .safetensors files path_lower = path.lower() if only_safetensors and not (path_lower.endswith(".safetensors") or path_lower.endswith(".safetensor")): continue total_size += (size or 0) display_label = f"{path} ({format_size(size)})" choices.append((display_label, path)) files_map[path] = { "path": path, "size": size, "xet_hash": xet_hash, "display": display_label, } count = len(choices) filter_label = ".safetensors " if only_safetensors else "" if count == 0: msg = f"ℹ️ Bucket '{bucket_id}' contains no {filter_label}files" + (f" matching prefix '{prefix}'." if prefix else ".") else: msg = f"✅ Found {count} {filter_label}file{'s' if count != 1 else ''} ({format_size(total_size)}) in bucket `{bucket_id}`." return gr.update(choices=choices, value=[]), msg, files_map def select_all_files(files_map: Dict[str, Any]) -> List[str]: """Select all available files.""" return list(files_map.keys()) def deselect_all_files() -> List[str]: """Deselect all files.""" return [] def transfer_files( src_bucket_input: str, selected_files: List[str], files_metadata: Dict[str, Any], dest_type: str, dest_id_input: str, dest_subfolder: str, preserve_structure: bool, create_dest_if_missing: bool, is_private_dest: bool, delete_source_files: bool = False, custom_token: str | None = None ) -> Generator[str, None, None]: """Execute transfer of files from source bucket to target bucket, dataset, or model.""" src_bucket = clean_identifier(src_bucket_input, kind="bucket") dest_id = clean_identifier(dest_id_input, kind=dest_type.lower()) if not src_bucket: yield "❌ Error: Source bucket is not specified." return if not selected_files: yield "⚠️ No files selected. Please check at least one file to transfer." return if not dest_id: yield f"❌ Error: Destination {dest_type} identifier is required." return token = get_token(custom_token) if not token: yield "⚠️ Warning: No Hugging Face token detected. Operations on private buckets/repos or write actions may fail. Configure HF_TOKEN secret or input your token." api = HfApi(token=token) dest_kind = dest_type.lower() # 'bucket', 'dataset', or 'model' subfolder = (dest_subfolder or "").strip().strip("/") logs = [] def log(msg: str): logs.append(f"[{time.strftime('%H:%M:%S')}] {msg}") return "\n".join(logs) yield log(f"🚀 Initializing transfer of {len(selected_files)} file(s) from bucket `{src_bucket}` to {dest_type} `{dest_id}`...") if delete_source_files: yield log("⚠️ Post-transfer deletion enabled: Successfully transferred files will be deleted from the source bucket.") # Step 1: Ensure destination exists or create it if create_dest_if_missing: try: if dest_kind == "bucket": yield log(f"Checking/Creating destination bucket `{dest_id}`...") api.create_bucket(dest_id, private=is_private_dest, exist_ok=True) elif dest_kind in ["dataset", "model"]: yield log(f"Checking/Creating destination {dest_kind} `{dest_id}`...") api.create_repo(dest_id, repo_type=dest_kind, private=is_private_dest, exist_ok=True) yield log(f" Destination `{dest_id}` is ready.") except Exception as e: yield log(f"⚠️ Destination verification warning: {e}. Attempting transfer regardless...") # Step 2: Temporary staging directory in the Space container temp_dir = tempfile.mkdtemp(prefix="hf2hf_transfer_") total_files = len(selected_files) successful_transfers: List[str] = [] fail_count = 0 start_time = time.time() try: for idx, file_rel_path in enumerate(selected_files, start=1): file_meta = files_metadata.get(file_rel_path, {}) xet_hash = file_meta.get("xet_hash") file_size = file_meta.get("size") file_size_str = format_size(file_size) # Determine destination path in repo/bucket if preserve_structure: target_rel_path = f"{subfolder}/{file_rel_path}" if subfolder else file_rel_path else: filename = Path(file_rel_path).name target_rel_path = f"{subfolder}/{filename}" if subfolder else filename target_rel_path = target_rel_path.lstrip("/") yield log(f"\n({idx}/{total_files}) Transferring `{file_rel_path}` ({file_size_str}) -> `{target_rel_path}`...") # Scenario A: Destination is another Bucket if dest_kind == "bucket": server_side_done = False # Try server-side copy if xet_hash exists if xet_hash: try: yield log(f" Attempting instant server-side copy via xet hash `{xet_hash[:12]}...`") api.batch_bucket_files( bucket_id=dest_id, copy=[("bucket", src_bucket, xet_hash, target_rel_path)] ) server_side_done = True yield log(f" ⚡ Instant server-side copy succeeded!") except Exception as err: yield log(f" ⚠️ Server-side copy skipped ({err}). Falling back to cloud-buffered transfer...") if not server_side_done: # Download to Space container temporary disk and upload to target bucket local_dest = Path(temp_dir) / Path(file_rel_path).name try: yield log(f" Downloading from `{src_bucket}` to Space buffer...") api.download_bucket_files(src_bucket, [(file_rel_path, str(local_dest))]) yield log(f" Uploading from Space buffer to bucket `{dest_id}`...") api.batch_bucket_files(dest_id, add=[(str(local_dest), target_rel_path)]) yield log(f" Transferred to bucket successfully!") except Exception as err: fail_count += 1 yield log(f" ❌ Transfer failed: {err}") continue finally: if local_dest.exists(): try: local_dest.unlink() except Exception: pass successful_transfers.append(file_rel_path) # Scenario B: Destination is Dataset or Model repo else: local_dest = Path(temp_dir) / Path(file_rel_path).name try: yield log(f" Downloading `{file_rel_path}` from bucket to Space buffer...") api.download_bucket_files(src_bucket, [(file_rel_path, str(local_dest))]) yield log(f" Uploading to {dest_kind} `{dest_id}` at `{target_rel_path}`...") api.upload_file( path_or_fileobj=str(local_dest), path_in_repo=target_rel_path, repo_id=dest_id, repo_type=dest_kind, commit_message=f"HF2HF: Transfer {target_rel_path} from bucket {src_bucket}" ) successful_transfers.append(file_rel_path) yield log(f" Committed and uploaded to {dest_kind} `{dest_id}`!") except Exception as err: fail_count += 1 yield log(f" ❌ Transfer failed: {err}") finally: # Always clean up local buffer immediately if local_dest.exists(): try: local_dest.unlink() except Exception: pass finally: # Clean up staging directory shutil.rmtree(temp_dir, ignore_errors=True) # Step 3: Optional deletion from original bucket if delete_source_files and successful_transfers: yield log(f"\n🗑️ Deleting {len(successful_transfers)} transferred file(s) from source bucket `{src_bucket}`...") try: api.batch_bucket_files( bucket_id=src_bucket, delete=successful_transfers, token=token ) yield log(f"✅ Successfully deleted {len(successful_transfers)} file(s) from original bucket `{src_bucket}`.") except Exception as del_err: yield log(f"⚠️ Warning: Could not delete source files ({del_err}).") elapsed = time.time() - start_time yield log("\n" + "=" * 50) yield log(f"🎉 Transfer Completed in {elapsed:.1f}s!") yield log(f" Successfully transferred: {len(successful_transfers)}/{total_files} file(s)") if delete_source_files and successful_transfers: yield log(f" Deleted from source: {len(successful_transfers)} file(s)") if fail_count > 0: yield log(f"⚠️ Failed: {fail_count} file(s)") dest_url = f"https://huggingface.co/{'buckets/' if dest_kind == 'bucket' else ('datasets/' if dest_kind == 'dataset' else '')}{dest_id}" yield log(f"🔗 View destination: {dest_url}") # Build Gradio UI with gr.Blocks(title="HF2HF - Hugging Face Bucket & Repo Transfer") as demo: files_state = gr.State({}) gr.Markdown( """ # 🔀 HF2HF: Direct Cloud-to-Cloud Storage Transfer Transfer files directly between **Hugging Face Buckets**, **Datasets**, and **Models** inside Hugging Face's cloud infrastructure — **no local downloading or intermediary bandwidth needed!** """ ) with gr.Row(): with gr.Column(scale=1): gr.Markdown("### 1️⃣ Source Bucket") with gr.Row(): src_bucket = gr.Dropdown( label="Source Bucket", choices=get_user_buckets(), allow_custom_value=True, info="Select your bucket or type custom bucket ID (e.g. username/my-bucket)", scale=4, ) refresh_buckets_btn = gr.Button("🔄", scale=1, min_width=50) with gr.Row(): prefix_input = gr.Textbox( label="Prefix / Subfolder Filter (Optional)", placeholder="e.g. checkpoints/ or data/ (leave empty for entire bucket)", scale=3, ) only_safetensors = gr.Checkbox( label="Only .safetensors files", value=True, scale=2, ) list_btn = gr.Button("🔍 List Files in Bucket", variant="secondary") source_status = gr.Markdown("Select or enter a source bucket and click **List Files**.") with gr.Column(scale=1): gr.Markdown("### 2️⃣ Destination Target") dest_type = gr.Radio( label="Destination Type", choices=["Bucket", "Dataset", "Model"], value="Bucket", ) dest_id = gr.Textbox( label="Destination Identifier", placeholder="e.g. username/target-bucket or username/my-dataset", ) dest_subfolder = gr.Textbox( label="Destination Subfolder / Path (Optional)", placeholder="e.g. exports/ or leave blank to keep original path", ) with gr.Row(): preserve_structure = gr.Checkbox( label="Preserve directory structure", value=True, ) create_missing = gr.Checkbox( label="Create destination if missing", value=True, ) is_private = gr.Checkbox( label="Make new destination private", value=True, ) delete_source_files = gr.Checkbox( label="🗑️ Delete transferred files from source bucket after transfer", value=False, ) gr.Markdown("### 3️⃣ File Selection") with gr.Row(): select_all_btn = gr.Button("✅ Select All", size="sm") deselect_all_btn = gr.Button("❌ Deselect All", size="sm") file_checkboxes = gr.CheckboxGroup( label="Files to Transfer", choices=[], value=[], ) with gr.Row(): transfer_btn = gr.Button("🚀 Start Cloud Transfer", variant="primary", size="lg") with gr.Accordion("⚙️ Advanced Settings / HF Token Override", open=False): token_input = gr.Textbox( label="Custom Hugging Face Access Token (Optional)", type="password", placeholder="Leave empty to use Space's default HF_TOKEN secret", ) gr.Markdown("### 4️⃣ Live Transfer Log") log_output = gr.Code( label="Progress and Logs", value="Awaiting transfer initiation...", lines=15, interactive=False, ) # Event handlers def on_refresh_buckets(tok: str): b = get_user_buckets(tok) return gr.update(choices=b) refresh_buckets_btn.click( fn=on_refresh_buckets, inputs=[token_input], outputs=[src_bucket], ) list_btn.click( fn=list_files_in_bucket, inputs=[src_bucket, prefix_input, only_safetensors, token_input], outputs=[file_checkboxes, source_status, files_state], ) select_all_btn.click( fn=select_all_files, inputs=[files_state], outputs=[file_checkboxes], ) deselect_all_btn.click( fn=deselect_all_files, inputs=[], outputs=[file_checkboxes], ) transfer_btn.click( fn=transfer_files, inputs=[ src_bucket, file_checkboxes, files_state, dest_type, dest_id, dest_subfolder, preserve_structure, create_missing, is_private, delete_source_files, token_input, ], outputs=[log_output], ) if __name__ == "__main__": demo.launch(theme="soft", mcp_server=True)