HF2HF / app.py
Quantumbraid's picture
Upload app.py with huggingface_hub
e327872 verified
Raw History Blame Contribute Delete
18.5 kB
import os
import sys
import time
import shutil
import tempfile
import traceback
from pathlib import Path
from typing import Generator, List, Tuple, Dict, Any
import gradio as gr
from huggingface_hub import HfApi, create_repo, create_bucket
from huggingface_hub.utils import HfHubHTTPError, RepositoryNotFoundError, EntryNotFoundError
def get_token(custom_token: str | None = None) -> str | None:
"""Return explicit custom token if provided, else fall back to HF_TOKEN env var."""
if custom_token and custom_token.strip():
return custom_token.strip()
return os.environ.get("HF_TOKEN") or None
def format_size(size_bytes: int | float | None) -> str:
"""Format byte count into human-readable string."""
if size_bytes is None:
return "Unknown size"
try:
size = float(size_bytes)
except (ValueError, TypeError):
return "Unknown size"
if size < 1024:
return f"{int(size)} B"
elif size < 1024 * 1024:
return f"{size / 1024:.1f} KB"
elif size < 1024 * 1024 * 1024:
return f"{size / (1024 * 1024):.1f} MB"
else:
return f"{size / (1024 * 1024 * 1024):.2f} GB"
def clean_identifier(uri_or_id: str, kind: str = "bucket") -> str:
"""Normalize user input by stripping hf:// URIs and prefixes."""
val = (uri_or_id or "").strip()
if not val:
return ""
# Strip hf:// URI prefixes if user copied full URI
if val.startswith("hf://"):
val = val[len("hf://"):]
if val.startswith("buckets/"):
val = val[len("buckets/"):]
elif val.startswith("datasets/"):
val = val[len("datasets/"):]
elif val.startswith("models/"):
val = val[len("models/"):]
return val.strip("/")
def get_user_buckets(custom_token: str | None = None) -> List[str]:
"""Fetch existing buckets for the logged-in user or orgs."""
token = get_token(custom_token)
api = HfApi(token=token)
buckets = []
try:
user_info = api.whoami()
username = user_info.get("name")
orgs = [org["name"] for org in user_info.get("orgs", []) if "name" in org]
namespaces = [username] + orgs if username else []
for ns in namespaces:
try:
for b in api.list_buckets(namespace=ns):
buckets.append(b.id)
except Exception:
pass
except Exception:
# Fallback to general list if whoami fails
try:
for b in api.list_buckets():
buckets.append(b.id)
except Exception:
pass
return sorted(list(set(buckets)))
def list_files_in_bucket(
bucket_input: str,
prefix_filter: str,
only_safetensors: bool = True,
custom_token: str | None = None
) -> Tuple[Any, str, Dict[str, Any]]:
"""List files in the selected source bucket and prepare CheckboxGroup choices."""
bucket_id = clean_identifier(bucket_input, kind="bucket")
if not bucket_id:
return gr.update(choices=[], value=[]), "⚠️ Please enter or select a source bucket name.", {}
token = get_token(custom_token)
api = HfApi(token=token)
prefix = prefix_filter.strip() if prefix_filter else None
try:
tree = list(api.list_bucket_tree(bucket_id, prefix=prefix, recursive=True))
except Exception as e:
return gr.update(choices=[], value=[]), f"❌ Error accessing bucket '{bucket_id}': {str(e)}", {}
# Filter only file items and .safetensors if enabled
files_map = {}
choices = []
total_size = 0
for item in tree:
# Item has attribute 'type', 'path', 'size', 'xet_hash'
item_type = getattr(item, "type", "file")
if item_type != "file":
continue
path = getattr(item, "path", None) or str(item)
size = getattr(item, "size", 0)
xet_hash = getattr(item, "xet_hash", None)
# Filter .safetensors files
path_lower = path.lower()
if only_safetensors and not (path_lower.endswith(".safetensors") or path_lower.endswith(".safetensor")):
continue
total_size += (size or 0)
display_label = f"{path} ({format_size(size)})"
choices.append((display_label, path))
files_map[path] = {
"path": path,
"size": size,
"xet_hash": xet_hash,
"display": display_label,
}
count = len(choices)
filter_label = ".safetensors " if only_safetensors else ""
if count == 0:
msg = f"ℹ️ Bucket '{bucket_id}' contains no {filter_label}files" + (f" matching prefix '{prefix}'." if prefix else ".")
else:
msg = f"βœ… Found {count} {filter_label}file{'s' if count != 1 else ''} ({format_size(total_size)}) in bucket `{bucket_id}`."
return gr.update(choices=choices, value=[]), msg, files_map
def select_all_files(files_map: Dict[str, Any]) -> List[str]:
"""Select all available files."""
return list(files_map.keys())
def deselect_all_files() -> List[str]:
"""Deselect all files."""
return []
def transfer_files(
src_bucket_input: str,
selected_files: List[str],
files_metadata: Dict[str, Any],
dest_type: str,
dest_id_input: str,
dest_subfolder: str,
preserve_structure: bool,
create_dest_if_missing: bool,
is_private_dest: bool,
delete_source_files: bool = False,
custom_token: str | None = None
) -> Generator[str, None, None]:
"""Execute transfer of files from source bucket to target bucket, dataset, or model."""
src_bucket = clean_identifier(src_bucket_input, kind="bucket")
dest_id = clean_identifier(dest_id_input, kind=dest_type.lower())
if not src_bucket:
yield "❌ Error: Source bucket is not specified."
return
if not selected_files:
yield "⚠️ No files selected. Please check at least one file to transfer."
return
if not dest_id:
yield f"❌ Error: Destination {dest_type} identifier is required."
return
token = get_token(custom_token)
if not token:
yield "⚠️ Warning: No Hugging Face token detected. Operations on private buckets/repos or write actions may fail. Configure HF_TOKEN secret or input your token."
api = HfApi(token=token)
dest_kind = dest_type.lower() # 'bucket', 'dataset', or 'model'
subfolder = (dest_subfolder or "").strip().strip("/")
logs = []
def log(msg: str):
logs.append(f"[{time.strftime('%H:%M:%S')}] {msg}")
return "\n".join(logs)
yield log(f"πŸš€ Initializing transfer of {len(selected_files)} file(s) from bucket `{src_bucket}` to {dest_type} `{dest_id}`...")
if delete_source_files:
yield log("⚠️ Post-transfer deletion enabled: Successfully transferred files will be deleted from the source bucket.")
# Step 1: Ensure destination exists or create it
if create_dest_if_missing:
try:
if dest_kind == "bucket":
yield log(f"Checking/Creating destination bucket `{dest_id}`...")
api.create_bucket(dest_id, private=is_private_dest, exist_ok=True)
elif dest_kind in ["dataset", "model"]:
yield log(f"Checking/Creating destination {dest_kind} `{dest_id}`...")
api.create_repo(dest_id, repo_type=dest_kind, private=is_private_dest, exist_ok=True)
yield log(f" Destination `{dest_id}` is ready.")
except Exception as e:
yield log(f"⚠️ Destination verification warning: {e}. Attempting transfer regardless...")
# Step 2: Temporary staging directory in the Space container
temp_dir = tempfile.mkdtemp(prefix="hf2hf_transfer_")
total_files = len(selected_files)
successful_transfers: List[str] = []
fail_count = 0
start_time = time.time()
try:
for idx, file_rel_path in enumerate(selected_files, start=1):
file_meta = files_metadata.get(file_rel_path, {})
xet_hash = file_meta.get("xet_hash")
file_size = file_meta.get("size")
file_size_str = format_size(file_size)
# Determine destination path in repo/bucket
if preserve_structure:
target_rel_path = f"{subfolder}/{file_rel_path}" if subfolder else file_rel_path
else:
filename = Path(file_rel_path).name
target_rel_path = f"{subfolder}/{filename}" if subfolder else filename
target_rel_path = target_rel_path.lstrip("/")
yield log(f"\n({idx}/{total_files}) Transferring `{file_rel_path}` ({file_size_str}) -> `{target_rel_path}`...")
# Scenario A: Destination is another Bucket
if dest_kind == "bucket":
server_side_done = False
# Try server-side copy if xet_hash exists
if xet_hash:
try:
yield log(f" Attempting instant server-side copy via xet hash `{xet_hash[:12]}...`")
api.batch_bucket_files(
bucket_id=dest_id,
copy=[("bucket", src_bucket, xet_hash, target_rel_path)]
)
server_side_done = True
yield log(f" ⚑ Instant server-side copy succeeded!")
except Exception as err:
yield log(f" ⚠️ Server-side copy skipped ({err}). Falling back to cloud-buffered transfer...")
if not server_side_done:
# Download to Space container temporary disk and upload to target bucket
local_dest = Path(temp_dir) / Path(file_rel_path).name
try:
yield log(f" Downloading from `{src_bucket}` to Space buffer...")
api.download_bucket_files(src_bucket, [(file_rel_path, str(local_dest))])
yield log(f" Uploading from Space buffer to bucket `{dest_id}`...")
api.batch_bucket_files(dest_id, add=[(str(local_dest), target_rel_path)])
yield log(f" Transferred to bucket successfully!")
except Exception as err:
fail_count += 1
yield log(f" ❌ Transfer failed: {err}")
continue
finally:
if local_dest.exists():
try:
local_dest.unlink()
except Exception:
pass
successful_transfers.append(file_rel_path)
# Scenario B: Destination is Dataset or Model repo
else:
local_dest = Path(temp_dir) / Path(file_rel_path).name
try:
yield log(f" Downloading `{file_rel_path}` from bucket to Space buffer...")
api.download_bucket_files(src_bucket, [(file_rel_path, str(local_dest))])
yield log(f" Uploading to {dest_kind} `{dest_id}` at `{target_rel_path}`...")
api.upload_file(
path_or_fileobj=str(local_dest),
path_in_repo=target_rel_path,
repo_id=dest_id,
repo_type=dest_kind,
commit_message=f"HF2HF: Transfer {target_rel_path} from bucket {src_bucket}"
)
successful_transfers.append(file_rel_path)
yield log(f" Committed and uploaded to {dest_kind} `{dest_id}`!")
except Exception as err:
fail_count += 1
yield log(f" ❌ Transfer failed: {err}")
finally:
# Always clean up local buffer immediately
if local_dest.exists():
try:
local_dest.unlink()
except Exception:
pass
finally:
# Clean up staging directory
shutil.rmtree(temp_dir, ignore_errors=True)
# Step 3: Optional deletion from original bucket
if delete_source_files and successful_transfers:
yield log(f"\nπŸ—‘οΈ Deleting {len(successful_transfers)} transferred file(s) from source bucket `{src_bucket}`...")
try:
api.batch_bucket_files(
bucket_id=src_bucket,
delete=successful_transfers,
token=token
)
yield log(f"βœ… Successfully deleted {len(successful_transfers)} file(s) from original bucket `{src_bucket}`.")
except Exception as del_err:
yield log(f"⚠️ Warning: Could not delete source files ({del_err}).")
elapsed = time.time() - start_time
yield log("\n" + "=" * 50)
yield log(f"πŸŽ‰ Transfer Completed in {elapsed:.1f}s!")
yield log(f" Successfully transferred: {len(successful_transfers)}/{total_files} file(s)")
if delete_source_files and successful_transfers:
yield log(f" Deleted from source: {len(successful_transfers)} file(s)")
if fail_count > 0:
yield log(f"⚠️ Failed: {fail_count} file(s)")
dest_url = f"https://huggingface.co/{'buckets/' if dest_kind == 'bucket' else ('datasets/' if dest_kind == 'dataset' else '')}{dest_id}"
yield log(f"πŸ”— View destination: {dest_url}")
# Build Gradio UI
with gr.Blocks(title="HF2HF - Hugging Face Bucket & Repo Transfer") as demo:
files_state = gr.State({})
gr.Markdown(
"""
# πŸ”€ HF2HF: Direct Cloud-to-Cloud Storage Transfer
Transfer files directly between **Hugging Face Buckets**, **Datasets**, and **Models** inside Hugging Face's cloud infrastructure β€” **no local downloading or intermediary bandwidth needed!**
"""
)
with gr.Row():
with gr.Column(scale=1):
gr.Markdown("### 1️⃣ Source Bucket")
with gr.Row():
src_bucket = gr.Dropdown(
label="Source Bucket",
choices=get_user_buckets(),
allow_custom_value=True,
info="Select your bucket or type custom bucket ID (e.g. username/my-bucket)",
scale=4,
)
refresh_buckets_btn = gr.Button("πŸ”„", scale=1, min_width=50)
with gr.Row():
prefix_input = gr.Textbox(
label="Prefix / Subfolder Filter (Optional)",
placeholder="e.g. checkpoints/ or data/ (leave empty for entire bucket)",
scale=3,
)
only_safetensors = gr.Checkbox(
label="Only .safetensors files",
value=True,
scale=2,
)
list_btn = gr.Button("πŸ” List Files in Bucket", variant="secondary")
source_status = gr.Markdown("Select or enter a source bucket and click **List Files**.")
with gr.Column(scale=1):
gr.Markdown("### 2️⃣ Destination Target")
dest_type = gr.Radio(
label="Destination Type",
choices=["Bucket", "Dataset", "Model"],
value="Bucket",
)
dest_id = gr.Textbox(
label="Destination Identifier",
placeholder="e.g. username/target-bucket or username/my-dataset",
)
dest_subfolder = gr.Textbox(
label="Destination Subfolder / Path (Optional)",
placeholder="e.g. exports/ or leave blank to keep original path",
)
with gr.Row():
preserve_structure = gr.Checkbox(
label="Preserve directory structure",
value=True,
)
create_missing = gr.Checkbox(
label="Create destination if missing",
value=True,
)
is_private = gr.Checkbox(
label="Make new destination private",
value=True,
)
delete_source_files = gr.Checkbox(
label="πŸ—‘οΈ Delete transferred files from source bucket after transfer",
value=False,
)
gr.Markdown("### 3️⃣ File Selection")
with gr.Row():
select_all_btn = gr.Button("βœ… Select All", size="sm")
deselect_all_btn = gr.Button("❌ Deselect All", size="sm")
file_checkboxes = gr.CheckboxGroup(
label="Files to Transfer",
choices=[],
value=[],
)
with gr.Row():
transfer_btn = gr.Button("πŸš€ Start Cloud Transfer", variant="primary", size="lg")
with gr.Accordion("βš™οΈ Advanced Settings / HF Token Override", open=False):
token_input = gr.Textbox(
label="Custom Hugging Face Access Token (Optional)",
type="password",
placeholder="Leave empty to use Space's default HF_TOKEN secret",
)
gr.Markdown("### 4️⃣ Live Transfer Log")
log_output = gr.Code(
label="Progress and Logs",
value="Awaiting transfer initiation...",
lines=15,
interactive=False,
)
# Event handlers
def on_refresh_buckets(tok: str):
b = get_user_buckets(tok)
return gr.update(choices=b)
refresh_buckets_btn.click(
fn=on_refresh_buckets,
inputs=[token_input],
outputs=[src_bucket],
)
list_btn.click(
fn=list_files_in_bucket,
inputs=[src_bucket, prefix_input, only_safetensors, token_input],
outputs=[file_checkboxes, source_status, files_state],
)
select_all_btn.click(
fn=select_all_files,
inputs=[files_state],
outputs=[file_checkboxes],
)
deselect_all_btn.click(
fn=deselect_all_files,
inputs=[],
outputs=[file_checkboxes],
)
transfer_btn.click(
fn=transfer_files,
inputs=[
src_bucket,
file_checkboxes,
files_state,
dest_type,
dest_id,
dest_subfolder,
preserve_structure,
create_missing,
is_private,
delete_source_files,
token_input,
],
outputs=[log_output],
)
if __name__ == "__main__":
demo.launch(theme="soft", mcp_server=True)