Spaces:
Running
Running
Download app.py from Quantumbraid/HF2HF: direct link, hf CLI and curl.
- Browser
- Download file 18.5 kB
-
https://huggingface.co/spaces/Quantumbraid/HF2HF/resolve/main/app.py
- Command line
-
hf download hf://spaces/Quantumbraid/HF2HF/app.py
-
curl -L -o app.py https://huggingface.co/spaces/Quantumbraid/HF2HF/resolve/main/app.py
18.5 kB
| import os | |
| import sys | |
| import time | |
| import shutil | |
| import tempfile | |
| import traceback | |
| from pathlib import Path | |
| from typing import Generator, List, Tuple, Dict, Any | |
| import gradio as gr | |
| from huggingface_hub import HfApi, create_repo, create_bucket | |
| from huggingface_hub.utils import HfHubHTTPError, RepositoryNotFoundError, EntryNotFoundError | |
| def get_token(custom_token: str | None = None) -> str | None: | |
| """Return explicit custom token if provided, else fall back to HF_TOKEN env var.""" | |
| if custom_token and custom_token.strip(): | |
| return custom_token.strip() | |
| return os.environ.get("HF_TOKEN") or None | |
| def format_size(size_bytes: int | float | None) -> str: | |
| """Format byte count into human-readable string.""" | |
| if size_bytes is None: | |
| return "Unknown size" | |
| try: | |
| size = float(size_bytes) | |
| except (ValueError, TypeError): | |
| return "Unknown size" | |
| if size < 1024: | |
| return f"{int(size)} B" | |
| elif size < 1024 * 1024: | |
| return f"{size / 1024:.1f} KB" | |
| elif size < 1024 * 1024 * 1024: | |
| return f"{size / (1024 * 1024):.1f} MB" | |
| else: | |
| return f"{size / (1024 * 1024 * 1024):.2f} GB" | |
| def clean_identifier(uri_or_id: str, kind: str = "bucket") -> str: | |
| """Normalize user input by stripping hf:// URIs and prefixes.""" | |
| val = (uri_or_id or "").strip() | |
| if not val: | |
| return "" | |
| # Strip hf:// URI prefixes if user copied full URI | |
| if val.startswith("hf://"): | |
| val = val[len("hf://"):] | |
| if val.startswith("buckets/"): | |
| val = val[len("buckets/"):] | |
| elif val.startswith("datasets/"): | |
| val = val[len("datasets/"):] | |
| elif val.startswith("models/"): | |
| val = val[len("models/"):] | |
| return val.strip("/") | |
| def get_user_buckets(custom_token: str | None = None) -> List[str]: | |
| """Fetch existing buckets for the logged-in user or orgs.""" | |
| token = get_token(custom_token) | |
| api = HfApi(token=token) | |
| buckets = [] | |
| try: | |
| user_info = api.whoami() | |
| username = user_info.get("name") | |
| orgs = [org["name"] for org in user_info.get("orgs", []) if "name" in org] | |
| namespaces = [username] + orgs if username else [] | |
| for ns in namespaces: | |
| try: | |
| for b in api.list_buckets(namespace=ns): | |
| buckets.append(b.id) | |
| except Exception: | |
| pass | |
| except Exception: | |
| # Fallback to general list if whoami fails | |
| try: | |
| for b in api.list_buckets(): | |
| buckets.append(b.id) | |
| except Exception: | |
| pass | |
| return sorted(list(set(buckets))) | |
| def list_files_in_bucket( | |
| bucket_input: str, | |
| prefix_filter: str, | |
| only_safetensors: bool = True, | |
| custom_token: str | None = None | |
| ) -> Tuple[Any, str, Dict[str, Any]]: | |
| """List files in the selected source bucket and prepare CheckboxGroup choices.""" | |
| bucket_id = clean_identifier(bucket_input, kind="bucket") | |
| if not bucket_id: | |
| return gr.update(choices=[], value=[]), "β οΈ Please enter or select a source bucket name.", {} | |
| token = get_token(custom_token) | |
| api = HfApi(token=token) | |
| prefix = prefix_filter.strip() if prefix_filter else None | |
| try: | |
| tree = list(api.list_bucket_tree(bucket_id, prefix=prefix, recursive=True)) | |
| except Exception as e: | |
| return gr.update(choices=[], value=[]), f"β Error accessing bucket '{bucket_id}': {str(e)}", {} | |
| # Filter only file items and .safetensors if enabled | |
| files_map = {} | |
| choices = [] | |
| total_size = 0 | |
| for item in tree: | |
| # Item has attribute 'type', 'path', 'size', 'xet_hash' | |
| item_type = getattr(item, "type", "file") | |
| if item_type != "file": | |
| continue | |
| path = getattr(item, "path", None) or str(item) | |
| size = getattr(item, "size", 0) | |
| xet_hash = getattr(item, "xet_hash", None) | |
| # Filter .safetensors files | |
| path_lower = path.lower() | |
| if only_safetensors and not (path_lower.endswith(".safetensors") or path_lower.endswith(".safetensor")): | |
| continue | |
| total_size += (size or 0) | |
| display_label = f"{path} ({format_size(size)})" | |
| choices.append((display_label, path)) | |
| files_map[path] = { | |
| "path": path, | |
| "size": size, | |
| "xet_hash": xet_hash, | |
| "display": display_label, | |
| } | |
| count = len(choices) | |
| filter_label = ".safetensors " if only_safetensors else "" | |
| if count == 0: | |
| msg = f"βΉοΈ Bucket '{bucket_id}' contains no {filter_label}files" + (f" matching prefix '{prefix}'." if prefix else ".") | |
| else: | |
| msg = f"β Found {count} {filter_label}file{'s' if count != 1 else ''} ({format_size(total_size)}) in bucket `{bucket_id}`." | |
| return gr.update(choices=choices, value=[]), msg, files_map | |
| def select_all_files(files_map: Dict[str, Any]) -> List[str]: | |
| """Select all available files.""" | |
| return list(files_map.keys()) | |
| def deselect_all_files() -> List[str]: | |
| """Deselect all files.""" | |
| return [] | |
| def transfer_files( | |
| src_bucket_input: str, | |
| selected_files: List[str], | |
| files_metadata: Dict[str, Any], | |
| dest_type: str, | |
| dest_id_input: str, | |
| dest_subfolder: str, | |
| preserve_structure: bool, | |
| create_dest_if_missing: bool, | |
| is_private_dest: bool, | |
| delete_source_files: bool = False, | |
| custom_token: str | None = None | |
| ) -> Generator[str, None, None]: | |
| """Execute transfer of files from source bucket to target bucket, dataset, or model.""" | |
| src_bucket = clean_identifier(src_bucket_input, kind="bucket") | |
| dest_id = clean_identifier(dest_id_input, kind=dest_type.lower()) | |
| if not src_bucket: | |
| yield "β Error: Source bucket is not specified." | |
| return | |
| if not selected_files: | |
| yield "β οΈ No files selected. Please check at least one file to transfer." | |
| return | |
| if not dest_id: | |
| yield f"β Error: Destination {dest_type} identifier is required." | |
| return | |
| token = get_token(custom_token) | |
| if not token: | |
| yield "β οΈ Warning: No Hugging Face token detected. Operations on private buckets/repos or write actions may fail. Configure HF_TOKEN secret or input your token." | |
| api = HfApi(token=token) | |
| dest_kind = dest_type.lower() # 'bucket', 'dataset', or 'model' | |
| subfolder = (dest_subfolder or "").strip().strip("/") | |
| logs = [] | |
| def log(msg: str): | |
| logs.append(f"[{time.strftime('%H:%M:%S')}] {msg}") | |
| return "\n".join(logs) | |
| yield log(f"π Initializing transfer of {len(selected_files)} file(s) from bucket `{src_bucket}` to {dest_type} `{dest_id}`...") | |
| if delete_source_files: | |
| yield log("β οΈ Post-transfer deletion enabled: Successfully transferred files will be deleted from the source bucket.") | |
| # Step 1: Ensure destination exists or create it | |
| if create_dest_if_missing: | |
| try: | |
| if dest_kind == "bucket": | |
| yield log(f"Checking/Creating destination bucket `{dest_id}`...") | |
| api.create_bucket(dest_id, private=is_private_dest, exist_ok=True) | |
| elif dest_kind in ["dataset", "model"]: | |
| yield log(f"Checking/Creating destination {dest_kind} `{dest_id}`...") | |
| api.create_repo(dest_id, repo_type=dest_kind, private=is_private_dest, exist_ok=True) | |
| yield log(f" Destination `{dest_id}` is ready.") | |
| except Exception as e: | |
| yield log(f"β οΈ Destination verification warning: {e}. Attempting transfer regardless...") | |
| # Step 2: Temporary staging directory in the Space container | |
| temp_dir = tempfile.mkdtemp(prefix="hf2hf_transfer_") | |
| total_files = len(selected_files) | |
| successful_transfers: List[str] = [] | |
| fail_count = 0 | |
| start_time = time.time() | |
| try: | |
| for idx, file_rel_path in enumerate(selected_files, start=1): | |
| file_meta = files_metadata.get(file_rel_path, {}) | |
| xet_hash = file_meta.get("xet_hash") | |
| file_size = file_meta.get("size") | |
| file_size_str = format_size(file_size) | |
| # Determine destination path in repo/bucket | |
| if preserve_structure: | |
| target_rel_path = f"{subfolder}/{file_rel_path}" if subfolder else file_rel_path | |
| else: | |
| filename = Path(file_rel_path).name | |
| target_rel_path = f"{subfolder}/{filename}" if subfolder else filename | |
| target_rel_path = target_rel_path.lstrip("/") | |
| yield log(f"\n({idx}/{total_files}) Transferring `{file_rel_path}` ({file_size_str}) -> `{target_rel_path}`...") | |
| # Scenario A: Destination is another Bucket | |
| if dest_kind == "bucket": | |
| server_side_done = False | |
| # Try server-side copy if xet_hash exists | |
| if xet_hash: | |
| try: | |
| yield log(f" Attempting instant server-side copy via xet hash `{xet_hash[:12]}...`") | |
| api.batch_bucket_files( | |
| bucket_id=dest_id, | |
| copy=[("bucket", src_bucket, xet_hash, target_rel_path)] | |
| ) | |
| server_side_done = True | |
| yield log(f" β‘ Instant server-side copy succeeded!") | |
| except Exception as err: | |
| yield log(f" β οΈ Server-side copy skipped ({err}). Falling back to cloud-buffered transfer...") | |
| if not server_side_done: | |
| # Download to Space container temporary disk and upload to target bucket | |
| local_dest = Path(temp_dir) / Path(file_rel_path).name | |
| try: | |
| yield log(f" Downloading from `{src_bucket}` to Space buffer...") | |
| api.download_bucket_files(src_bucket, [(file_rel_path, str(local_dest))]) | |
| yield log(f" Uploading from Space buffer to bucket `{dest_id}`...") | |
| api.batch_bucket_files(dest_id, add=[(str(local_dest), target_rel_path)]) | |
| yield log(f" Transferred to bucket successfully!") | |
| except Exception as err: | |
| fail_count += 1 | |
| yield log(f" β Transfer failed: {err}") | |
| continue | |
| finally: | |
| if local_dest.exists(): | |
| try: | |
| local_dest.unlink() | |
| except Exception: | |
| pass | |
| successful_transfers.append(file_rel_path) | |
| # Scenario B: Destination is Dataset or Model repo | |
| else: | |
| local_dest = Path(temp_dir) / Path(file_rel_path).name | |
| try: | |
| yield log(f" Downloading `{file_rel_path}` from bucket to Space buffer...") | |
| api.download_bucket_files(src_bucket, [(file_rel_path, str(local_dest))]) | |
| yield log(f" Uploading to {dest_kind} `{dest_id}` at `{target_rel_path}`...") | |
| api.upload_file( | |
| path_or_fileobj=str(local_dest), | |
| path_in_repo=target_rel_path, | |
| repo_id=dest_id, | |
| repo_type=dest_kind, | |
| commit_message=f"HF2HF: Transfer {target_rel_path} from bucket {src_bucket}" | |
| ) | |
| successful_transfers.append(file_rel_path) | |
| yield log(f" Committed and uploaded to {dest_kind} `{dest_id}`!") | |
| except Exception as err: | |
| fail_count += 1 | |
| yield log(f" β Transfer failed: {err}") | |
| finally: | |
| # Always clean up local buffer immediately | |
| if local_dest.exists(): | |
| try: | |
| local_dest.unlink() | |
| except Exception: | |
| pass | |
| finally: | |
| # Clean up staging directory | |
| shutil.rmtree(temp_dir, ignore_errors=True) | |
| # Step 3: Optional deletion from original bucket | |
| if delete_source_files and successful_transfers: | |
| yield log(f"\nποΈ Deleting {len(successful_transfers)} transferred file(s) from source bucket `{src_bucket}`...") | |
| try: | |
| api.batch_bucket_files( | |
| bucket_id=src_bucket, | |
| delete=successful_transfers, | |
| token=token | |
| ) | |
| yield log(f"β Successfully deleted {len(successful_transfers)} file(s) from original bucket `{src_bucket}`.") | |
| except Exception as del_err: | |
| yield log(f"β οΈ Warning: Could not delete source files ({del_err}).") | |
| elapsed = time.time() - start_time | |
| yield log("\n" + "=" * 50) | |
| yield log(f"π Transfer Completed in {elapsed:.1f}s!") | |
| yield log(f" Successfully transferred: {len(successful_transfers)}/{total_files} file(s)") | |
| if delete_source_files and successful_transfers: | |
| yield log(f" Deleted from source: {len(successful_transfers)} file(s)") | |
| if fail_count > 0: | |
| yield log(f"β οΈ Failed: {fail_count} file(s)") | |
| dest_url = f"https://huggingface.co/{'buckets/' if dest_kind == 'bucket' else ('datasets/' if dest_kind == 'dataset' else '')}{dest_id}" | |
| yield log(f"π View destination: {dest_url}") | |
| # Build Gradio UI | |
| with gr.Blocks(title="HF2HF - Hugging Face Bucket & Repo Transfer") as demo: | |
| files_state = gr.State({}) | |
| gr.Markdown( | |
| """ | |
| # π HF2HF: Direct Cloud-to-Cloud Storage Transfer | |
| Transfer files directly between **Hugging Face Buckets**, **Datasets**, and **Models** inside Hugging Face's cloud infrastructure β **no local downloading or intermediary bandwidth needed!** | |
| """ | |
| ) | |
| with gr.Row(): | |
| with gr.Column(scale=1): | |
| gr.Markdown("### 1οΈβ£ Source Bucket") | |
| with gr.Row(): | |
| src_bucket = gr.Dropdown( | |
| label="Source Bucket", | |
| choices=get_user_buckets(), | |
| allow_custom_value=True, | |
| info="Select your bucket or type custom bucket ID (e.g. username/my-bucket)", | |
| scale=4, | |
| ) | |
| refresh_buckets_btn = gr.Button("π", scale=1, min_width=50) | |
| with gr.Row(): | |
| prefix_input = gr.Textbox( | |
| label="Prefix / Subfolder Filter (Optional)", | |
| placeholder="e.g. checkpoints/ or data/ (leave empty for entire bucket)", | |
| scale=3, | |
| ) | |
| only_safetensors = gr.Checkbox( | |
| label="Only .safetensors files", | |
| value=True, | |
| scale=2, | |
| ) | |
| list_btn = gr.Button("π List Files in Bucket", variant="secondary") | |
| source_status = gr.Markdown("Select or enter a source bucket and click **List Files**.") | |
| with gr.Column(scale=1): | |
| gr.Markdown("### 2οΈβ£ Destination Target") | |
| dest_type = gr.Radio( | |
| label="Destination Type", | |
| choices=["Bucket", "Dataset", "Model"], | |
| value="Bucket", | |
| ) | |
| dest_id = gr.Textbox( | |
| label="Destination Identifier", | |
| placeholder="e.g. username/target-bucket or username/my-dataset", | |
| ) | |
| dest_subfolder = gr.Textbox( | |
| label="Destination Subfolder / Path (Optional)", | |
| placeholder="e.g. exports/ or leave blank to keep original path", | |
| ) | |
| with gr.Row(): | |
| preserve_structure = gr.Checkbox( | |
| label="Preserve directory structure", | |
| value=True, | |
| ) | |
| create_missing = gr.Checkbox( | |
| label="Create destination if missing", | |
| value=True, | |
| ) | |
| is_private = gr.Checkbox( | |
| label="Make new destination private", | |
| value=True, | |
| ) | |
| delete_source_files = gr.Checkbox( | |
| label="ποΈ Delete transferred files from source bucket after transfer", | |
| value=False, | |
| ) | |
| gr.Markdown("### 3οΈβ£ File Selection") | |
| with gr.Row(): | |
| select_all_btn = gr.Button("β Select All", size="sm") | |
| deselect_all_btn = gr.Button("β Deselect All", size="sm") | |
| file_checkboxes = gr.CheckboxGroup( | |
| label="Files to Transfer", | |
| choices=[], | |
| value=[], | |
| ) | |
| with gr.Row(): | |
| transfer_btn = gr.Button("π Start Cloud Transfer", variant="primary", size="lg") | |
| with gr.Accordion("βοΈ Advanced Settings / HF Token Override", open=False): | |
| token_input = gr.Textbox( | |
| label="Custom Hugging Face Access Token (Optional)", | |
| type="password", | |
| placeholder="Leave empty to use Space's default HF_TOKEN secret", | |
| ) | |
| gr.Markdown("### 4οΈβ£ Live Transfer Log") | |
| log_output = gr.Code( | |
| label="Progress and Logs", | |
| value="Awaiting transfer initiation...", | |
| lines=15, | |
| interactive=False, | |
| ) | |
| # Event handlers | |
| def on_refresh_buckets(tok: str): | |
| b = get_user_buckets(tok) | |
| return gr.update(choices=b) | |
| refresh_buckets_btn.click( | |
| fn=on_refresh_buckets, | |
| inputs=[token_input], | |
| outputs=[src_bucket], | |
| ) | |
| list_btn.click( | |
| fn=list_files_in_bucket, | |
| inputs=[src_bucket, prefix_input, only_safetensors, token_input], | |
| outputs=[file_checkboxes, source_status, files_state], | |
| ) | |
| select_all_btn.click( | |
| fn=select_all_files, | |
| inputs=[files_state], | |
| outputs=[file_checkboxes], | |
| ) | |
| deselect_all_btn.click( | |
| fn=deselect_all_files, | |
| inputs=[], | |
| outputs=[file_checkboxes], | |
| ) | |
| transfer_btn.click( | |
| fn=transfer_files, | |
| inputs=[ | |
| src_bucket, | |
| file_checkboxes, | |
| files_state, | |
| dest_type, | |
| dest_id, | |
| dest_subfolder, | |
| preserve_structure, | |
| create_missing, | |
| is_private, | |
| delete_source_files, | |
| token_input, | |
| ], | |
| outputs=[log_output], | |
| ) | |
| if __name__ == "__main__": | |
| demo.launch(theme="soft", mcp_server=True) | |