File size: 18,531 Bytes
61d2843
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
e327872
61d2843
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
e327872
61d2843
 
 
 
 
 
 
 
 
 
 
 
 
e327872
 
 
 
 
61d2843
 
 
 
 
 
 
 
 
 
 
e327872
61d2843
e327872
61d2843
e327872
61d2843
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
e327872
61d2843
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
e327872
 
61d2843
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
e327872
61d2843
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
e327872
61d2843
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
e327872
61d2843
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
e327872
 
 
 
 
 
 
 
 
 
 
 
 
61d2843
 
 
e327872
 
 
61d2843
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
e327872
 
 
 
 
 
 
 
 
 
 
61d2843
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
e327872
61d2843
 
 
 
 
 
 
 
 
 
e327872
 
 
 
61d2843
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
e327872
61d2843
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
e327872
61d2843
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
import os
import sys
import time
import shutil
import tempfile
import traceback
from pathlib import Path
from typing import Generator, List, Tuple, Dict, Any

import gradio as gr
from huggingface_hub import HfApi, create_repo, create_bucket
from huggingface_hub.utils import HfHubHTTPError, RepositoryNotFoundError, EntryNotFoundError


def get_token(custom_token: str | None = None) -> str | None:
    """Return explicit custom token if provided, else fall back to HF_TOKEN env var."""
    if custom_token and custom_token.strip():
        return custom_token.strip()
    return os.environ.get("HF_TOKEN") or None


def format_size(size_bytes: int | float | None) -> str:
    """Format byte count into human-readable string."""
    if size_bytes is None:
        return "Unknown size"
    try:
        size = float(size_bytes)
    except (ValueError, TypeError):
        return "Unknown size"
    if size < 1024:
        return f"{int(size)} B"
    elif size < 1024 * 1024:
        return f"{size / 1024:.1f} KB"
    elif size < 1024 * 1024 * 1024:
        return f"{size / (1024 * 1024):.1f} MB"
    else:
        return f"{size / (1024 * 1024 * 1024):.2f} GB"


def clean_identifier(uri_or_id: str, kind: str = "bucket") -> str:
    """Normalize user input by stripping hf:// URIs and prefixes."""
    val = (uri_or_id or "").strip()
    if not val:
        return ""
    # Strip hf:// URI prefixes if user copied full URI
    if val.startswith("hf://"):
        val = val[len("hf://"):]
    if val.startswith("buckets/"):
        val = val[len("buckets/"):]
    elif val.startswith("datasets/"):
        val = val[len("datasets/"):]
    elif val.startswith("models/"):
        val = val[len("models/"):]
    return val.strip("/")


def get_user_buckets(custom_token: str | None = None) -> List[str]:
    """Fetch existing buckets for the logged-in user or orgs."""
    token = get_token(custom_token)
    api = HfApi(token=token)
    buckets = []
    try:
        user_info = api.whoami()
        username = user_info.get("name")
        orgs = [org["name"] for org in user_info.get("orgs", []) if "name" in org]
        namespaces = [username] + orgs if username else []

        for ns in namespaces:
            try:
                for b in api.list_buckets(namespace=ns):
                    buckets.append(b.id)
            except Exception:
                pass
    except Exception:
        # Fallback to general list if whoami fails
        try:
            for b in api.list_buckets():
                buckets.append(b.id)
        except Exception:
            pass
    return sorted(list(set(buckets)))


def list_files_in_bucket(
    bucket_input: str,
    prefix_filter: str,
    only_safetensors: bool = True,
    custom_token: str | None = None
) -> Tuple[Any, str, Dict[str, Any]]:
    """List files in the selected source bucket and prepare CheckboxGroup choices."""
    bucket_id = clean_identifier(bucket_input, kind="bucket")
    if not bucket_id:
        return gr.update(choices=[], value=[]), "⚠️ Please enter or select a source bucket name.", {}

    token = get_token(custom_token)
    api = HfApi(token=token)
    prefix = prefix_filter.strip() if prefix_filter else None

    try:
        tree = list(api.list_bucket_tree(bucket_id, prefix=prefix, recursive=True))
    except Exception as e:
        return gr.update(choices=[], value=[]), f"❌ Error accessing bucket '{bucket_id}': {str(e)}", {}

    # Filter only file items and .safetensors if enabled
    files_map = {}
    choices = []
    total_size = 0

    for item in tree:
        # Item has attribute 'type', 'path', 'size', 'xet_hash'
        item_type = getattr(item, "type", "file")
        if item_type != "file":
            continue
        path = getattr(item, "path", None) or str(item)
        size = getattr(item, "size", 0)
        xet_hash = getattr(item, "xet_hash", None)

        # Filter .safetensors files
        path_lower = path.lower()
        if only_safetensors and not (path_lower.endswith(".safetensors") or path_lower.endswith(".safetensor")):
            continue

        total_size += (size or 0)
        display_label = f"{path}  ({format_size(size)})"
        choices.append((display_label, path))
        files_map[path] = {
            "path": path,
            "size": size,
            "xet_hash": xet_hash,
            "display": display_label,
        }

    count = len(choices)
    filter_label = ".safetensors " if only_safetensors else ""
    if count == 0:
        msg = f"ℹ️ Bucket '{bucket_id}' contains no {filter_label}files" + (f" matching prefix '{prefix}'." if prefix else ".")
    else:
        msg = f"βœ… Found {count} {filter_label}file{'s' if count != 1 else ''} ({format_size(total_size)}) in bucket `{bucket_id}`."

    return gr.update(choices=choices, value=[]), msg, files_map


def select_all_files(files_map: Dict[str, Any]) -> List[str]:
    """Select all available files."""
    return list(files_map.keys())


def deselect_all_files() -> List[str]:
    """Deselect all files."""
    return []


def transfer_files(
    src_bucket_input: str,
    selected_files: List[str],
    files_metadata: Dict[str, Any],
    dest_type: str,
    dest_id_input: str,
    dest_subfolder: str,
    preserve_structure: bool,
    create_dest_if_missing: bool,
    is_private_dest: bool,
    delete_source_files: bool = False,
    custom_token: str | None = None
) -> Generator[str, None, None]:
    """Execute transfer of files from source bucket to target bucket, dataset, or model."""
    src_bucket = clean_identifier(src_bucket_input, kind="bucket")
    dest_id = clean_identifier(dest_id_input, kind=dest_type.lower())

    if not src_bucket:
        yield "❌ Error: Source bucket is not specified."
        return

    if not selected_files:
        yield "⚠️ No files selected. Please check at least one file to transfer."
        return

    if not dest_id:
        yield f"❌ Error: Destination {dest_type} identifier is required."
        return

    token = get_token(custom_token)
    if not token:
        yield "⚠️ Warning: No Hugging Face token detected. Operations on private buckets/repos or write actions may fail. Configure HF_TOKEN secret or input your token."

    api = HfApi(token=token)
    dest_kind = dest_type.lower()  # 'bucket', 'dataset', or 'model'
    subfolder = (dest_subfolder or "").strip().strip("/")

    logs = []
    def log(msg: str):
        logs.append(f"[{time.strftime('%H:%M:%S')}] {msg}")
        return "\n".join(logs)

    yield log(f"πŸš€ Initializing transfer of {len(selected_files)} file(s) from bucket `{src_bucket}` to {dest_type} `{dest_id}`...")
    if delete_source_files:
        yield log("⚠️ Post-transfer deletion enabled: Successfully transferred files will be deleted from the source bucket.")

    # Step 1: Ensure destination exists or create it
    if create_dest_if_missing:
        try:
            if dest_kind == "bucket":
                yield log(f"Checking/Creating destination bucket `{dest_id}`...")
                api.create_bucket(dest_id, private=is_private_dest, exist_ok=True)
            elif dest_kind in ["dataset", "model"]:
                yield log(f"Checking/Creating destination {dest_kind} `{dest_id}`...")
                api.create_repo(dest_id, repo_type=dest_kind, private=is_private_dest, exist_ok=True)
            yield log(f" Destination `{dest_id}` is ready.")
        except Exception as e:
            yield log(f"⚠️ Destination verification warning: {e}. Attempting transfer regardless...")

    # Step 2: Temporary staging directory in the Space container
    temp_dir = tempfile.mkdtemp(prefix="hf2hf_transfer_")

    total_files = len(selected_files)
    successful_transfers: List[str] = []
    fail_count = 0
    start_time = time.time()

    try:
        for idx, file_rel_path in enumerate(selected_files, start=1):
            file_meta = files_metadata.get(file_rel_path, {})
            xet_hash = file_meta.get("xet_hash")
            file_size = file_meta.get("size")
            file_size_str = format_size(file_size)

            # Determine destination path in repo/bucket
            if preserve_structure:
                target_rel_path = f"{subfolder}/{file_rel_path}" if subfolder else file_rel_path
            else:
                filename = Path(file_rel_path).name
                target_rel_path = f"{subfolder}/{filename}" if subfolder else filename
            target_rel_path = target_rel_path.lstrip("/")

            yield log(f"\n({idx}/{total_files}) Transferring `{file_rel_path}` ({file_size_str}) -> `{target_rel_path}`...")

            # Scenario A: Destination is another Bucket
            if dest_kind == "bucket":
                server_side_done = False
                # Try server-side copy if xet_hash exists
                if xet_hash:
                    try:
                        yield log(f"  Attempting instant server-side copy via xet hash `{xet_hash[:12]}...`")
                        api.batch_bucket_files(
                            bucket_id=dest_id,
                            copy=[("bucket", src_bucket, xet_hash, target_rel_path)]
                        )
                        server_side_done = True
                        yield log(f"  ⚑ Instant server-side copy succeeded!")
                    except Exception as err:
                        yield log(f"  ⚠️ Server-side copy skipped ({err}). Falling back to cloud-buffered transfer...")

                if not server_side_done:
                    # Download to Space container temporary disk and upload to target bucket
                    local_dest = Path(temp_dir) / Path(file_rel_path).name
                    try:
                        yield log(f"  Downloading from `{src_bucket}` to Space buffer...")
                        api.download_bucket_files(src_bucket, [(file_rel_path, str(local_dest))])

                        yield log(f"  Uploading from Space buffer to bucket `{dest_id}`...")
                        api.batch_bucket_files(dest_id, add=[(str(local_dest), target_rel_path)])
                        yield log(f"   Transferred to bucket successfully!")
                    except Exception as err:
                        fail_count += 1
                        yield log(f"  ❌ Transfer failed: {err}")
                        continue
                    finally:
                        if local_dest.exists():
                            try:
                                local_dest.unlink()
                            except Exception:
                                pass

                successful_transfers.append(file_rel_path)

            # Scenario B: Destination is Dataset or Model repo
            else:
                local_dest = Path(temp_dir) / Path(file_rel_path).name
                try:
                    yield log(f"  Downloading `{file_rel_path}` from bucket to Space buffer...")
                    api.download_bucket_files(src_bucket, [(file_rel_path, str(local_dest))])

                    yield log(f"  Uploading to {dest_kind} `{dest_id}` at `{target_rel_path}`...")
                    api.upload_file(
                        path_or_fileobj=str(local_dest),
                        path_in_repo=target_rel_path,
                        repo_id=dest_id,
                        repo_type=dest_kind,
                        commit_message=f"HF2HF: Transfer {target_rel_path} from bucket {src_bucket}"
                    )
                    successful_transfers.append(file_rel_path)
                    yield log(f"   Committed and uploaded to {dest_kind} `{dest_id}`!")
                except Exception as err:
                    fail_count += 1
                    yield log(f"  ❌ Transfer failed: {err}")
                finally:
                    # Always clean up local buffer immediately
                    if local_dest.exists():
                        try:
                            local_dest.unlink()
                        except Exception:
                            pass

    finally:
        # Clean up staging directory
        shutil.rmtree(temp_dir, ignore_errors=True)

    # Step 3: Optional deletion from original bucket
    if delete_source_files and successful_transfers:
        yield log(f"\nπŸ—‘οΈ Deleting {len(successful_transfers)} transferred file(s) from source bucket `{src_bucket}`...")
        try:
            api.batch_bucket_files(
                bucket_id=src_bucket,
                delete=successful_transfers,
                token=token
            )
            yield log(f"βœ… Successfully deleted {len(successful_transfers)} file(s) from original bucket `{src_bucket}`.")
        except Exception as del_err:
            yield log(f"⚠️ Warning: Could not delete source files ({del_err}).")

    elapsed = time.time() - start_time
    yield log("\n" + "=" * 50)
    yield log(f"πŸŽ‰ Transfer Completed in {elapsed:.1f}s!")
    yield log(f" Successfully transferred: {len(successful_transfers)}/{total_files} file(s)")
    if delete_source_files and successful_transfers:
        yield log(f" Deleted from source: {len(successful_transfers)} file(s)")
    if fail_count > 0:
        yield log(f"⚠️ Failed: {fail_count} file(s)")
    
    dest_url = f"https://huggingface.co/{'buckets/' if dest_kind == 'bucket' else ('datasets/' if dest_kind == 'dataset' else '')}{dest_id}"
    yield log(f"πŸ”— View destination: {dest_url}")


# Build Gradio UI
with gr.Blocks(title="HF2HF - Hugging Face Bucket & Repo Transfer") as demo:
    files_state = gr.State({})

    gr.Markdown(
        """
        # πŸ”€ HF2HF: Direct Cloud-to-Cloud Storage Transfer
        Transfer files directly between **Hugging Face Buckets**, **Datasets**, and **Models** inside Hugging Face's cloud infrastructure β€” **no local downloading or intermediary bandwidth needed!**
        """
    )

    with gr.Row():
        with gr.Column(scale=1):
            gr.Markdown("### 1️⃣ Source Bucket")
            with gr.Row():
                src_bucket = gr.Dropdown(
                    label="Source Bucket",
                    choices=get_user_buckets(),
                    allow_custom_value=True,
                    info="Select your bucket or type custom bucket ID (e.g. username/my-bucket)",
                    scale=4,
                )
                refresh_buckets_btn = gr.Button("πŸ”„", scale=1, min_width=50)

            with gr.Row():
                prefix_input = gr.Textbox(
                    label="Prefix / Subfolder Filter (Optional)",
                    placeholder="e.g. checkpoints/ or data/ (leave empty for entire bucket)",
                    scale=3,
                )
                only_safetensors = gr.Checkbox(
                    label="Only .safetensors files",
                    value=True,
                    scale=2,
                )
            list_btn = gr.Button("πŸ” List Files in Bucket", variant="secondary")

            source_status = gr.Markdown("Select or enter a source bucket and click **List Files**.")

        with gr.Column(scale=1):
            gr.Markdown("### 2️⃣ Destination Target")
            dest_type = gr.Radio(
                label="Destination Type",
                choices=["Bucket", "Dataset", "Model"],
                value="Bucket",
            )
            dest_id = gr.Textbox(
                label="Destination Identifier",
                placeholder="e.g. username/target-bucket or username/my-dataset",
            )
            dest_subfolder = gr.Textbox(
                label="Destination Subfolder / Path (Optional)",
                placeholder="e.g. exports/ or leave blank to keep original path",
            )
            with gr.Row():
                preserve_structure = gr.Checkbox(
                    label="Preserve directory structure",
                    value=True,
                )
                create_missing = gr.Checkbox(
                    label="Create destination if missing",
                    value=True,
                )
                is_private = gr.Checkbox(
                    label="Make new destination private",
                    value=True,
                )
            delete_source_files = gr.Checkbox(
                label="πŸ—‘οΈ Delete transferred files from source bucket after transfer",
                value=False,
            )

    gr.Markdown("### 3️⃣ File Selection")
    with gr.Row():
        select_all_btn = gr.Button("βœ… Select All", size="sm")
        deselect_all_btn = gr.Button("❌ Deselect All", size="sm")

    file_checkboxes = gr.CheckboxGroup(
        label="Files to Transfer",
        choices=[],
        value=[],
    )

    with gr.Row():
        transfer_btn = gr.Button("πŸš€ Start Cloud Transfer", variant="primary", size="lg")

    with gr.Accordion("βš™οΈ Advanced Settings / HF Token Override", open=False):
        token_input = gr.Textbox(
            label="Custom Hugging Face Access Token (Optional)",
            type="password",
            placeholder="Leave empty to use Space's default HF_TOKEN secret",
        )

    gr.Markdown("### 4️⃣ Live Transfer Log")
    log_output = gr.Code(
        label="Progress and Logs",
        value="Awaiting transfer initiation...",
        lines=15,
        interactive=False,
    )

    # Event handlers
    def on_refresh_buckets(tok: str):
        b = get_user_buckets(tok)
        return gr.update(choices=b)

    refresh_buckets_btn.click(
        fn=on_refresh_buckets,
        inputs=[token_input],
        outputs=[src_bucket],
    )

    list_btn.click(
        fn=list_files_in_bucket,
        inputs=[src_bucket, prefix_input, only_safetensors, token_input],
        outputs=[file_checkboxes, source_status, files_state],
    )

    select_all_btn.click(
        fn=select_all_files,
        inputs=[files_state],
        outputs=[file_checkboxes],
    )

    deselect_all_btn.click(
        fn=deselect_all_files,
        inputs=[],
        outputs=[file_checkboxes],
    )

    transfer_btn.click(
        fn=transfer_files,
        inputs=[
            src_bucket,
            file_checkboxes,
            files_state,
            dest_type,
            dest_id,
            dest_subfolder,
            preserve_structure,
            create_missing,
            is_private,
            delete_source_files,
            token_input,
        ],
        outputs=[log_output],
    )

if __name__ == "__main__":
    demo.launch(theme="soft", mcp_server=True)