rootlocalghost commited on
Commit
eb37dc2
ยท
verified ยท
1 Parent(s): 2e31a32

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +40 -59
app.py CHANGED
@@ -1,6 +1,11 @@
 
 
 
 
1
  import gradio as gr
2
- from huggingface_hub import HfApi, CommitOperationAdd, hf_hub_url
3
- import requests
 
4
 
5
  def fetch_files(source_repo, hf_token):
6
  if not hf_token.strip():
@@ -15,7 +20,6 @@ def fetch_files(source_repo, hf_token):
15
  try:
16
  yield gr.update(), gr.update(), f"Status: Fetching file list from '{source_repo}'...", []
17
 
18
- # files_metadata=True populates the siblings list with file details
19
  info = api.model_info(repo_id=source_repo.strip(), files_metadata=True)
20
  files = sorted([f.rfilename for f in info.siblings])
21
 
@@ -54,7 +58,7 @@ def copy_repository_files(source_repo, target_repo, hf_token, selected_files):
54
  tgt_clean = target_repo.strip()
55
 
56
  try:
57
- # Step 1: Ensure target repository exists
58
  yield f"Status: Verifying target repository '{tgt_clean}'..."
59
  api.create_repo(
60
  repo_id=tgt_clean,
@@ -63,65 +67,42 @@ def copy_repository_files(source_repo, target_repo, hf_token, selected_files):
63
  private=True
64
  )
65
 
66
- # Step 2: Query repository tree with LFS metadata
67
- yield f"Status: Querying LFS metadata from '{src_clean}'..."
68
- model_info = api.model_info(repo_id=src_clean, files_metadata=True)
69
- siblings_map = {f.rfilename: f for f in model_info.siblings}
70
-
71
- headers = {"Authorization": f"Bearer {token}"}
72
- operations = []
73
- selected_set = set(selected_files)
74
- total = len(selected_set)
75
-
76
- for idx, file_path in enumerate(selected_files, start=1):
77
- yield f"Status: Preparing pointer/content ({idx}/{total}): {file_path}..."
78
- sibling = siblings_map.get(file_path)
79
-
80
- if sibling and sibling.lfs is not None:
81
- # File is LFS: Reconstruct canonical 3-line Git LFS pointer
82
- # sibling.lfs contains: {'oid': '<sha256>', 'size': <int>}
83
- sha256 = sibling.lfs.get("oid") or sibling.lfs.get("sha256")
84
- size = sibling.lfs.get("size")
85
- pointer_content = (
86
- f"version https://git-lfs.github.com/spec/v1\n"
87
- f"oid sha256:{sha256}\n"
88
- f"size {size}\n"
89
- ).encode("utf-8")
90
-
91
- operations.append(
92
- CommitOperationAdd(
93
- path_in_repo=file_path,
94
- path_or_fileobj=pointer_content
95
- )
96
  )
97
- else:
98
- # Regular non-LFS file (JSON, markdown, small code files)
99
- url = hf_hub_url(repo_id=src_clean, filename=file_path, repo_type="model")
100
- res = requests.get(url, headers=headers)
101
- res.raise_for_status()
102
-
103
- operations.append(
104
- CommitOperationAdd(
105
- path_in_repo=file_path,
106
- path_or_fileobj=res.content
107
- )
108
  )
109
 
110
- # Step 3: Commit operations in batches of 25 to avoid payload size/timeout errors
111
- batch_size = 25
112
- num_batches = (len(operations) + batch_size - 1) // batch_size
113
 
114
- for b_idx in range(num_batches):
115
- batch = operations[b_idx * batch_size : (b_idx + 1) * batch_size]
116
- yield f"Status: Committing batch {b_idx + 1}/{num_batches} ({len(batch)} files) to '{tgt_clean}'..."
117
- api.create_commit(
118
- repo_id=tgt_clean,
119
- repo_type="model",
120
- operations=batch,
121
- commit_message=f"Selective clone part {b_idx + 1}/{num_batches} from {src_clean}"
122
- )
123
 
124
- yield f"Success! Transferred {total} files across {num_batches} commit(s). View model: https://huggingface.co/{tgt_clean}"
125
 
126
  except Exception as e:
127
  yield f"An error occurred: {str(e)}"
@@ -130,7 +111,7 @@ def copy_repository_files(source_repo, target_repo, hf_token, selected_files):
130
  with gr.Blocks(title="Selective HF Model Cloner") as demo:
131
  gr.Markdown("# ๐Ÿ“‘ Selective Hugging Face Model Cloner")
132
  gr.Markdown(
133
- "Copy selected files from gated repositories directly using zero-bandwidth LFS pointer replication."
134
  )
135
 
136
  with gr.Row():
 
1
+ import os
2
+ import shutil
3
+ import tempfile
4
+ import warnings
5
  import gradio as gr
6
+ from huggingface_hub import HfApi, hf_hub_download
7
+
8
+ warnings.filterwarnings("ignore", category=ResourceWarning)
9
 
10
  def fetch_files(source_repo, hf_token):
11
  if not hf_token.strip():
 
20
  try:
21
  yield gr.update(), gr.update(), f"Status: Fetching file list from '{source_repo}'...", []
22
 
 
23
  info = api.model_info(repo_id=source_repo.strip(), files_metadata=True)
24
  files = sorted([f.rfilename for f in info.siblings])
25
 
 
58
  tgt_clean = target_repo.strip()
59
 
60
  try:
61
+ # Step 1: Ensure destination repo exists
62
  yield f"Status: Verifying target repository '{tgt_clean}'..."
63
  api.create_repo(
64
  repo_id=tgt_clean,
 
67
  private=True
68
  )
69
 
70
+ total = len(selected_files)
71
+ # Use a localized temporary directory to avoid saturating disk space
72
+ temp_dir = tempfile.mkdtemp(prefix="hf_clone_")
73
+
74
+ try:
75
+ for idx, file_path in enumerate(selected_files, start=1):
76
+ yield f"Status: Transferring ({idx}/{total}): {file_path}..."
77
+
78
+ # Step 2: Stream file from source repo to temp storage
79
+ # hf_hub_download handles LFS resolution, authentication, and resumable downloads
80
+ downloaded_path = hf_hub_download(
81
+ repo_id=src_clean,
82
+ filename=file_path,
83
+ repo_type="model",
84
+ token=token,
85
+ local_dir=temp_dir,
86
+ force_download=True
 
 
 
 
 
 
 
 
 
 
 
 
 
87
  )
88
+
89
+ # Step 3: Upload directly to target repo via native LFS upload endpoint
90
+ api.upload_file(
91
+ path_or_fileobj=downloaded_path,
92
+ path_in_repo=file_path,
93
+ repo_id=tgt_clean,
94
+ repo_type="model",
95
+ commit_message=f"Add {file_path} from {src_clean}"
 
 
 
96
  )
97
 
98
+ # Step 4: Immediately delete local file to preserve disk space
99
+ if os.path.exists(downloaded_path):
100
+ os.remove(downloaded_path)
101
 
102
+ finally:
103
+ shutil.rmtree(temp_dir, ignore_errors=True)
 
 
 
 
 
 
 
104
 
105
+ yield f"Success! Transferred {total} files. View model: https://huggingface.co/{tgt_clean}"
106
 
107
  except Exception as e:
108
  yield f"An error occurred: {str(e)}"
 
111
  with gr.Blocks(title="Selective HF Model Cloner") as demo:
112
  gr.Markdown("# ๐Ÿ“‘ Selective Hugging Face Model Cloner")
113
  gr.Markdown(
114
+ "Copy selected files from gated repositories to your model space using authenticated sequential streaming."
115
  )
116
 
117
  with gr.Row():