rootlocalghost commited on
Commit
2e31a32
·
verified ·
1 Parent(s): 4c54cf1

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +34 -29
app.py CHANGED
@@ -1,6 +1,5 @@
1
- import io
2
  import gradio as gr
3
- from huggingface_hub import HfApi, CommitOperationAdd, get_hf_file_metadata, hf_hub_url
4
  import requests
5
 
6
  def fetch_files(source_repo, hf_token):
@@ -16,8 +15,9 @@ def fetch_files(source_repo, hf_token):
16
  try:
17
  yield gr.update(), gr.update(), f"Status: Fetching file list from '{source_repo}'...", []
18
 
19
- files = api.list_repo_files(repo_id=source_repo.strip(), repo_type="model")
20
- files.sort()
 
21
 
22
  choices = []
23
  for f in files:
@@ -63,25 +63,25 @@ def copy_repository_files(source_repo, target_repo, hf_token, selected_files):
63
  private=True
64
  )
65
 
 
 
 
 
 
66
  headers = {"Authorization": f"Bearer {token}"}
67
  operations = []
68
- total = len(selected_files)
 
69
 
70
- # Step 2: Prepare commit operations (Zero-weight-download transfer)
71
  for idx, file_path in enumerate(selected_files, start=1):
72
- yield f"Status: Preparing file metadata ({idx}/{total}): {file_path}..."
73
-
74
- # Query Hugging Face file metadata to check if the file is Git LFS
75
- file_meta = get_hf_file_metadata(
76
- hf_hub_url(repo_id=src_clean, filename=file_path, repo_type="model"),
77
- token=token
78
- )
79
-
80
- if file_meta.lfs is not None:
81
- # File is managed by Git LFS (Safetensors, weights, etc.)
82
- # HF resolves the blob automatically if we upload the canonical pointer
83
- sha256 = file_meta.lfs.get("sha256")
84
- size = file_meta.lfs.get("size")
85
  pointer_content = (
86
  f"version https://git-lfs.github.com/spec/v1\n"
87
  f"oid sha256:{sha256}\n"
@@ -95,7 +95,7 @@ def copy_repository_files(source_repo, target_repo, hf_token, selected_files):
95
  )
96
  )
97
  else:
98
- # Regular non-LFS file (JSON configs, code, text, small YAML)
99
  url = hf_hub_url(repo_id=src_clean, filename=file_path, repo_type="model")
100
  res = requests.get(url, headers=headers)
101
  res.raise_for_status()
@@ -107,16 +107,21 @@ def copy_repository_files(source_repo, target_repo, hf_token, selected_files):
107
  )
108
  )
109
 
110
- # Step 3: Atomic commit on target repository
111
- yield f"Status: Committing {len(operations)} files to '{tgt_clean}'..."
112
- api.create_commit(
113
- repo_id=tgt_clean,
114
- repo_type="model",
115
- operations=operations,
116
- commit_message=f"Selective clone from {src_clean}"
117
- )
 
 
 
 
 
118
 
119
- yield f"Success! Transferred {total} files. View model: https://huggingface.co/{tgt_clean}"
120
 
121
  except Exception as e:
122
  yield f"An error occurred: {str(e)}"
 
 
1
  import gradio as gr
2
+ from huggingface_hub import HfApi, CommitOperationAdd, hf_hub_url
3
  import requests
4
 
5
  def fetch_files(source_repo, hf_token):
 
15
  try:
16
  yield gr.update(), gr.update(), f"Status: Fetching file list from '{source_repo}'...", []
17
 
18
+ # files_metadata=True populates the siblings list with file details
19
+ info = api.model_info(repo_id=source_repo.strip(), files_metadata=True)
20
+ files = sorted([f.rfilename for f in info.siblings])
21
 
22
  choices = []
23
  for f in files:
 
63
  private=True
64
  )
65
 
66
+ # Step 2: Query repository tree with LFS metadata
67
+ yield f"Status: Querying LFS metadata from '{src_clean}'..."
68
+ model_info = api.model_info(repo_id=src_clean, files_metadata=True)
69
+ siblings_map = {f.rfilename: f for f in model_info.siblings}
70
+
71
  headers = {"Authorization": f"Bearer {token}"}
72
  operations = []
73
+ selected_set = set(selected_files)
74
+ total = len(selected_set)
75
 
 
76
  for idx, file_path in enumerate(selected_files, start=1):
77
+ yield f"Status: Preparing pointer/content ({idx}/{total}): {file_path}..."
78
+ sibling = siblings_map.get(file_path)
79
+
80
+ if sibling and sibling.lfs is not None:
81
+ # File is LFS: Reconstruct canonical 3-line Git LFS pointer
82
+ # sibling.lfs contains: {'oid': '<sha256>', 'size': <int>}
83
+ sha256 = sibling.lfs.get("oid") or sibling.lfs.get("sha256")
84
+ size = sibling.lfs.get("size")
 
 
 
 
 
85
  pointer_content = (
86
  f"version https://git-lfs.github.com/spec/v1\n"
87
  f"oid sha256:{sha256}\n"
 
95
  )
96
  )
97
  else:
98
+ # Regular non-LFS file (JSON, markdown, small code files)
99
  url = hf_hub_url(repo_id=src_clean, filename=file_path, repo_type="model")
100
  res = requests.get(url, headers=headers)
101
  res.raise_for_status()
 
107
  )
108
  )
109
 
110
+ # Step 3: Commit operations in batches of 25 to avoid payload size/timeout errors
111
+ batch_size = 25
112
+ num_batches = (len(operations) + batch_size - 1) // batch_size
113
+
114
+ for b_idx in range(num_batches):
115
+ batch = operations[b_idx * batch_size : (b_idx + 1) * batch_size]
116
+ yield f"Status: Committing batch {b_idx + 1}/{num_batches} ({len(batch)} files) to '{tgt_clean}'..."
117
+ api.create_commit(
118
+ repo_id=tgt_clean,
119
+ repo_type="model",
120
+ operations=batch,
121
+ commit_message=f"Selective clone part {b_idx + 1}/{num_batches} from {src_clean}"
122
+ )
123
 
124
+ yield f"Success! Transferred {total} files across {num_batches} commit(s). View model: https://huggingface.co/{tgt_clean}"
125
 
126
  except Exception as e:
127
  yield f"An error occurred: {str(e)}"