Remove temporary sync script
Browse files- sync_hf_repo.py +0 -101
sync_hf_repo.py
DELETED
|
@@ -1,101 +0,0 @@
|
|
| 1 |
-
from huggingface_hub import HfApi, CommitOperationDelete
|
| 2 |
-
import os
|
| 3 |
-
from pathlib import Path
|
| 4 |
-
|
| 5 |
-
REPO_ID = "NickMystic/DeepDream-MLX"
|
| 6 |
-
LOCAL_REPO_PATH = Path(".")
|
| 7 |
-
|
| 8 |
-
def get_local_files(path: Path):
|
| 9 |
-
"""Recursively list all files in the local repository respecting .gitignore."""
|
| 10 |
-
# Use git to get the list of files that *should* be in the repo
|
| 11 |
-
# This is more robust than walking the directory and parsing .gitignore manually.
|
| 12 |
-
import subprocess
|
| 13 |
-
result = subprocess.run(
|
| 14 |
-
["git", "ls-files", "--full-name"],
|
| 15 |
-
cwd=path,
|
| 16 |
-
capture_output=True,
|
| 17 |
-
text=True,
|
| 18 |
-
check=True
|
| 19 |
-
)
|
| 20 |
-
return set(result.stdout.splitlines())
|
| 21 |
-
|
| 22 |
-
def main():
|
| 23 |
-
api = HfApi()
|
| 24 |
-
|
| 25 |
-
print(f"--- Synchronizing Hugging Face Repo: {REPO_ID} ---")
|
| 26 |
-
|
| 27 |
-
# 1. Get comprehensive list of files on the remote
|
| 28 |
-
print("Fetching remote file list...")
|
| 29 |
-
remote_files = set(api.list_repo_files(repo_id=REPO_ID))
|
| 30 |
-
print(f"Found {len(remote_files)} files on remote.")
|
| 31 |
-
|
| 32 |
-
# 2. Get comprehensive list of files locally (respecting .gitignore)
|
| 33 |
-
print("Fetching local file list (respecting .gitignore)...")
|
| 34 |
-
local_files = get_local_files(LOCAL_REPO_PATH)
|
| 35 |
-
print(f"Found {len(local_files)} files locally (tracked by git).")
|
| 36 |
-
|
| 37 |
-
# 3. Identify files to delete from remote (present on remote, not locally)
|
| 38 |
-
files_to_delete = remote_files - local_files
|
| 39 |
-
# Also explicitly add the `toConvert` folder if it exists on remote but not locally anymore.
|
| 40 |
-
# We explicitly deleted `toConvert` locally, so ensure it's gone from remote.
|
| 41 |
-
if "toConvert/" in remote_files or any(f.startswith("toConvert/") for f in remote_files):
|
| 42 |
-
print("`toConvert` folder found on remote, adding to deletion list.")
|
| 43 |
-
# Mark all files within toConvert/ for deletion
|
| 44 |
-
files_to_delete.update([f for f in remote_files if f.startswith("toConvert/")])
|
| 45 |
-
|
| 46 |
-
if "comparisons/" in remote_files or any(f.startswith("comparisons/") for f in remote_files):
|
| 47 |
-
print("`comparisons` folder found on remote, ensuring it's synced (may have changed contents).")
|
| 48 |
-
# For comparisons, we handle with upload_folder, so don't delete entire contents unless absolutely sure.
|
| 49 |
-
# But we want `comparisons/torch_dream.py` to be the only thing. If there are other files, they should be deleted.
|
| 50 |
-
pass # Handle below with upload_folder
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
# 4. Identify files to upload/update (present locally, or different checksum)
|
| 54 |
-
# This is handled by `upload_folder` below for the entire directory, making it simpler.
|
| 55 |
-
|
| 56 |
-
operations = []
|
| 57 |
-
if files_to_delete:
|
| 58 |
-
print(f"\nIdentified {len(files_to_delete)} files for deletion from remote:")
|
| 59 |
-
for file_path in sorted(list(files_to_delete)):
|
| 60 |
-
print(f" - {file_path}")
|
| 61 |
-
operations.append(CommitOperationDelete(path_in_repo=file_path))
|
| 62 |
-
else:
|
| 63 |
-
print("\nNo files identified for deletion from remote.")
|
| 64 |
-
|
| 65 |
-
if operations:
|
| 66 |
-
try:
|
| 67 |
-
api.create_commit(
|
| 68 |
-
repo_id=REPO_ID,
|
| 69 |
-
operations=operations,
|
| 70 |
-
commit_message="CLEANUP: Removing obsolete/untracked files from remote"
|
| 71 |
-
)
|
| 72 |
-
print("✅ Successfully executed deletions on Hugging Face.")
|
| 73 |
-
except Exception as e:
|
| 74 |
-
print(f"❌ Error during remote deletion commit: {e}")
|
| 75 |
-
# If commit failed, maybe it was due to non-existent files?
|
| 76 |
-
# Re-list repo files to debug.
|
| 77 |
-
try:
|
| 78 |
-
remote_files_after_attempt = set(api.list_repo_files(repo_id=REPO_ID, recursive=True))
|
| 79 |
-
# Identify which files *actually* remained after the delete attempt
|
| 80 |
-
remaining_unwanted_files = files_to_delete.intersection(remote_files_after_attempt)
|
| 81 |
-
if remaining_unwanted_files:
|
| 82 |
-
print(f"Warning: Some files intended for deletion are still on remote: {remaining_unwanted_files}")
|
| 83 |
-
except Exception as inner_e:
|
| 84 |
-
print(f"Error re-listing files after failed delete attempt: {inner_e}")
|
| 85 |
-
|
| 86 |
-
|
| 87 |
-
# 5. Upload/Update all local files (this handles assets, new files, and updates existing ones)
|
| 88 |
-
print(f"\nUploading/updating all local tracked files to {REPO_ID}...")
|
| 89 |
-
try:
|
| 90 |
-
api.upload_folder(
|
| 91 |
-
folder_path=LOCAL_REPO_PATH,
|
| 92 |
-
repo_id=REPO_ID,
|
| 93 |
-
commit_message="SYNC: Uploading latest local repository state",
|
| 94 |
-
repo_type="model" # Explicitly specifying type
|
| 95 |
-
)
|
| 96 |
-
print("✅ Successfully synced local repository state to Hugging Face.")
|
| 97 |
-
except Exception as e:
|
| 98 |
-
print(f"❌ Error during final upload/sync: {e}")
|
| 99 |
-
|
| 100 |
-
if __name__ == "__main__":
|
| 101 |
-
main()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|