NickMystic commited on
Commit
322a11c
·
verified ·
1 Parent(s): 7f18358

Remove temporary sync script

Browse files
Files changed (1) hide show
  1. sync_hf_repo.py +0 -101
sync_hf_repo.py DELETED
@@ -1,101 +0,0 @@
1
- from huggingface_hub import HfApi, CommitOperationDelete
2
- import os
3
- from pathlib import Path
4
-
5
- REPO_ID = "NickMystic/DeepDream-MLX"
6
- LOCAL_REPO_PATH = Path(".")
7
-
8
- def get_local_files(path: Path):
9
- """Recursively list all files in the local repository respecting .gitignore."""
10
- # Use git to get the list of files that *should* be in the repo
11
- # This is more robust than walking the directory and parsing .gitignore manually.
12
- import subprocess
13
- result = subprocess.run(
14
- ["git", "ls-files", "--full-name"],
15
- cwd=path,
16
- capture_output=True,
17
- text=True,
18
- check=True
19
- )
20
- return set(result.stdout.splitlines())
21
-
22
- def main():
23
- api = HfApi()
24
-
25
- print(f"--- Synchronizing Hugging Face Repo: {REPO_ID} ---")
26
-
27
- # 1. Get comprehensive list of files on the remote
28
- print("Fetching remote file list...")
29
- remote_files = set(api.list_repo_files(repo_id=REPO_ID))
30
- print(f"Found {len(remote_files)} files on remote.")
31
-
32
- # 2. Get comprehensive list of files locally (respecting .gitignore)
33
- print("Fetching local file list (respecting .gitignore)...")
34
- local_files = get_local_files(LOCAL_REPO_PATH)
35
- print(f"Found {len(local_files)} files locally (tracked by git).")
36
-
37
- # 3. Identify files to delete from remote (present on remote, not locally)
38
- files_to_delete = remote_files - local_files
39
- # Also explicitly add the `toConvert` folder if it exists on remote but not locally anymore.
40
- # We explicitly deleted `toConvert` locally, so ensure it's gone from remote.
41
- if "toConvert/" in remote_files or any(f.startswith("toConvert/") for f in remote_files):
42
- print("`toConvert` folder found on remote, adding to deletion list.")
43
- # Mark all files within toConvert/ for deletion
44
- files_to_delete.update([f for f in remote_files if f.startswith("toConvert/")])
45
-
46
- if "comparisons/" in remote_files or any(f.startswith("comparisons/") for f in remote_files):
47
- print("`comparisons` folder found on remote, ensuring it's synced (may have changed contents).")
48
- # For comparisons, we handle with upload_folder, so don't delete entire contents unless absolutely sure.
49
- # But we want `comparisons/torch_dream.py` to be the only thing. If there are other files, they should be deleted.
50
- pass # Handle below with upload_folder
51
-
52
-
53
- # 4. Identify files to upload/update (present locally, or different checksum)
54
- # This is handled by `upload_folder` below for the entire directory, making it simpler.
55
-
56
- operations = []
57
- if files_to_delete:
58
- print(f"\nIdentified {len(files_to_delete)} files for deletion from remote:")
59
- for file_path in sorted(list(files_to_delete)):
60
- print(f" - {file_path}")
61
- operations.append(CommitOperationDelete(path_in_repo=file_path))
62
- else:
63
- print("\nNo files identified for deletion from remote.")
64
-
65
- if operations:
66
- try:
67
- api.create_commit(
68
- repo_id=REPO_ID,
69
- operations=operations,
70
- commit_message="CLEANUP: Removing obsolete/untracked files from remote"
71
- )
72
- print("✅ Successfully executed deletions on Hugging Face.")
73
- except Exception as e:
74
- print(f"❌ Error during remote deletion commit: {e}")
75
- # If commit failed, maybe it was due to non-existent files?
76
- # Re-list repo files to debug.
77
- try:
78
- remote_files_after_attempt = set(api.list_repo_files(repo_id=REPO_ID, recursive=True))
79
- # Identify which files *actually* remained after the delete attempt
80
- remaining_unwanted_files = files_to_delete.intersection(remote_files_after_attempt)
81
- if remaining_unwanted_files:
82
- print(f"Warning: Some files intended for deletion are still on remote: {remaining_unwanted_files}")
83
- except Exception as inner_e:
84
- print(f"Error re-listing files after failed delete attempt: {inner_e}")
85
-
86
-
87
- # 5. Upload/Update all local files (this handles assets, new files, and updates existing ones)
88
- print(f"\nUploading/updating all local tracked files to {REPO_ID}...")
89
- try:
90
- api.upload_folder(
91
- folder_path=LOCAL_REPO_PATH,
92
- repo_id=REPO_ID,
93
- commit_message="SYNC: Uploading latest local repository state",
94
- repo_type="model" # Explicitly specifying type
95
- )
96
- print("✅ Successfully synced local repository state to Hugging Face.")
97
- except Exception as e:
98
- print(f"❌ Error during final upload/sync: {e}")
99
-
100
- if __name__ == "__main__":
101
- main()