Upload folder using huggingface_hub

sync.py

@@ -0,0 +1,29 @@
+from huggingface_hub import HfApi
+import os
+
+REPO_ID = "MintplexLabs/multilingual-e5-small"
+LOCAL_PATH = "."  # your local directory
+
+api = HfApi()
+
+# Get list of files in repo
+remote_files = {f for f in api.list_repo_files(repo_id=REPO_ID)}
+
+# Get list of files locally (recursively)
+local_files = set()
+for root, _, files in os.walk(LOCAL_PATH):
+    for file in files:
+        full_path = os.path.join(root, file)
+        relative_path = os.path.relpath(full_path, LOCAL_PATH)
+        local_files.add(relative_path.replace("\\", "/"))
+
+# Find files that exist remotely but not locally
+files_to_delete = remote_files - local_files
+
+# Delete them
+for file in files_to_delete:
+    print(f"Deleting {file} from repo...")
+    api.delete_file(path_in_repo=file, repo_id=REPO_ID)
+
+# Now re-upload local files using the CLI
+os.system(f"huggingface-cli upload {REPO_ID} {LOCAL_PATH}")