Spaces:
Running
Running
File size: 2,183 Bytes
963d10b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 | from huggingface_hub import HfApi, HfFileSystem
import urllib.parse
import os
import sys, io
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')
api = HfApi()
fs = HfFileSystem()
def get_language(filename):
name = urllib.parse.unquote(filename).lower()
if 'english' in name or 'american' in name or 'british' in name or 'patois' in name or 'pidgin' in name or 'south african' in name:
return 'English'
if 'korean' in name:
return 'Korean'
if 'igbo' in name:
return 'Asusu Igbo'
if 'tagalog' in name or 'filipino' in name:
return 'Filipino'
if 'creole' in name:
return 'Creole'
if 'indonesian' in name:
return 'Indonesian'
return 'Other'
repo_id = "toecm/IEDID"
def migrate():
files = fs.ls(f"datasets/{repo_id}", detail=False)
# Strip prefix 'datasets/toecm/IEDID/'
files = [f.replace(f"datasets/{repo_id}/", "") for f in files]
operations = []
for f in files:
if f in ['.gitattributes', 'README.md', 'minted_history.csv', 'audio', 'audio_files']:
continue
lang = get_language(f)
# Decide if it goes to /data/ or /personas/
if f.endswith('.csv'):
new_path = f"{lang}/data/{f}"
elif f.endswith('.json'):
new_path = f"{lang}/personas/{f}"
else:
continue
print(f"Moving {f} -> {new_path}")
operations.append({
"path_in_repo": f,
"path_in_repo_new": new_path,
})
if not operations:
print("No files to migrate.")
return
print("Executing move operations...")
for op in operations:
try:
# We use fs.mv for moving files on the Hugging Face filesystem
src = f"datasets/{repo_id}/{op['path_in_repo']}"
dest = f"datasets/{repo_id}/{op['path_in_repo_new']}"
fs.mv(src, dest)
print(f"Success: {op['path_in_repo']}")
except Exception as e:
print(f"Failed to move {op['path_in_repo']}: {e}")
if __name__ == "__main__":
migrate()
|