clean_upload.py

import os
import fnmatch
import sys
from huggingface_hub import HfApi, CommitOperationAdd, CommitOperationDelete

def load_hfignore():
    if not os.path.exists('.hfignore'):
        print('No .hfignore found!')
        return []
    with open('.hfignore', 'r') as f:
        lines = [line.strip() for line in f if line.strip() and not line.startswith('#')]
    return lines

def build_allowed_files(ignore_patterns):
    allowed_files = []
    
    # Pre-parse directory ignores and file ignores
    dir_ignores = []
    file_ignores = []
    for pat in ignore_patterns:
        if pat.endswith('/'):
            dir_ignores.append(pat.rstrip('/'))
        else:
            file_ignores.append(pat)
            
    # Always ignore .git directory
    dir_ignores.append('.git')
    
    for root, dirs, files in os.walk('.'):
        # Filter directories in place so we don't traverse ignored directories
        filtered_dirs = []
        for d in dirs:
            dir_path = os.path.relpath(os.path.join(root, d), '.')
            components = dir_path.split(os.sep)
            
            # Check if any component is .git or matches dir_ignores
            ignored = False
            for parent_dir in components:
                if parent_dir == '.git' or parent_dir in dir_ignores or any(fnmatch.fnmatch(parent_dir, pat) for pat in dir_ignores):
                    ignored = True
                    break
            if not ignored:
                filtered_dirs.append(d)
        dirs[:] = filtered_dirs
        
        for file in files:
            full_path = os.path.join(root, file)
            
            # Skip if it is a symlink or not an actual file on local filesystem
            if os.path.islink(full_path) or not os.path.isfile(full_path):
                continue
                
            rel_path = os.path.relpath(full_path, '.')
            components = rel_path.split(os.sep)
            
            # Check if any part of the path is '.git' or ignored
            ignored = False
            for comp in components:
                if comp == '.git' or comp in dir_ignores or any(fnmatch.fnmatch(comp, pat) for pat in dir_ignores):
                    ignored = True
                    break
            if ignored:
                continue
                
            # Check file ignores
            for pat in file_ignores:
                if fnmatch.fnmatch(file, pat) or fnmatch.fnmatch(rel_path, pat):
                    ignored = True
                    break
            if not ignored:
                allowed_files.append(rel_path)
                
    return allowed_files

def main():
    repo_id = 'Leon4gr45/matomo'
    repo_type = 'space'
    
    print('Loading .hfignore...')
    ignores = load_hfignore()
    print('Ignore patterns:', ignores)
    
    print('Walking directories to find allowed files...')
    allowed = build_allowed_files(ignores)
    print(f'Found {len(allowed)} non-ignored files.')
    
    if len(allowed) > 20000:
        print('ERROR: Allowed file count is still over 20,000!')
        sys.exit(1)
        
    api = HfApi()
    
    # Prune any existing files from remote first so we have a clean slate
    print('Listing remote files...')
    remote_files = api.list_repo_files(repo_id=repo_id, repo_type=repo_type)
    print(f'Remote repo currently has {len(remote_files)} files.')
    
    to_delete = [f for f in remote_files if f != '.gitattributes']
    if to_delete:
        print(f'Pruning {len(to_delete)} remote files first...')
        chunk_size = 500
        for i in range(0, len(to_delete), chunk_size):
            chunk = [CommitOperationDelete(path_in_repo=f) for f in to_delete[i:i+chunk_size]]
            api.create_commit(
                repo_id=repo_id,
                repo_type=repo_type,
                operations=chunk,
                commit_message=f'Prune remote files part {i//chunk_size}'
            )
            print(f'Pruned chunk {i//chunk_size}')
            
    # Upload allowed files in chunks of 300 to be perfectly safe and avoid timeouts
    chunk_size = 300
    print(f'Uploading {len(allowed)} files in chunks of {chunk_size}...')
    for idx, i in enumerate(range(0, len(allowed), chunk_size)):
        chunk = allowed[i:i+chunk_size]
        operations = []
        for file in chunk:
            operations.append(
                CommitOperationAdd(
                    path_in_repo=file,
                    path_or_fileobj=file
                )
            )
        print(f'Uploading chunk {idx + 1}/{(len(allowed) + chunk_size - 1) // chunk_size} ({len(chunk)} files)...')
        api.create_commit(
            repo_id=repo_id,
            repo_type=repo_type,
            operations=operations,
            commit_message=f'Upload source files chunk {idx + 1}'
        )
        print(f'Chunk {idx + 1} uploaded successfully.')
        
    print('ALL CHUNKS UPLOADED SUCCESSFULLY!')

if __name__ == '__main__':
    main()