Adding Hugging Face data persistence tool and related configurations

- Add rules file to define writing requirements for project code and documentation.
- Add LFS storage cleanup configuration option in the configuration file.
- Added Python scripts to implement Hugging Face data persistence features, including uploading, managing, and restoring archived files.
- Updated Shell scripts to call new Python modules to streamline archive processing.

.cursor/rules/rule.mdc

@@ -0,0 +1,7 @@
+---
+description: 
+globs: 
+alwaysApply: true
+---
+- All project code must be written in English.
+- Documentation should only be written when I request it.

configs/persistence.conf

@@ -58,6 +58,22 @@ FORCE_SYNC_RESTORE=true
 # Enable data integrity verification after restore
 ENABLE_INTEGRITY_CHECK=true
 
+# =============================================================================
+# LFS Storage Cleanup Configuration (NEW)
+# =============================================================================
+
+# LFS Cleanup Method Options:
+# - 'squash': Use super_squash_history API (gentle, may not always work)
+# - 'recreate': Delete and recreate dataset (aggressive, always works)
+# - 'none': No LFS cleanup (default old behavior)
+LFS_CLEANUP_METHOD=squash
+
+# Force LFS cleanup even if only 1 file was deleted (be careful with 'recreate')
+FORCE_LFS_CLEANUP=false
+
+# Enable LFS storage monitoring and warnings
+ENABLE_LFS_MONITORING=true
+
 # Log Configuration
 # Log file path (using user directory for better permissions)
 LOG_FILE="/home/user/logs/persistence.log"

scripts/utils/hf_persistence.py

@@ -0,0 +1,351 @@
+#!/usr/bin/env python3
+"""
+Hugging Face Data Persistence Tool
+
+This module provides functionality for interacting with Hugging Face Dataset, including:
+- Uploading archive files
+- Managing archive file count
+- LFS storage cleanup
+- Downloading and restoring archive files
+- Listing available archive files
+"""
+
+import sys
+import os
+import traceback
+import time
+import tempfile
+import subprocess
+import argparse
+from pathlib import Path
+from huggingface_hub import HfApi
+
+# Set UTC timezone
+os.environ['TZ'] = 'UTC'
+time.tzset()
+
+# Set Hugging Face cache directory
+cache_dir = '/home/user/.cache/huggingface'
+os.makedirs(cache_dir, exist_ok=True)
+os.environ['HF_HOME'] = cache_dir
+os.environ['HUGGINGFACE_HUB_CACHE'] = cache_dir
+
+
+class HFPersistenceManager:
+    """Hugging Face Data Persistence Manager"""
+    
+    def __init__(self, token: str, dataset_id: str):
+        """
+        Initialize the manager
+        
+        Args:
+            token: Hugging Face access token
+            dataset_id: Dataset ID
+        """
+        self.token = token
+        self.dataset_id = dataset_id
+        self.api = None
+        
+        # Set authentication
+        os.environ['HUGGING_FACE_HUB_TOKEN'] = token
+        
+    def _get_api(self) -> HfApi:
+        """Get HfApi instance"""
+        if self.api is None:
+            self.api = HfApi()
+        return self.api
+    
+    def upload_archive(self, local_path: str, remote_path: str) -> bool:
+        """
+        Upload archive file to Hugging Face Dataset
+        
+        Args:
+            local_path: Local file path
+            remote_path: Remote file path
+            
+        Returns:
+            bool: Whether upload was successful
+        """
+        try:
+            api = self._get_api()
+            api.upload_file(
+                path_or_fileobj=local_path,
+                path_in_repo=remote_path,
+                repo_id=self.dataset_id,
+                repo_type='dataset'
+            )
+            print(f'✓ Archive uploaded successfully: {remote_path}')
+            return True
+        except Exception as e:
+            print(f'✗ Archive upload failed: {str(e)}')
+            traceback.print_exc()
+            return False
+    
+    def manage_archives(self, archive_prefix: str, archive_extension: str, max_files: int) -> bool:
+        """
+        Manage archive file count, delete old archives exceeding the limit
+        
+        Args:
+            archive_prefix: Archive file prefix
+            archive_extension: Archive file extension
+            max_files: Maximum number of files to keep
+            
+        Returns:
+            bool: Whether operation was successful
+        """
+        try:
+            api = self._get_api()
+            files = api.list_repo_files(repo_id=self.dataset_id, repo_type='dataset')
+            archive_files = [f for f in files if f.startswith(archive_prefix) and f.endswith(f'.{archive_extension}')]
+            archive_files.sort()
+            
+            files_to_delete = []
+            if len(archive_files) >= max_files:
+                files_to_delete = archive_files[:(len(archive_files) - max_files)]
+                for file_to_delete in files_to_delete:
+                    try:
+                        api.delete_file(path_in_repo=file_to_delete, repo_id=self.dataset_id, repo_type='dataset')
+                        print(f'✓ Deleted old archive: {file_to_delete}')
+                    except Exception as e:
+                        print(f'✗ Deletion failed {file_to_delete}: {str(e)}')
+            
+            # LFS cleanup options
+            cleanup_method = os.environ.get('LFS_CLEANUP_METHOD', 'none')
+            
+            if cleanup_method == 'squash' and len(files_to_delete) > 0:
+                self._cleanup_lfs_squash(api)
+            elif cleanup_method == 'recreate' and len(files_to_delete) > 0:
+                if not self._cleanup_lfs_recreate(api, archive_files, max_files, archive_prefix, archive_extension):
+                    return False
+            
+            print(f'✓ Archive management completed, currently keeping {min(len(archive_files), max_files)} archives')
+            return True
+        except Exception as e:
+            print(f'✗ Archive management failed: {str(e)}')
+            return False
+    
+    def _cleanup_lfs_squash(self, api: HfApi) -> None:
+        """Clean LFS history using super_squash_history"""
+        try:
+            print('🔄 Attempting to clean LFS history with super_squash_history...')
+            api.super_squash_history(repo_id=self.dataset_id, repo_type='dataset')
+            print('✅ LFS history cleanup attempted - storage may take time to reflect changes')
+        except Exception as e:
+            print(f'⚠️ LFS history cleanup failed: {str(e)}')
+            print('💡 Consider setting LFS_CLEANUP_METHOD=recreate for stronger cleanup')
+    
+    def _cleanup_lfs_recreate(self, api: HfApi, archive_files: list, max_files: int, 
+                              archive_prefix: str, archive_extension: str) -> bool:
+        """Force delete and recreate dataset to clean LFS storage"""
+        print('🚨 WARNING: Force recreate mode enabled')
+        print('📋 This will delete and recreate the entire dataset to clean LFS storage')
+        
+        # Backup files that need to be preserved
+        remaining_files = archive_files[-(max_files-1):] if max_files > 1 else []
+        if remaining_files:
+            print(f'📦 Backing up {len(remaining_files)} files for restoration...')
+            backup_data = []
+            for file_name in remaining_files:
+                try:
+                    # Download file content
+                    file_path = api.hf_hub_download(
+                        repo_id=self.dataset_id,
+                        filename=file_name,
+                        repo_type='dataset'
+                    )
+                    with open(file_path, 'rb') as f:
+                        backup_data.append((file_name, f.read()))
+                    print(f'✓ Backed up: {file_name}')
+                except Exception as e:
+                    print(f'✗ Backup failed for {file_name}: {str(e)}')
+            
+            # Delete dataset
+            try:
+                print('🗑️ Deleting dataset to clean LFS storage...')
+                api.delete_repo(repo_id=self.dataset_id, repo_type='dataset')
+                print('✓ Dataset deleted successfully')
+                
+                # Wait for deletion to complete
+                time.sleep(10)
+                
+                # Recreate dataset
+                print('🔨 Recreating dataset...')
+                api.create_repo(repo_id=self.dataset_id, repo_type='dataset', exist_ok=True)
+                print('✓ Dataset recreated successfully')
+                
+                # Restore backed up files
+                print('📤 Restoring backed up files...')
+                for file_name, file_content in backup_data:
+                    try:
+                        # Create temporary file
+                        with tempfile.NamedTemporaryFile(delete=False) as temp_file:
+                            temp_file.write(file_content)
+                            temp_path = temp_file.name
+                        
+                        # Re-upload
+                        api.upload_file(
+                            path_or_fileobj=temp_path,
+                            path_in_repo=file_name,
+                            repo_id=self.dataset_id,
+                            repo_type='dataset'
+                        )
+                        
+                        # Clean up temporary file
+                        os.unlink(temp_path)
+                        print(f'✓ Restored: {file_name}')
+                    except Exception as e:
+                        print(f'✗ Restore failed for {file_name}: {str(e)}')
+                
+                print('🎉 Dataset recreation and LFS cleanup completed!')
+                return True
+                
+            except Exception as e:
+                print(f'✗ Dataset recreation failed: {str(e)}')
+                print('⚠️ Manual intervention may be required')
+                return False
+        else:
+            print('📝 No files to preserve, skipping backup')
+            return True
+    
+    def list_available_archives(self, archive_prefix: str, archive_extension: str) -> tuple[bool, str]:
+        """
+        List available archive files
+        
+        Args:
+            archive_prefix: Archive file prefix
+            archive_extension: Archive file extension
+            
+        Returns:
+            tuple: (success status, latest archive filename)
+        """
+        try:
+            api = self._get_api()
+            files = api.list_repo_files(self.dataset_id, repo_type='dataset')
+            archive_files = [f for f in files if f.startswith(archive_prefix) and f.endswith(f'.{archive_extension}')]
+            archive_files.sort(reverse=True)
+            
+            if archive_files:
+                print('Available archive list:')
+                for i, archive in enumerate(archive_files, 1):
+                    print(f'  {i}. {archive}')
+                # Return latest archive filename
+                print(f'LATEST_BACKUP:{archive_files[0]}')
+                return True, archive_files[0]
+            else:
+                print('No archive files found')
+                return False, ""
+        except Exception as e:
+            print(f'Failed to get archive list: {str(e)}')
+            traceback.print_exc()
+            return False, ""
+    
+    def restore_from_archive(self, archive_name: str, restore_path: str) -> bool:
+        """
+        Restore archive from Hugging Face Dataset
+        
+        Args:
+            archive_name: Archive filename
+            restore_path: Restore path
+            
+        Returns:
+            bool: Whether restoration was successful
+        """
+        try:
+            api = self._get_api()
+            
+            # Ensure download directory exists with proper permissions
+            download_dir = '/home/user/downloads'
+            os.makedirs(download_dir, exist_ok=True)
+
+            # Download archive file to user directory
+            print(f'Downloading archive: {archive_name}')
+            local_path = api.hf_hub_download(
+                repo_id=self.dataset_id,
+                filename=archive_name,
+                repo_type='dataset',
+                local_dir=download_dir
+            )
+
+            # Extract archive with better error handling
+            print(f'Extracting archive to: {restore_path}')
+
+            # Use subprocess for better control and error handling
+            # Set UTC timezone for timestamp consistency
+            env = os.environ.copy()
+            env['TZ'] = 'UTC'
+
+            extract_cmd = [
+                'tar', '-xzf', local_path, '-C', restore_path,
+                '--warning=no-timestamp',  # Suppress timestamp warnings
+                '--warning=no-unknown-keyword',  # Suppress unknown keyword warnings
+                '--no-same-owner',  # Don't try to restore original ownership
+                '--no-same-permissions',  # Don't try to restore original permissions
+                '--touch'  # Set extracted files timestamps to current UTC time
+            ]
+
+            result = subprocess.run(extract_cmd, capture_output=True, text=True, env=env)
+
+            if result.returncode == 0:
+                print(f'✓ Archive restored successfully: {archive_name}')
+                print('✓ Timestamps normalized to UTC timezone')
+
+                # Clean up temporary file
+                os.remove(local_path)
+                return True
+            else:
+                print(f'✗ Archive extraction failed with return code: {result.returncode}')
+                if result.stderr:
+                    print(f'Error output: {result.stderr}')
+                return False
+
+        except Exception as e:
+            print(f'✗ Archive restoration failed: {str(e)}')
+            traceback.print_exc()
+            return False
+
+
+def main():
+    """Command line entry point"""
+    parser = argparse.ArgumentParser(description='Hugging Face Data Persistence Tool')
+    parser.add_argument('action', choices=['upload', 'list', 'restore'], 
+                       help='Action to perform')
+    parser.add_argument('--token', required=True, help='Hugging Face access token')
+    parser.add_argument('--dataset-id', required=True, help='Dataset ID')
+    parser.add_argument('--archive-file', help='Archive file path (for upload)')
+    parser.add_argument('--filename', help='Remote filename (for upload)')
+    parser.add_argument('--archive-prefix', default='backup', help='Archive file prefix')
+    parser.add_argument('--archive-extension', default='tar.gz', help='Archive file extension')
+    parser.add_argument('--max-archives', type=int, default=5, help='Maximum number of archives to keep')
+    parser.add_argument('--archive-name', help='Archive name to restore (for restore)')
+    parser.add_argument('--restore-path', default='./', help='Restore path (for restore)')
+    
+    args = parser.parse_args()
+    
+    manager = HFPersistenceManager(args.token, args.dataset_id)
+    
+    if args.action == 'upload':
+        if not args.archive_file or not args.filename:
+            print('✗ upload action requires --archive-file and --filename parameters')
+            sys.exit(1)
+        
+        success = manager.upload_archive(args.archive_file, args.filename)
+        if success:
+            success = manager.manage_archives(args.archive_prefix, args.archive_extension, args.max_archives)
+        
+        sys.exit(0 if success else 1)
+    
+    elif args.action == 'list':
+        success, latest = manager.list_available_archives(args.archive_prefix, args.archive_extension)
+        sys.exit(0 if success else 1)
+    
+    elif args.action == 'restore':
+        if not args.archive_name:
+            print('✗ restore action requires --archive-name parameter')
+            sys.exit(1)
+        
+        success = manager.restore_from_archive(args.archive_name, args.restore_path)
+        sys.exit(0 if success else 1)
+
+
+if __name__ == '__main__':
+    main()

scripts/utils/persistence.sh

@@ -67,6 +67,11 @@ set_default_configuration() {
     # Logging configuration
     export LOG_FILE="${LOG_FILE:-}"
     export LOG_LEVEL="${LOG_LEVEL:-}"
+    
+    # LFS Storage Cleanup configuration
+    export LFS_CLEANUP_METHOD="${LFS_CLEANUP_METHOD:-none}"
+    export FORCE_LFS_CLEANUP="${FORCE_LFS_CLEANUP:-false}"
+    export ENABLE_LFS_MONITORING="${ENABLE_LFS_MONITORING:-false}"
 }
 
 # Validate required environment variables
@@ -178,7 +183,7 @@ create_archive() {
     fi
 }
 
-# Embedded Python upload handler
+# Call Python upload handler
 run_upload_handler() {
     local archive_file="$1"
     local filename="$2"
@@ -188,74 +193,18 @@ run_upload_handler() {
     local max_backups="$6"
     local token="$7"
 
-    python3 - <<EOF
-import sys
-import os
-import traceback
-import time
-from huggingface_hub import HfApi
-
-# Set UTC timezone for consistent timestamp handling
-os.environ['TZ'] = 'UTC'
-time.tzset()
-
-# Set up Hugging Face cache directory with proper permissions
-cache_dir = '/home/user/.cache/huggingface'
-os.makedirs(cache_dir, exist_ok=True)
-os.environ['HF_HOME'] = cache_dir
-os.environ['HUGGINGFACE_HUB_CACHE'] = cache_dir
-def upload_archive(api, local_path, remote_path, repo_id):
-    """Upload archive file to Hugging Face Dataset"""
-    try:
-        api.upload_file(
-            path_or_fileobj=local_path,
-            path_in_repo=remote_path,
-            repo_id=repo_id,
-            repo_type='dataset'
-        )
-        print(f'✓ Archive uploaded successfully: {remote_path}')
-        return True
-    except Exception as e:
-        print(f'✗ Archive upload failed: {str(e)}')
-        traceback.print_exc()
-        return False
-def manage_archives(api, repo_id, archive_prefix, archive_extension, max_files):
-    """Manage archive file count, delete old archives exceeding limit"""
-    try:
-        files = api.list_repo_files(repo_id=repo_id, repo_type='dataset')
-        archive_files = [f for f in files if f.startswith(archive_prefix) and f.endswith(f'.{archive_extension}')]
-        archive_files.sort()
-        if len(archive_files) >= max_files:
-            files_to_delete = archive_files[:(len(archive_files) - max_files)]
-            for file_to_delete in files_to_delete:
-                try:
-                    api.delete_file(path_in_repo=file_to_delete, repo_id=repo_id, repo_type='dataset')
-                    print(f'✓ Deleted old archive: {file_to_delete}')
-                except Exception as e:
-                    print(f'✗ Deletion failed {file_to_delete}: {str(e)}')
-        print(f'✓ Archive management completed, currently keeping {min(len(archive_files), max_files)} archives')
-        return True
-    except Exception as e:
-        print(f'✗ Archive management failed: {str(e)}')
-        return False
-# Set authentication
-os.environ['HUGGING_FACE_HUB_TOKEN'] = '$token'
-try:
-    api = HfApi()
-    # Upload file
-    if upload_archive(api, '$archive_file', '$filename', '$dataset_id'):
-        # Manage archive count
-        if manage_archives(api, '$dataset_id', '$backup_prefix', '$backup_extension', $max_backups):
-            sys.exit(0)
-        else:
-            sys.exit(1)
-    else:
-        sys.exit(1)
-except Exception as e:
-    print(f'✗ Upload process error: {str(e)}')
-    traceback.print_exc()
-    sys.exit(1)
-EOF
+    # Get script directory for relative imports
+    local script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+    
+    # Call the standalone Python module
+    python3 "${script_dir}/hf_persistence.py" upload \
+        --token "$token" \
+        --dataset-id "$dataset_id" \
+        --archive-file "$archive_file" \
+        --filename "$filename" \
+        --archive-prefix "$backup_prefix" \
+        --archive-extension "$backup_extension" \
+        --max-archives "$max_backups"
 }
 
 # Upload archive to Hugging Face
@@ -319,62 +268,22 @@ sync_daemon() {
     done
 }
 
-# Embedded Python archive lister
+# Call Python archive lister
 run_archive_lister() {
     local dataset_id="$1"
     local backup_prefix="$2"
     local backup_extension="$3"
     local token="$4"
 
-    python3 - <<EOF
-import sys
-import os
-import traceback
-import time
-from huggingface_hub import HfApi
-
-# Set up Hugging Face cache directory with proper permissions
-cache_dir = '/home/user/.cache/huggingface'
-os.makedirs(cache_dir, exist_ok=True)
-os.environ['HF_HOME'] = cache_dir
-os.environ['HUGGINGFACE_HUB_CACHE'] = cache_dir
-
-# Set UTC timezone for consistent timestamp handling
-os.environ['TZ'] = 'UTC'
-time.tzset()
-def list_available_archives(api, repo_id, archive_prefix, archive_extension):
-    """List available archive files"""
-    try:
-        files = api.list_repo_files(repo_id, repo_type='dataset')
-        archive_files = [f for f in files if f.startswith(archive_prefix) and f.endswith(f'.{archive_extension}')]
-        archive_files.sort(reverse=True)
-        if archive_files:
-            print('Available archive list:')
-            for i, archive in enumerate(archive_files, 1):
-                print(f'  {i}. {archive}')
-            # Return latest archive filename
-            print(f'LATEST_BACKUP:{archive_files[0]}')
-            return True
-        else:
-            print('No archive files found')
-            return False
-    except Exception as e:
-        print(f'Failed to get archive list: {str(e)}')
-        traceback.print_exc()
-        return False
-# Set authentication
-os.environ['HUGGING_FACE_HUB_TOKEN'] = '$token'
-try:
-    api = HfApi()
-    if list_available_archives(api, '$dataset_id', '$backup_prefix', '$backup_extension'):
-        sys.exit(0)
-    else:
-        sys.exit(1)
-except Exception as e:
-    print(f'List retrieval process error: {str(e)}')
-    traceback.print_exc()
-    sys.exit(1)
-EOF
+    # Get script directory for relative imports
+    local script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+    
+    # Call the standalone Python module
+    python3 "${script_dir}/hf_persistence.py" list \
+        --token "$token" \
+        --dataset-id "$dataset_id" \
+        --archive-prefix "$backup_prefix" \
+        --archive-extension "$backup_extension"
 }
 
 # List available archives
@@ -392,97 +301,22 @@ list_archives() {
     run_archive_lister "$DATASET_ID" "$ARCHIVE_PREFIX" "$ARCHIVE_EXTENSION" "$HF_TOKEN"
 }
 
-# Embedded Python download handler
+# Call Python download handler
 run_download_handler() {
     local backup_name="$1"
     local dataset_id="$2"
     local restore_path="$3"
     local token="$4"
 
-    python3 - <<EOF
-import sys
-import os
-import traceback
-import subprocess
-import time
-from huggingface_hub import HfApi
-
-# Set UTC timezone for consistent timestamp handling
-os.environ['TZ'] = 'UTC'
-time.tzset()
-
-# Set up Hugging Face cache directory with proper permissions
-cache_dir = '/home/user/.cache/huggingface'
-os.makedirs(cache_dir, exist_ok=True)
-os.environ['HF_HOME'] = cache_dir
-os.environ['HUGGINGFACE_HUB_CACHE'] = cache_dir
-
-def restore_from_archive(api, repo_id, archive_name, restore_path):
-    """Restore archive from Hugging Face Dataset"""
-    try:
-        # Ensure download directory exists with proper permissions
-        download_dir = '/home/user/downloads'
-        os.makedirs(download_dir, exist_ok=True)
-
-        # Download archive file to user-owned directory
-        print(f'Downloading archive: {archive_name}')
-        local_path = api.hf_hub_download(
-            repo_id=repo_id,
-            filename=archive_name,
-            repo_type='dataset',
-            local_dir=download_dir
-        )
-
-        # Extract archive with better error handling
-        print(f'Extracting archive to: {restore_path}')
-
-        # Use subprocess for better control and error handling
-        # Set UTC timezone for consistent timestamp handling
-        env = os.environ.copy()
-        env['TZ'] = 'UTC'
-
-        extract_cmd = [
-            'tar', '-xzf', local_path, '-C', restore_path,
-            '--warning=no-timestamp',  # Suppress timestamp warnings
-            '--warning=no-unknown-keyword',  # Suppress unknown keyword warnings
-            '--no-same-owner',  # Don't try to restore original ownership
-            '--no-same-permissions',  # Don't try to restore original permissions
-            '--touch'  # Set extracted files' timestamps to current time in UTC
-        ]
-
-        result = subprocess.run(extract_cmd, capture_output=True, text=True, env=env)
-
-        if result.returncode == 0:
-            print(f'✓ Archive restored successfully: {archive_name}')
-            print('✓ Timestamps normalized to UTC timezone')
-
-            # Clean up temporary files
-            os.remove(local_path)
-            return True
-        else:
-            print(f'✗ Archive extraction failed with return code: {result.returncode}')
-            if result.stderr:
-                print(f'Error output: {result.stderr}')
-            return False
-
-    except Exception as e:
-        print(f'✗ Archive restoration failed: {str(e)}')
-        traceback.print_exc()
-        return False
-
-# Set authentication
-os.environ['HUGGING_FACE_HUB_TOKEN'] = '$token'
-try:
-    api = HfApi()
-    if restore_from_archive(api, '$dataset_id', '$backup_name', '$restore_path'):
-        sys.exit(0)
-    else:
-        sys.exit(1)
-except Exception as e:
-    print(f'Restoration process error: {str(e)}')
-    traceback.print_exc()
-    sys.exit(1)
-EOF
+    # Get script directory for relative imports
+    local script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+    
+    # Call the standalone Python module
+    python3 "${script_dir}/hf_persistence.py" restore \
+        --token "$token" \
+        --dataset-id "$dataset_id" \
+        --archive-name "$backup_name" \
+        --restore-path "$restore_path"
 }
 
 # Verify data integrity after restoration
@@ -619,8 +453,6 @@ restore_archive() {
     fi
 }
 
-
-
 # Main program entry
 main() {
     local command="start"