Remove LFS storage cleanup configuration and refactor pre-upload checking and cleanup logic

- Remove LFS cleanup related settings from the configuration file.
- Rename and refactor the `manage_archives` method to `check_and_cleanup_before_upload` to check the archive file count and perform the necessary cleanup before upload.
- Updated scripts to invoke new cleanup logic before upload to ensure more efficient archive management.

configs/persistence.conf

@@ -58,22 +58,6 @@ FORCE_SYNC_RESTORE=true
 # Enable data integrity verification after restore
 ENABLE_INTEGRITY_CHECK=true
 
-# =============================================================================
-# LFS Storage Cleanup Configuration (NEW)
-# =============================================================================
-
-# LFS Cleanup Method Options:
-# - 'squash': Use super_squash_history API (gentle, may not always work)
-# - 'recreate': Delete and recreate dataset (aggressive, always works)
-# - 'none': No LFS cleanup (default old behavior)
-LFS_CLEANUP_METHOD=squash
-
-# Force LFS cleanup even if only 1 file was deleted (be careful with 'recreate')
-FORCE_LFS_CLEANUP=false
-
-# Enable LFS storage monitoring and warnings
-ENABLE_LFS_MONITORING=true
-
 # Log Configuration
 # Log file path (using user directory for better permissions)
 LOG_FILE="/home/user/logs/persistence.log"

scripts/utils/hf_persistence.py

@@ -81,9 +81,9 @@ class HFPersistenceManager:
             traceback.print_exc()
             return False
     
-    def manage_archives(self, archive_prefix: str, archive_extension: str, max_files: int) -> bool:
+    def check_and_cleanup_before_upload(self, archive_prefix: str, archive_extension: str, max_files: int) -> bool:
         """
-        Manage archive file count, delete old archives exceeding the limit
+        Check if adding new archive would exceed limit, if so, force recreate dataset first
         
         Args:
             archive_prefix: Archive file prefix
@@ -97,51 +97,37 @@ class HFPersistenceManager:
             api = self._get_api()
             files = api.list_repo_files(repo_id=self.dataset_id, repo_type='dataset')
             archive_files = [f for f in files if f.startswith(archive_prefix) and f.endswith(f'.{archive_extension}')]
-            archive_files.sort()
+            archive_files.sort(reverse=True)  # Sort newest first
             
-            files_to_delete = []
-            if len(archive_files) >= max_files:
-                files_to_delete = archive_files[:(len(archive_files) - max_files)]
-                for file_to_delete in files_to_delete:
-                    try:
-                        api.delete_file(path_in_repo=file_to_delete, repo_id=self.dataset_id, repo_type='dataset')
-                        print(f'✓ Deleted old archive: {file_to_delete}')
-                    except Exception as e:
-                        print(f'✗ Deletion failed {file_to_delete}: {str(e)}')
-            
-            # LFS cleanup options
-            cleanup_method = os.environ.get('LFS_CLEANUP_METHOD', 'none')
-            
-            if cleanup_method == 'squash' and len(files_to_delete) > 0:
-                self._cleanup_lfs_squash(api)
-            elif cleanup_method == 'recreate' and len(files_to_delete) > 0:
-                if not self._cleanup_lfs_recreate(api, archive_files, max_files, archive_prefix, archive_extension):
+            # Check if adding 1 new file would exceed the limit
+            if len(archive_files) + 1 > max_files:
+                print(f'🚨 Adding new archive would exceed limit ({len(archive_files)} + 1 > {max_files})')
+                print('🚨 Starting force dataset recreation to clean up old archives')
+                
+                # Keep only the newest (max_files - 1) files to make room for the new one
+                files_to_keep = archive_files[:max_files-1] if max_files > 1 else []
+                
+                if not self._cleanup_lfs_recreate(api, files_to_keep, max_files, archive_prefix, archive_extension):
                     return False
+                    
+                print(f'✓ Dataset recreation completed, ready for new archive upload')
+            else:
+                print(f'✓ Archive count check passed ({len(archive_files)} + 1 <= {max_files})')
             
-            print(f'✓ Archive management completed, currently keeping {min(len(archive_files), max_files)} archives')
             return True
         except Exception as e:
-            print(f'✗ Archive management failed: {str(e)}')
+            print(f'✗ Pre-upload cleanup failed: {str(e)}')
             return False
     
-    def _cleanup_lfs_squash(self, api: HfApi) -> None:
-        """Clean LFS history using super_squash_history"""
-        try:
-            print('🔄 Attempting to clean LFS history with super_squash_history...')
-            api.super_squash_history(repo_id=self.dataset_id, repo_type='dataset')
-            print('✅ LFS history cleanup attempted - storage may take time to reflect changes')
-        except Exception as e:
-            print(f'⚠️ LFS history cleanup failed: {str(e)}')
-            print('💡 Consider setting LFS_CLEANUP_METHOD=recreate for stronger cleanup')
-    
-    def _cleanup_lfs_recreate(self, api: HfApi, archive_files: list, max_files: int, 
+
+    def _cleanup_lfs_recreate(self, api: HfApi, files_to_keep: list, max_files: int, 
                               archive_prefix: str, archive_extension: str) -> bool:
         """Force delete and recreate dataset to clean LFS storage"""
         print('🚨 WARNING: Force recreate mode enabled')
         print('📋 This will delete and recreate the entire dataset to clean LFS storage')
         
         # Backup files that need to be preserved
-        remaining_files = archive_files[-(max_files-1):] if max_files > 1 else []
+        remaining_files = files_to_keep
         if remaining_files:
             print(f'📦 Backing up {len(remaining_files)} files for restoration...')
             backup_data = []
@@ -204,8 +190,28 @@ class HFPersistenceManager:
                 print('⚠️ Manual intervention may be required')
                 return False
         else:
-            print('📝 No files to preserve, skipping backup')
-            return True
+            print('📝 No files to preserve, proceeding with dataset cleanup')
+            # Delete dataset even when no files need to be preserved
+            try:
+                print('🗑️ Deleting dataset to clean LFS storage...')
+                api.delete_repo(repo_id=self.dataset_id, repo_type='dataset')
+                print('✓ Dataset deleted successfully')
+                
+                # Wait for deletion to complete
+                time.sleep(10)
+                
+                # Recreate dataset
+                print('🔨 Recreating dataset...')
+                api.create_repo(repo_id=self.dataset_id, repo_type='dataset', exist_ok=True)
+                print('✓ Dataset recreated successfully')
+                
+                print('🎉 Dataset recreation and LFS cleanup completed!')
+                return True
+                
+            except Exception as e:
+                print(f'✗ Dataset recreation failed: {str(e)}')
+                print('⚠️ Manual intervention may be required')
+                return False
     
     def list_available_archives(self, archive_prefix: str, archive_extension: str) -> tuple[bool, str]:
         """
@@ -328,9 +334,10 @@ def main():
             print('✗ upload action requires --archive-file and --filename parameters')
             sys.exit(1)
         
-        success = manager.upload_archive(args.archive_file, args.filename)
+        # First check and cleanup if needed BEFORE uploading
+        success = manager.check_and_cleanup_before_upload(args.archive_prefix, args.archive_extension, args.max_archives)
         if success:
-            success = manager.manage_archives(args.archive_prefix, args.archive_extension, args.max_archives)
+            success = manager.upload_archive(args.archive_file, args.filename)
         
         sys.exit(0 if success else 1)
     

scripts/utils/persistence.sh

@@ -67,11 +67,6 @@ set_default_configuration() {
     # Logging configuration
     export LOG_FILE="${LOG_FILE:-}"
     export LOG_LEVEL="${LOG_LEVEL:-}"
-    
-    # LFS Storage Cleanup configuration
-    export LFS_CLEANUP_METHOD="${LFS_CLEANUP_METHOD:-none}"
-    export FORCE_LFS_CLEANUP="${FORCE_LFS_CLEANUP:-false}"
-    export ENABLE_LFS_MONITORING="${ENABLE_LFS_MONITORING:-false}"
 }
 
 # Validate required environment variables
@@ -113,7 +108,7 @@ create_archive() {
 
     # Build exclude arguments - only exclude files that would negatively impact HuggingFace datasets backup
     local exclude_args=""
-    local default_excludes="__pycache__,*.tmp"
+    local default_excludes="__pycache__,*.tmp,*/temp,*/cache,*/.cache,*/log,*/logs"
     local combined_patterns="${EXCLUDE_PATTERNS:-},${default_excludes}"
 
     if [[ -n "$combined_patterns" ]]; then
@@ -183,7 +178,7 @@ create_archive() {
     fi
 }
 
-# Call Python upload handler
+# Call Python upload handler with pre-upload cleanup
 run_upload_handler() {
     local archive_file="$1"
     local filename="$2"
@@ -196,7 +191,7 @@ run_upload_handler() {
     # Get script directory for relative imports
     local script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
     
-    # Call the standalone Python module
+    # Call the standalone Python module (now with pre-upload cleanup logic)
     python3 "${script_dir}/hf_persistence.py" upload \
         --token "$token" \
         --dataset-id "$dataset_id" \