Update download.py

download.py

@@ -195,12 +195,28 @@ def download(download_list: list, output_dir: str, is_clean_cache: bool):
             if is_clean_cache:
                 clean_huggingface_cache(output_dir, repo)
             
-            # unzip the file 
+            # unzip the file
             if rel_path.endswith('.zip'):
                 zip_file = join(output_dir, rel_path)
+                hash_name = os.path.basename(rel_path).replace('.zip', '')
+                # Create target directory: output_dir/batch/hash_name
+                ofile = join(output_dir, os.path.dirname(rel_path), hash_name)
+                os.makedirs(ofile, exist_ok=True)
+
                 with zipfile.ZipFile(zip_file, 'r') as zip_ref:
-                    ofile = join(output_dir, os.path.dirname(rel_path))
                     zip_ref.extractall(ofile)
+
+                # Check if there's a nested hash/hash/ structure
+                inner_hash_dir = join(ofile, hash_name)
+                if os.path.exists(inner_hash_dir) and os.path.isdir(inner_hash_dir):
+                    # Move all contents from inner hash dir to outer hash dir
+                    for item in os.listdir(inner_hash_dir):
+                        src = join(inner_hash_dir, item)
+                        dst = join(ofile, item)
+                        shutil.move(src, dst)
+                    # Remove the now-empty inner hash directory
+                    os.rmdir(inner_hash_dir)
+
                 os.remove(zip_file)
         else:
             print(f'Download {rel_path} failed')