Update app.py

app.py

@@ -8,7 +8,9 @@ import base64
 from time import gmtime, strftime
 from csv import writer
 import json
-
+import zipfile
+from os import listdir
+from os.path import isfile, join, isdir
 from datasets import load_dataset
 from hfserver import HuggingFaceDatasetSaver, HuggingFaceDatasetJSONSaver
 
@@ -21,7 +23,8 @@ ENVS = ['ShadowHand', 'ShadowHandCatchAbreast', 'ShadowHandOver', 'ShadowHandBlo
 # download data from huggingface dataset
 # dataset = load_dataset("quantumiracle-git/robotinder-data")
 os.remove('.git/hooks/pre-push')  # https://github.com/git-lfs/git-lfs/issues/853
-LOAD_DATA_GOOGLE_DRIVE = True
+LOAD_DATA_GOOGLE_DRIVE = False
+
 if LOAD_DATA_GOOGLE_DRIVE:  # download data from google drive
     # url = 'https://drive.google.com/drive/folders/1JuNQS4R7axTezWj1x4KRAuRt_L26ApxA?usp=sharing'  # './processed/' folder in google drive
     # url = 'https://drive.google.com/drive/folders/1o8Q9eX-J7F326zv4g2MZWlzR46uVkUF2?usp=sharing'  # './processed_zip/' folder in google drive
@@ -35,9 +38,6 @@ if LOAD_DATA_GOOGLE_DRIVE:  # download data from google drive
     # # VIDEO_PATH = 'filter_processed_zip'
     # VIDEO_PATH = 'split_processed_zip'
 
-    # import zipfile
-    # from os import listdir
-    # from os.path import isfile, join, isdir
     # # unzip the zip files to the same location and delete zip files
     # path_to_zip_file = VIDEO_PATH
     # zip_files = [join(path_to_zip_file, f) for f in listdir(path_to_zip_file)]
@@ -50,9 +50,6 @@ if LOAD_DATA_GOOGLE_DRIVE:  # download data from google drive
     #         os.remove(f)
 
     ### multiple urls to handle the retrieve error
-    import zipfile
-    from os import listdir
-    from os.path import isfile, join, isdir
     # urls = [
     #     'https://drive.google.com/drive/folders/1BbQe4XtcsalsvwGVLW9jWCkr-ln5pvyf?usp=share_link',  # './filter_processed_zip/1' folder in google drive
     #     'https://drive.google.com/drive/folders/1saUTUuObPhMJFguc2J_O0K5woCJjYHci?usp=share_link',  # './filter_processed_zip/2' folder in google drive
@@ -101,8 +98,21 @@ if LOAD_DATA_GOOGLE_DRIVE:  # download data from google drive
                     zip_ref.extractall(directory_to_extract_to)
                 os.remove(f)
 
-else:  # local data
-    VIDEO_PATH = 'robotinder-data'
+else:
+    VIDEO_PATH = 'processed-data'
+    path_to_zip_file = VIDEO_PATH
+    zip_files = [join(path_to_zip_file, f) for f in listdir(path_to_zip_file)]
+    for f in zip_files:
+        if f.endswith(".zip"):
+            directory_to_extract_to = path_to_zip_file # extracted file itself contains a folder
+            print(f'extract data {f} to {directory_to_extract_to}')
+            with zipfile.ZipFile(f, 'r') as zip_ref:
+                zip_ref.extractall(directory_to_extract_to)
+            os.remove(f)
+            
+# for test only
+# else:  # local data
+#     VIDEO_PATH = 'robotinder-data'
 
 VIDEO_INFO = os.path.join(VIDEO_PATH, 'video_info.json')