Upload folder using huggingface_hub

.idea/workspace.xml

@@ -5,9 +5,12 @@
   </component>
   <component name="ChangeListManager">
     <list default="true" id="98e215cf-e237-4c1b-a854-08c5c8eeb9db" name="Changes" comment="">
+      <change beforePath="$PROJECT_DIR$/.hugignore" beforeDir="false" afterPath="$PROJECT_DIR$/.hugignore" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/Loader.py" beforeDir="false" afterPath="$PROJECT_DIR$/Loader.py" afterDir="false" />
       <change beforePath="$PROJECT_DIR$/check_videos.py" beforeDir="false" afterPath="$PROJECT_DIR$/check_videos.py" afterDir="false" />
       <change beforePath="$PROJECT_DIR$/corrupted.txt" beforeDir="false" afterPath="$PROJECT_DIR$/corrupted.txt" afterDir="false" />
       <change beforePath="$PROJECT_DIR$/helpers.py" beforeDir="false" afterPath="$PROJECT_DIR$/helpers.py" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/train.py" beforeDir="false" afterPath="$PROJECT_DIR$/train.py" afterDir="false" />
     </list>
     <option name="SHOW_DIALOG" value="false" />
     <option name="HIGHLIGHT_CONFLICTS" value="true" />
@@ -39,19 +42,19 @@
     <option name="hideEmptyMiddlePackages" value="true" />
     <option name="showLibraryContents" value="true" />
   </component>
-  <component name="PropertiesComponent">{
-  &quot;keyToString&quot;: {
-    &quot;RunOnceActivity.OpenProjectViewOnStart&quot;: &quot;true&quot;,
-    &quot;RunOnceActivity.ShowReadmeOnStart&quot;: &quot;true&quot;,
-    &quot;WebServerToolWindowFactoryState&quot;: &quot;false&quot;,
-    &quot;last_opened_file_path&quot;: &quot;/home/milselarch/projects/SUTD/50-035/lipread-project&quot;,
-    &quot;node.js.detected.package.eslint&quot;: &quot;true&quot;,
-    &quot;node.js.detected.package.tslint&quot;: &quot;true&quot;,
-    &quot;node.js.selected.package.eslint&quot;: &quot;(autodetect)&quot;,
-    &quot;node.js.selected.package.tslint&quot;: &quot;(autodetect)&quot;,
-    &quot;vue.rearranger.settings.migration&quot;: &quot;true&quot;
+  <component name="PropertiesComponent"><![CDATA[{
+  "keyToString": {
+    "RunOnceActivity.OpenProjectViewOnStart": "true",
+    "RunOnceActivity.ShowReadmeOnStart": "true",
+    "WebServerToolWindowFactoryState": "false",
+    "last_opened_file_path": "/home/milselarch/projects/SUTD/capstone/dataset-downloaders",
+    "node.js.detected.package.eslint": "true",
+    "node.js.detected.package.tslint": "true",
+    "node.js.selected.package.eslint": "(autodetect)",
+    "node.js.selected.package.tslint": "(autodetect)",
+    "vue.rearranger.settings.migration": "true"
   }
-}</component>
+}]]></component>
   <component name="RecentsManager">
     <key name="CopyFile.RECENT_KEYS">
       <recent name="$PROJECT_DIR$" />
@@ -70,7 +73,8 @@
       <workItem from="1698579018554" duration="14141000" />
       <workItem from="1698650359117" duration="2013000" />
       <workItem from="1698671103984" duration="599000" />
-      <workItem from="1698775864388" duration="1213000" />
+      <workItem from="1698775864388" duration="1530000" />
+      <workItem from="1698837318225" duration="2874000" />
     </task>
     <servers />
   </component>

Loader.py

@@ -15,11 +15,15 @@ class GridLoader(object):
         self.usable_video_filepaths = None
 
     def load_videos(
-        self, verbose=False, blacklist=frozenset({
-            'GRID-dataset/videos/s8/lgazzs.mpg',
-            'GRID-dataset/videos/s8/lbwx9n.mpg'
-        })
+        self, verbose=False, blacklist=frozenset({})
     ) -> List[str]:
+        """
+        :param verbose:
+        whether to
+        :param blacklist:
+        set of filepaths to exclude from training
+        :return:
+        """
         usable_video_filepaths = []
         videos_without_alignment = []
 

train.py

@@ -25,6 +25,7 @@ corrupt_video_paths = set(
 
 loader = GridLoader()
 video_filepaths = loader.load_videos(blacklist=corrupt_video_paths)
+print(f'videos loaded: {len(video_filepaths)}')
 data = tf.data.Dataset.from_tensor_slices(video_filepaths)
 # print('DATA', data)