Upload combined_data_stats.json with huggingface_hub

combined_data_stats.json

@@ -0,0 +1,72 @@
+{
+  "total_samples": 15271,
+  "data_sources": {
+    "1_2_m_academic_v0_1": 865,
+    "None": 8262,
+    "30_60_s_academic_v0_1": 2096,
+    "0_30_s_perceptiontest": 470,
+    "0_30_s_academic_v0_1": 1411,
+    "2_3_m_academic_v0_1": 806,
+    "0_30_s_nextqa": 131,
+    "1_2_m_nextqa": 95,
+    "30_60_s_perceptiontest": 167,
+    "30_60_s_nextqa": 139,
+    "clevrer": 829
+  },
+  "assistant_stats": {
+    "characters": {
+      "min": 37.0,
+      "max": 10918.0,
+      "avg": 1817.258201820444,
+      "count": 15271
+    },
+    "words": {
+      "min": 4.0,
+      "max": 1918.0,
+      "avg": 321.41202278829155,
+      "count": 15271
+    },
+    "tokens": {
+      "min": 9.25,
+      "max": 2729.5,
+      "avg": 454.314550455111,
+      "count": 15271,
+      "definition": "tokens = characters / 4.0"
+    }
+  },
+  "unique_videos": {
+    "total_unique_overall": 11816,
+    "per_data_source_unique_counts": {
+      "1_2_m_academic_v0_1": 635,
+      "None": 6349,
+      "30_60_s_academic_v0_1": 1563,
+      "0_30_s_perceptiontest": 402,
+      "0_30_s_academic_v0_1": 1060,
+      "2_3_m_academic_v0_1": 551,
+      "0_30_s_nextqa": 121,
+      "1_2_m_nextqa": 92,
+      "30_60_s_perceptiontest": 140,
+      "30_60_s_nextqa": 131,
+      "clevrer": 814
+    }
+  },
+  "video_metadata": {
+    "ffprobe_available": true,
+    "probed_videos_count": 11816,
+    "duration_seconds": {
+      "min": 5.0,
+      "max": 179.932,
+      "mean": 48.94285781990521,
+      "count": 11816
+    },
+    "resolution_groups": {
+      "below_480p": 5463,
+      "between_480p_720p": 2514,
+      "between_720p_1080p": 2298,
+      "above_1080p": 1541,
+      "unknown": 0
+    },
+    "errors_total": 0,
+    "errors_sample": []
+  }
+}