perf: faster video loading

src/app/[org]/[dataset]/[episode]/episode-viewer.tsx

@@ -69,20 +69,29 @@ function EpisodeViewerInner({ data, org, dataset }: { data: any; org?: string; d
     currentPage * pageSize,
   );
   
-  // Preload adjacent episodes' videos
+  // Preload adjacent episodes' videos via <link rel="preload"> tags
   useEffect(() => {
     if (!org || !dataset) return;
-    
-    const preloadAdjacent = async () => {
-      try {
-        await getAdjacentEpisodesVideoInfo(org, dataset, episodeId, 2);
-        // Preload adjacent episodes for smoother navigation
-      } catch {
-        // Skip preloading on error
-      }
+    const links: HTMLLinkElement[] = [];
+
+    getAdjacentEpisodesVideoInfo(org, dataset, episodeId, 2)
+      .then((adjacentVideos) => {
+        for (const ep of adjacentVideos) {
+          for (const v of ep.videosInfo) {
+            const link = document.createElement("link");
+            link.rel = "preload";
+            link.as = "video";
+            link.href = v.url;
+            document.head.appendChild(link);
+            links.push(link);
+          }
+        }
+      })
+      .catch(() => {});
+
+    return () => {
+      links.forEach((l) => l.remove());
     };
-    
-    preloadAdjacent();
   }, [org, dataset, episodeId]);
 
   // Initialize based on URL time parameter

src/app/[org]/[dataset]/[episode]/fetch-data.ts

@@ -1,13 +1,11 @@
 import {
   DatasetMetadata,
-  fetchJson,
   fetchParquetFile,
   formatStringWithVars,
-  readParquetColumn,
   readParquetAsObjects,
 } from "@/utils/parquetUtils";
 import { pick } from "@/utils/pick";
-import { getDatasetVersion, buildVersionedUrl } from "@/utils/versionUtils";
+import { getDatasetVersionAndInfo, buildVersionedUrl } from "@/utils/versionUtils";
 
 const SERIES_NAME_DELIMITER = " | ";
 
@@ -18,16 +16,13 @@ export async function getEpisodeData(
 ) {
   const repoId = `${org}/${dataset}`;
   try {
-    // Check for compatible dataset version (v3.0, v2.1, or v2.0)
-    const version = await getDatasetVersion(repoId);
-    const jsonUrl = buildVersionedUrl(repoId, version, "meta/info.json");
-    const info = await fetchJson<DatasetMetadata>(jsonUrl);
+    const { version, info: rawInfo } = await getDatasetVersionAndInfo(repoId);
+    const info = rawInfo as unknown as DatasetMetadata;
 
     if (info.video_path === null) {
       throw new Error("Only videos datasets are supported in this visualizer.\nPlease use Rerun visualizer for images datasets.");
     }
 
-    // Handle different versions
     if (version === "v3.0") {
       return await getEpisodeDataV3(repoId, version, info, episodeId);
     } else {
@@ -39,7 +34,6 @@ export async function getEpisodeData(
   }
 }
 
-// Get video info for adjacent episodes (for preloading)
 export async function getAdjacentEpisodesVideoInfo(
   org: string,
   dataset: string,
@@ -48,9 +42,8 @@ export async function getAdjacentEpisodesVideoInfo(
 ) {
   const repoId = `${org}/${dataset}`;
   try {
-    const version = await getDatasetVersion(repoId);
-    const jsonUrl = buildVersionedUrl(repoId, version, "meta/info.json");
-    const info = await fetchJson<DatasetMetadata>(jsonUrl);
+    const { version, info: rawInfo } = await getDatasetVersionAndInfo(repoId);
+    const info = rawInfo as unknown as DatasetMetadata;
     
     const totalEpisodes = info.total_episodes;
     const adjacentVideos: Array<{episodeId: number; videosInfo: any[]}> = [];
@@ -196,47 +189,34 @@ async function getEpisodeDataV2(
   );
 
   const arrayBuffer = await fetchParquetFile(parquetUrl);
+  const allData = await readParquetAsObjects(arrayBuffer, []);
   
-  // Extract task - first check for language instructions (preferred), then fallback to task field or tasks.jsonl
+  // Extract task from language_instruction fields, task field, or tasks.jsonl
   let task: string | undefined;
-  let allData: any[] = [];
   
-  // Load data first
-  try {
-    allData = await readParquetAsObjects(arrayBuffer, []);
-  } catch (error) {
-    // Could not read parquet data
-  }
-  
-  // First check for language_instruction fields in the data (preferred)
   if (allData.length > 0) {
     const firstRow = allData[0];
     const languageInstructions: string[] = [];
     
-    // Check for language_instruction field
     if (firstRow.language_instruction) {
       languageInstructions.push(firstRow.language_instruction);
     }
     
-    // Check for numbered language_instruction fields
     let instructionNum = 2;
     while (firstRow[`language_instruction_${instructionNum}`]) {
       languageInstructions.push(firstRow[`language_instruction_${instructionNum}`]);
       instructionNum++;
     }
     
-    // Join all instructions with line breaks
     if (languageInstructions.length > 0) {
       task = languageInstructions.join('\n');
     }
   }
   
-  // If no language instructions found, try direct task field
   if (!task && allData.length > 0 && allData[0].task) {
     task = allData[0].task;
   }
   
-  // If still no task found, try loading from tasks.jsonl metadata file (v2.x format)
   if (!task && allData.length > 0) {
     try {
       const tasksUrl = buildVersionedUrl(repoId, version, "meta/tasks.jsonl");
@@ -244,7 +224,6 @@ async function getEpisodeDataV2(
       
       if (tasksResponse.ok) {
         const tasksText = await tasksResponse.text();
-        // Parse JSONL format (one JSON object per line)
         const tasksData = tasksText
           .split('\n')
           .filter(line => line.trim())
@@ -252,11 +231,7 @@ async function getEpisodeDataV2(
         
         if (tasksData && tasksData.length > 0) {
           const taskIndex = allData[0].task_index;
-          
-          // Convert BigInt to number for comparison
           const taskIndexNum = typeof taskIndex === 'bigint' ? Number(taskIndex) : taskIndex;
-          
-          // Find task by task_index
           const taskData = tasksData.find(t => t.task_index === taskIndexNum);
           if (taskData) {
             task = taskData.task;
@@ -268,19 +243,25 @@ async function getEpisodeDataV2(
     }
   }
   
-  const data = await readParquetColumn(arrayBuffer, filteredColumnNames);
-  // Flatten and map to array of objects for chartData
+  // Build chart data from already-parsed allData (no second parquet parse)
   const seriesNames = [
     "timestamp",
     ...columns.map(({ value }) => value).flat(),
   ];
 
-  const chartData = data.map((row) => {
-    const flatRow = row.flat();
+  const chartData = allData.map((row) => {
     const obj: Record<string, number> = {};
-    seriesNames.forEach((key, idx) => {
-      obj[key] = flatRow[idx];
-    });
+    obj["timestamp"] = row.timestamp;
+    for (const col of columns) {
+      const rawVal = row[col.key];
+      if (Array.isArray(rawVal)) {
+        rawVal.forEach((v: any, i: number) => {
+          if (i < col.value.length) obj[col.value[i]] = Number(v);
+        });
+      } else if (rawVal !== undefined) {
+        obj[col.value[0]] = Number(rawVal);
+      }
+    }
     return obj;
   });
 

src/app/[org]/[dataset]/[episode]/page.tsx

@@ -27,7 +27,7 @@ export default async function EpisodePage({
   const { data, error } = await getEpisodeDataSafe(org, dataset, episodeNumber);
   return (
     <Suspense fallback={null}>
-      <EpisodeViewer data={data} error={error} />
+      <EpisodeViewer data={data} error={error} org={org} dataset={dataset} />
     </Suspense>
   );
 }

src/components/simple-videos-player.tsx

@@ -252,7 +252,7 @@ export const SimpleVideosPlayer = ({
                   isEnlarged ? "max-h-[90vh] max-w-[90vw]" : ""
                 }`}
                 muted
-                preload="auto"
+                preload={isFirstVisible ? "auto" : "metadata"}
                 onPlay={(e) => handlePlay(e.currentTarget, info)}
                 onTimeUpdate={isFirstVisible ? handleTimeUpdate : undefined}
               >

src/components/videos-player.tsx

@@ -395,7 +395,7 @@ export const VideosPlayer = ({
                 }}
                 muted
                 loop
-                preload="auto"
+                preload={idx === firstVisibleIdx ? "auto" : "metadata"}
                 className={`w-full object-contain ${isEnlarged ? "max-h-[90vh] max-w-[90vw]" : ""}`}
                 onTimeUpdate={
                   idx === firstVisibleIdx ? handleTimeUpdate : undefined

src/utils/versionUtils.ts

@@ -23,19 +23,24 @@ interface DatasetInfo {
   features: Record<string, any>;
 }
 
-/**
- * Fetches dataset information from the main revision
- */
+// In-memory cache for dataset info (5 min TTL)
+const datasetInfoCache = new Map<string, { data: DatasetInfo; expiry: number }>();
+const CACHE_TTL_MS = 5 * 60 * 1000;
+
 export async function getDatasetInfo(repoId: string): Promise<DatasetInfo> {
+  const cached = datasetInfoCache.get(repoId);
+  if (cached && Date.now() < cached.expiry) {
+    return cached.data;
+  }
+
   try {
     const testUrl = `${DATASET_URL}/${repoId}/resolve/main/meta/info.json`;
     
     const controller = new AbortController();
-    const timeoutId = setTimeout(() => controller.abort(), 10000); // 10 second timeout
+    const timeoutId = setTimeout(() => controller.abort(), 10000);
     
     const response = await fetch(testUrl, { 
       method: "GET",
-      cache: "no-store",
       signal: controller.signal
     });
     
@@ -47,11 +52,11 @@ export async function getDatasetInfo(repoId: string): Promise<DatasetInfo> {
 
     const data = await response.json();
     
-    // Check if it has the required structure
     if (!data.features) {
       throw new Error("Dataset info.json does not have the expected features structure");
     }
     
+    datasetInfoCache.set(repoId, { data: data as DatasetInfo, expiry: Date.now() + CACHE_TTL_MS });
     return data as DatasetInfo;
   } catch (error) {
     if (error instanceof Error) {
@@ -64,40 +69,33 @@ export async function getDatasetInfo(repoId: string): Promise<DatasetInfo> {
   }
 }
 
+const SUPPORTED_VERSIONS = ["v3.0", "v2.1", "v2.0"];
 
 /**
- * Gets the dataset version by reading the codebase_version from the main revision's info.json
+ * Returns both the validated version string and the dataset info in one call,
+ * avoiding a duplicate info.json fetch.
  */
-export async function getDatasetVersion(repoId: string): Promise<string> {
-  try {
-    const datasetInfo = await getDatasetInfo(repoId);
-    
-    // Extract codebase_version
-    const codebaseVersion = datasetInfo.codebase_version;
-    if (!codebaseVersion) {
-      throw new Error("Dataset info.json does not contain codebase_version");
-    }
-    
-    // Validate that it's a supported version
-    const supportedVersions = ["v3.0", "v2.1", "v2.0"];
-    if (!supportedVersions.includes(codebaseVersion)) {
-      throw new Error(
-        `Dataset ${repoId} has codebase version ${codebaseVersion}, which is not supported. ` +
-        "This tool only works with dataset versions 3.0, 2.1, or 2.0. " +
-        "Please use a compatible dataset version."
-      );
-    }
-    
-    return codebaseVersion;
-  } catch (error) {
-    if (error instanceof Error) {
-      throw error;
-    }
+export async function getDatasetVersionAndInfo(repoId: string): Promise<{ version: string; info: DatasetInfo }> {
+  const info = await getDatasetInfo(repoId);
+
+  const version = info.codebase_version;
+  if (!version) {
+    throw new Error("Dataset info.json does not contain codebase_version");
+  }
+  if (!SUPPORTED_VERSIONS.includes(version)) {
     throw new Error(
-      `Dataset ${repoId} is not compatible with this visualizer. ` +
-      "Failed to read dataset information from the main revision."
+      `Dataset ${repoId} has codebase version ${version}, which is not supported. ` +
+      "This tool only works with dataset versions 3.0, 2.1, or 2.0. " +
+      "Please use a compatible dataset version."
     );
   }
+
+  return { version, info };
+}
+
+export async function getDatasetVersion(repoId: string): Promise<string> {
+  const { version } = await getDatasetVersionAndInfo(repoId);
+  return version;
 }
 
 export function buildVersionedUrl(repoId: string, version: string, path: string): string {