|
|
|
|
|
|
|
|
|
|
|
|
|
|
const DATASET_URL = process.env.DATASET_URL || "https://huggingface.co/datasets"; |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
interface DatasetInfo { |
|
|
codebase_version?: string; |
|
|
robot_type?: string | null; |
|
|
total_episodes: number; |
|
|
total_frames: number; |
|
|
total_tasks?: number; |
|
|
chunks_size?: number; |
|
|
data_files_size_in_mb?: number; |
|
|
video_files_size_in_mb?: number; |
|
|
fps: number; |
|
|
splits?: Record<string, string>; |
|
|
data_path: string; |
|
|
video_path: string; |
|
|
features: Record<string, any>; |
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
export async function getDatasetInfo(repoId: string): Promise<DatasetInfo> { |
|
|
try { |
|
|
const testUrl = `${DATASET_URL}/${repoId}/resolve/main/meta/info.json`; |
|
|
|
|
|
const controller = new AbortController(); |
|
|
const timeoutId = setTimeout(() => controller.abort(), 10000); |
|
|
|
|
|
|
|
|
const hfToken = process.env.HF_TOKEN; |
|
|
const headers: HeadersInit = { |
|
|
"Cache-Control": "no-store" |
|
|
}; |
|
|
|
|
|
|
|
|
if (hfToken) { |
|
|
headers["Authorization"] = `Bearer ${hfToken}`; |
|
|
} |
|
|
|
|
|
const response = await fetch(testUrl, { |
|
|
method: "GET", |
|
|
headers: headers, |
|
|
cache: "no-store", |
|
|
signal: controller.signal |
|
|
}); |
|
|
|
|
|
clearTimeout(timeoutId); |
|
|
|
|
|
if (!response.ok) { |
|
|
throw new Error(`Failed to fetch dataset info: ${response.status}`); |
|
|
} |
|
|
|
|
|
const data = await response.json(); |
|
|
|
|
|
|
|
|
if (!data.features) { |
|
|
throw new Error("Dataset info.json does not have the expected features structure"); |
|
|
} |
|
|
|
|
|
return data as DatasetInfo; |
|
|
} catch (error) { |
|
|
if (error instanceof Error) { |
|
|
throw error; |
|
|
} |
|
|
throw new Error( |
|
|
`Dataset ${repoId} is not compatible with this visualizer. ` + |
|
|
"Failed to read dataset information from the main revision." |
|
|
); |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
export async function getDatasetVersion(repoId: string): Promise<string> { |
|
|
try { |
|
|
const datasetInfo = await getDatasetInfo(repoId); |
|
|
|
|
|
|
|
|
const codebaseVersion = datasetInfo.codebase_version ?? "v2.0"; |
|
|
|
|
|
|
|
|
const supportedVersions = ["v3.0", "v2.1", "v2.0"]; |
|
|
if (!supportedVersions.includes(codebaseVersion)) { |
|
|
throw new Error( |
|
|
`Dataset ${repoId} has codebase version ${codebaseVersion}, which is not supported. ` + |
|
|
"This tool only works with dataset versions 3.0, 2.1, or 2.0. " + |
|
|
"Please use a compatible dataset version." |
|
|
); |
|
|
} |
|
|
|
|
|
return codebaseVersion; |
|
|
} catch (error) { |
|
|
if (error instanceof Error) { |
|
|
throw error; |
|
|
} |
|
|
throw new Error( |
|
|
`Dataset ${repoId} is not compatible with this visualizer. ` + |
|
|
"Failed to read dataset information from the main revision." |
|
|
); |
|
|
} |
|
|
} |
|
|
|
|
|
export function buildVersionedUrl(repoId: string, version: string, path: string): string { |
|
|
return `${DATASET_URL}/${repoId}/resolve/main/${path}`; |
|
|
} |
|
|
|
|
|
|