Fix 401 error: Add authenticated API proxy for private datasets

src/app/[org]/[dataset]/[episode]/fetch-data.ts

@@ -8,6 +8,7 @@ import {
 } from "@/utils/parquetUtils";
 import { pick } from "@/utils/pick";
 import { getDatasetVersion, buildVersionedUrl } from "@/utils/versionUtils";
+import { buildProxyUrl } from "@/utils/apiHelpers";
 
 const SERIES_NAME_DELIMITER = " | ";
 
@@ -241,7 +242,7 @@ async function getEpisodeDataV2(
     try {
       const tasksUrl = buildVersionedUrl(repoId, version, "meta/tasks.jsonl");
       // Use proxy for authenticated requests
-      const proxyUrl = `/api/hf-proxy?url=${encodeURIComponent(tasksUrl)}`;
+      const proxyUrl = buildProxyUrl(tasksUrl);
       const tasksResponse = await fetch(proxyUrl);
       
       if (tasksResponse.ok) {

src/app/api/hf-proxy/route.ts

@@ -15,15 +15,28 @@ export async function HEAD(request: NextRequest) {
 
 async function handleRequest(request: NextRequest, method: 'GET' | 'HEAD') {
   const searchParams = request.nextUrl.searchParams;
-  const url = searchParams.get('url');
+  const urlParam = searchParams.get('url');
   
-  if (!url) {
+  if (!urlParam) {
     return NextResponse.json(
       { error: 'url parameter is required' },
       { status: 400 }
     );
   }
 
+  // Decode and validate URL
+  let url: string;
+  try {
+    url = decodeURIComponent(urlParam);
+    // Validate it's a proper URL
+    new URL(url); // This will throw if invalid
+  } catch (error) {
+    return NextResponse.json(
+      { error: `Invalid URL: ${urlParam}. ${error instanceof Error ? error.message : 'Failed to parse URL'}` },
+      { status: 400 }
+    );
+  }
+
   // Get token from environment variable (set in Hugging Face Space secrets)
   // Try multiple possible variable names that Hugging Face Spaces might use
   const token = 

src/app/explore/page.tsx

@@ -6,6 +6,7 @@ import {
   formatStringWithVars,
 } from "@/utils/parquetUtils";
 import { getDatasetVersion, buildVersionedUrl } from "@/utils/versionUtils";
+import { buildProxyUrl } from "@/utils/apiHelpers";
 
 export default async function ExplorePage({
   searchParams,
@@ -73,7 +74,7 @@ export default async function ExplorePage({
             const url = buildVersionedUrl(repoId, version, videoPath);
             // Check if videoUrl exists (status 200) - use proxy for authenticated requests
             try {
-              const proxyUrl = `/api/hf-proxy?url=${encodeURIComponent(url)}`;
+              const proxyUrl = buildProxyUrl(url);
               const headRes = await fetch(proxyUrl, { method: "HEAD" });
               if (headRes.ok) {
                 videoUrl = url;

src/utils/parquetUtils.ts

@@ -1,4 +1,5 @@
 import { parquetRead, parquetReadObjects } from "hyparquet";
+import { buildProxyUrl } from './apiHelpers';
 
 export interface DatasetMetadata {
   codebase_version: string;
@@ -26,7 +27,7 @@ export interface DatasetMetadata {
 
 export async function fetchJson<T>(url: string): Promise<T> {
   // Use API proxy for authenticated requests (handles private repos)
-  const proxyUrl = `/api/hf-proxy?url=${encodeURIComponent(url)}`;
+  const proxyUrl = buildProxyUrl(url);
   const res = await fetch(proxyUrl);
   if (!res.ok) {
     const errorData = await res.json().catch(() => ({}));
@@ -47,7 +48,7 @@ export function formatStringWithVars(
 // Fetch and parse the Parquet file
 export async function fetchParquetFile(url: string): Promise<ArrayBuffer> {
   // Use API proxy for authenticated requests (handles private repos)
-  const proxyUrl = `/api/hf-proxy?url=${encodeURIComponent(url)}`;
+  const proxyUrl = buildProxyUrl(url);
   const res = await fetch(proxyUrl);
   
   if (!res.ok) {

src/utils/versionUtils.ts

@@ -2,6 +2,8 @@
  * Utility functions for checking dataset version compatibility
  */
 
+import { buildProxyUrl } from './apiHelpers';
+
 const DATASET_URL = process.env.DATASET_URL || "https://huggingface.co/datasets";
 
 /**
@@ -32,7 +34,7 @@ export async function getDatasetInfo(repoId: string): Promise<DatasetInfo> {
     const testUrl = `${DATASET_URL}/${repoId}/resolve/main/meta/info.json`;
     
     // Use API proxy for authenticated requests
-    const proxyUrl = `/api/hf-proxy?url=${encodeURIComponent(testUrl)}`;
+    const proxyUrl = buildProxyUrl(testUrl);
     
     const controller = new AbortController();
     const timeoutId = setTimeout(() => controller.abort(), 10000); // 10 second timeout