Fix 401 error: Add authenticated API proxy for private datasets

src/app/[org]/[dataset]/[episode]/fetch-data.ts

@@ -240,7 +240,9 @@ async function getEpisodeDataV2(
   if (!task && allData.length > 0) {
     try {
       const tasksUrl = buildVersionedUrl(repoId, version, "meta/tasks.jsonl");
-      const tasksResponse = await fetch(tasksUrl);
+      // Use proxy for authenticated requests
+      const proxyUrl = `/api/hf-proxy?url=${encodeURIComponent(tasksUrl)}`;
+      const tasksResponse = await fetch(proxyUrl);
       
       if (tasksResponse.ok) {
         const tasksText = await tasksResponse.text();

src/app/api/hf-proxy/route.ts

@@ -0,0 +1,97 @@
+import { NextRequest, NextResponse } from 'next/server';
+
+/**
+ * Generic proxy API route for Hugging Face requests with authentication
+ * Handles both JSON and binary (Parquet) files
+ * Supports GET and HEAD methods
+ */
+export async function GET(request: NextRequest) {
+  return handleRequest(request, 'GET');
+}
+
+export async function HEAD(request: NextRequest) {
+  return handleRequest(request, 'HEAD');
+}
+
+async function handleRequest(request: NextRequest, method: 'GET' | 'HEAD') {
+  const searchParams = request.nextUrl.searchParams;
+  const url = searchParams.get('url');
+  
+  if (!url) {
+    return NextResponse.json(
+      { error: 'url parameter is required' },
+      { status: 400 }
+    );
+  }
+
+  // Get token from environment variable (set in Hugging Face Space secrets)
+  const token = process.env.HF_TOKEN || process.env.HUGGINGFACE_TOKEN;
+  
+  if (!token) {
+    // If no token, try without auth (for public repos)
+    // But log a warning
+    console.warn('HF_TOKEN not configured in Space secrets - requests may fail for private repos');
+  }
+
+  try {
+    const headers: HeadersInit = {
+      'Cache-Control': 'no-store',
+    };
+    
+    // Add authentication if token is available
+    if (token) {
+      headers['Authorization'] = `Bearer ${token}`;
+    }
+    
+    const response = await fetch(url, {
+      method,
+      headers,
+      cache: 'no-store',
+    });
+
+    if (!response.ok) {
+      // Return error with status code
+      const errorText = await response.text().catch(() => 'Unknown error');
+      return NextResponse.json(
+        { error: `Failed to fetch: ${response.status} ${response.statusText}`, details: errorText },
+        { status: response.status }
+      );
+    }
+
+    // For HEAD requests, just return status without body
+    if (method === 'HEAD') {
+      return new NextResponse(null, {
+        status: response.status,
+        headers: {
+          'Content-Type': response.headers.get('content-type') || '',
+          'Content-Length': response.headers.get('content-length') || '0',
+        },
+      });
+    }
+
+    // Check content type to determine if it's binary or JSON
+    const contentType = response.headers.get('content-type') || '';
+    
+    if (contentType.includes('application/json')) {
+      // Return JSON
+      const data = await response.json();
+      return NextResponse.json(data);
+    } else {
+      // Return binary data (for Parquet files, videos, etc.)
+      const arrayBuffer = await response.arrayBuffer();
+      return new NextResponse(arrayBuffer, {
+        headers: {
+          'Content-Type': contentType,
+          'Content-Length': arrayBuffer.byteLength.toString(),
+        },
+      });
+    }
+  } catch (error) {
+    console.error('Proxy error:', error);
+    return NextResponse.json(
+      { error: error instanceof Error ? error.message : 'Unknown error' },
+      { status: 500 }
+    );
+  }
+}
+

src/app/explore/page.tsx

@@ -71,9 +71,10 @@ export default async function ExplorePage({
               episode_index: "0".padStart(6, "0"),
             });
             const url = buildVersionedUrl(repoId, version, videoPath);
-            // Check if videoUrl exists (status 200)
+            // Check if videoUrl exists (status 200) - use proxy for authenticated requests
             try {
-              const headRes = await fetch(url, { method: "HEAD" });
+              const proxyUrl = `/api/hf-proxy?url=${encodeURIComponent(url)}`;
+              const headRes = await fetch(proxyUrl, { method: "HEAD" });
               if (headRes.ok) {
                 videoUrl = url;
               }

src/utils/parquetUtils.ts

@@ -25,10 +25,13 @@ export interface DatasetMetadata {
 }
 
 export async function fetchJson<T>(url: string): Promise<T> {
-  const res = await fetch(url);
+  // Use API proxy for authenticated requests (handles private repos)
+  const proxyUrl = `/api/hf-proxy?url=${encodeURIComponent(url)}`;
+  const res = await fetch(proxyUrl);
   if (!res.ok) {
+    const errorData = await res.json().catch(() => ({}));
     throw new Error(
-      `Failed to fetch JSON ${url}: ${res.status} ${res.statusText}`,
+      errorData.error || `Failed to fetch JSON ${url}: ${res.status} ${res.statusText}`,
     );
   }
   return res.json() as Promise<T>;
@@ -43,10 +46,13 @@ export function formatStringWithVars(
 
 // Fetch and parse the Parquet file
 export async function fetchParquetFile(url: string): Promise<ArrayBuffer> {
-  const res = await fetch(url);
+  // Use API proxy for authenticated requests (handles private repos)
+  const proxyUrl = `/api/hf-proxy?url=${encodeURIComponent(url)}`;
+  const res = await fetch(proxyUrl);
   
   if (!res.ok) {
-    throw new Error(`Failed to fetch ${url}: ${res.status} ${res.statusText}`);
+    const errorData = await res.json().catch(() => ({}));
+    throw new Error(errorData.error || `Failed to fetch ${url}: ${res.status} ${res.statusText}`);
   }
   
   return res.arrayBuffer();

src/utils/versionUtils.ts

@@ -25,15 +25,19 @@ interface DatasetInfo {
 
 /**
  * Fetches dataset information from the main revision
+ * Uses authenticated API proxy to handle private datasets
  */
 export async function getDatasetInfo(repoId: string): Promise<DatasetInfo> {
   try {
     const testUrl = `${DATASET_URL}/${repoId}/resolve/main/meta/info.json`;
     
+    // Use API proxy for authenticated requests
+    const proxyUrl = `/api/hf-proxy?url=${encodeURIComponent(testUrl)}`;
+    
     const controller = new AbortController();
     const timeoutId = setTimeout(() => controller.abort(), 10000); // 10 second timeout
     
-    const response = await fetch(testUrl, { 
+    const response = await fetch(proxyUrl, { 
       method: "GET",
       cache: "no-store",
       signal: controller.signal
@@ -42,7 +46,8 @@ export async function getDatasetInfo(repoId: string): Promise<DatasetInfo> {
     clearTimeout(timeoutId);
     
     if (!response.ok) {
-      throw new Error(`Failed to fetch dataset info: ${response.status}`);
+      const errorData = await response.json().catch(() => ({}));
+      throw new Error(errorData.error || `Failed to fetch dataset info: ${response.status}`);
     }
 
     const data = await response.json();