Add ShareLM dataset analysis dashboard

.gitignore

@@ -0,0 +1,34 @@
+# dependencies
+/node_modules
+/.pnp
+.pnp.js
+
+# testing
+/coverage
+
+# next.js
+/.next/
+/out/
+
+# production
+/build
+
+# misc
+.DS_Store
+*.pem
+
+# debug
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+# local env files
+.env*.local
+.env
+
+# vercel
+.vercel
+
+# typescript
+*.tsbuildinfo
+next-env.d.ts

README.md

@@ -1,2 +1,54 @@
-# shareLM
+# ShareLM Dataset Analysis Dashboard
+
+A Next.js application for analyzing the ShareLM Hugging Face dataset (12GB) with interactive visualizations. The app uses serverless functions to process the dataset and displays:
+
+- **Doughnut chart**: Breakdown of conversations by source
+- **Time series chart**: Total count of conversations over time
+
+## Features
+
+- Streams and processes large datasets efficiently using Hugging Face Datasets Server API
+- In-memory caching to avoid reprocessing on every request
+- Responsive dashboard with beautiful charts using Recharts
+- Error handling and timeout management for large dataset processing
+- Deployed on Vercel serverless functions
+
+## Setup
+
+1. Install dependencies:
+```bash
+npm install
+```
+
+2. Run the development server:
+```bash
+npm run dev
+```
+
+3. Open [http://localhost:3000](http://localhost:3000) in your browser
+
+## Deployment to Vercel
+
+1. Push your code to GitHub
+2. Import the project in Vercel
+3. Deploy (no environment variables needed for basic functionality)
+
+The app will automatically:
+- Cache aggregated statistics to avoid reprocessing
+- Handle timeouts gracefully by processing smaller samples
+- Display loading states and error messages
+
+## Technical Details
+
+- **Framework**: Next.js 14 with TypeScript
+- **Charts**: Recharts
+- **Data Source**: Hugging Face Datasets Server API
+- **Caching**: In-memory cache (1 hour TTL)
+- **Timeout Management**: 25-second timeout with fallback to smaller samples
+
+## Notes
+
+- Due to Vercel's serverless function timeout limits (10s free, 50s pro), the app processes a sample of the dataset (10,000-50,000 rows) rather than the full 3.5M rows
+- Results are cached for 1 hour to improve performance
+- The dataset is accessed via Hugging Face's Datasets Server API
 

app/api/source-breakdown/route.ts

@@ -0,0 +1,122 @@
+import { NextResponse } from 'next/server';
+
+// In-memory cache
+let cachedBreakdown: {
+  data: Array<{ name: string; value: number }>;
+  lastUpdated: number;
+} | null = null;
+
+const CACHE_TTL = 60 * 60 * 1000; // 1 hour
+
+const HF_DATASET_API = 'https://datasets-server.huggingface.co/parquet';
+
+async function fetchDatasetSample(maxRows: number = 50000) {
+  const url = `${HF_DATASET_API}?dataset=shachardon%2FShareLM&config=default&split=train&offset=0&length=${maxRows}`;
+  
+  try {
+    const controller = new AbortController();
+    const timeoutId = setTimeout(() => controller.abort(), 25000); // 25 second timeout
+
+    const response = await fetch(url, {
+      headers: {
+        'Accept': 'application/json',
+      },
+      signal: controller.signal,
+    });
+
+    clearTimeout(timeoutId);
+
+    if (!response.ok) {
+      throw new Error(`HTTP error! status: ${response.status}`);
+    }
+
+    const data = await response.json();
+    return data;
+  } catch (error) {
+    if (error instanceof Error && error.name === 'AbortError') {
+      throw new Error('Request timeout - dataset is too large to process in time limit');
+    }
+    console.error('Error fetching from HF API:', error);
+    throw error;
+  }
+}
+
+export async function GET() {
+  try {
+    // Check cache first
+    if (cachedBreakdown && Date.now() - cachedBreakdown.lastUpdated < CACHE_TTL) {
+      return NextResponse.json({
+        data: cachedBreakdown.data,
+        cached: true,
+      });
+    }
+
+    // Fetch dataset sample - start with smaller sample
+    let datasetData;
+    let maxRows = 10000;
+    
+    try {
+      datasetData = await fetchDatasetSample(maxRows);
+    } catch (error) {
+      // If timeout, try with even smaller sample
+      if (error instanceof Error && error.message.includes('timeout')) {
+        maxRows = 5000;
+        datasetData = await fetchDatasetSample(maxRows);
+      } else {
+        throw error;
+      }
+    }
+    
+    const sourceCounts: Record<string, number> = {};
+    
+    // Handle different response formats
+    let rows: any[] = [];
+    if (datasetData.rows) {
+      rows = datasetData.rows;
+    } else if (Array.isArray(datasetData)) {
+      rows = datasetData;
+    } else if (datasetData.data) {
+      rows = datasetData.data;
+    }
+    
+    let processedCount = 0;
+
+    for (const row of rows) {
+      processedCount++;
+      
+      // Get row data - handle different formats
+      let rowData: any = {};
+      if (row.row) {
+        rowData = row.row;
+      } else if (typeof row === 'object') {
+        rowData = row;
+      }
+      
+      const source = rowData.source || 'unknown';
+      sourceCounts[source] = (sourceCounts[source] || 0) + 1;
+    }
+
+    // Convert to array format for chart
+    const data = Object.entries(sourceCounts)
+      .map(([name, value]) => ({ name, value }))
+      .sort((a, b) => b.value - a.value); // Sort by value descending
+
+    // Update cache
+    cachedBreakdown = {
+      data,
+      lastUpdated: Date.now(),
+    };
+
+    return NextResponse.json({
+      data,
+      processedCount,
+      cached: false,
+    });
+  } catch (error) {
+    console.error('Error processing source breakdown:', error);
+    return NextResponse.json(
+      { error: 'Failed to process dataset', details: error instanceof Error ? error.message : 'Unknown error' },
+      { status: 500 }
+    );
+  }
+}

app/api/stats/route.ts

@@ -0,0 +1,150 @@
+import { NextResponse } from 'next/server';
+
+// In-memory cache (will be reset on serverless function restart)
+let cachedStats: {
+  sourceBreakdown: Record<string, number>;
+  timeSeries: Record<string, number>;
+  lastUpdated: number;
+} | null = null;
+
+const CACHE_TTL = 60 * 60 * 1000; // 1 hour in milliseconds
+
+// Hugging Face API endpoint for dataset
+const HF_DATASET_API = 'https://datasets-server.huggingface.co/parquet';
+
+async function fetchDatasetSample(maxRows: number = 50000) {
+  // Use Hugging Face Datasets Server API to get a sample
+  // For large datasets, we'll process a sample to avoid timeout
+  const url = `${HF_DATASET_API}?dataset=shachardon%2FShareLM&config=default&split=train&offset=0&length=${maxRows}`;
+  
+  try {
+    const controller = new AbortController();
+    const timeoutId = setTimeout(() => controller.abort(), 25000); // 25 second timeout
+
+    const response = await fetch(url, {
+      headers: {
+        'Accept': 'application/json',
+      },
+      signal: controller.signal,
+    });
+
+    clearTimeout(timeoutId);
+
+    if (!response.ok) {
+      throw new Error(`HTTP error! status: ${response.status}`);
+    }
+
+    const data = await response.json();
+    return data;
+  } catch (error) {
+    if (error instanceof Error && error.name === 'AbortError') {
+      throw new Error('Request timeout - dataset is too large to process in time limit');
+    }
+    console.error('Error fetching from HF API:', error);
+    throw error;
+  }
+}
+
+export async function GET() {
+  try {
+    // Check cache first
+    if (cachedStats && Date.now() - cachedStats.lastUpdated < CACHE_TTL) {
+      return NextResponse.json({
+        sourceBreakdown: cachedStats.sourceBreakdown,
+        timeSeries: cachedStats.timeSeries,
+        cached: true,
+      });
+    }
+
+    // Fetch dataset sample from Hugging Face API
+    // Start with smaller sample to avoid timeout
+    let datasetData;
+    let maxRows = 10000;
+    
+    try {
+      datasetData = await fetchDatasetSample(maxRows);
+    } catch (error) {
+      // If timeout, try with even smaller sample
+      if (error instanceof Error && error.message.includes('timeout')) {
+        maxRows = 5000;
+        datasetData = await fetchDatasetSample(maxRows);
+      } else {
+        throw error;
+      }
+    }
+    
+    const sourceBreakdown: Record<string, number> = {};
+    const timeSeries: Record<string, number> = {};
+
+    // Process the data - handle different response formats
+    let rows: any[] = [];
+    if (datasetData.rows) {
+      rows = datasetData.rows;
+    } else if (Array.isArray(datasetData)) {
+      rows = datasetData;
+    } else if (datasetData.data) {
+      rows = datasetData.data;
+    }
+
+    let processedCount = 0;
+
+    for (const row of rows) {
+      processedCount++;
+
+      // Get row data - handle different formats
+      let rowData: any = {};
+      if (row.row) {
+        rowData = row.row;
+      } else if (typeof row === 'object') {
+        rowData = row;
+      }
+      
+      // Aggregate by source
+      const source = rowData.source || 'unknown';
+      sourceBreakdown[source] = (sourceBreakdown[source] || 0) + 1;
+
+      // Aggregate by timestamp (group by date)
+      if (rowData.timestamp) {
+        try {
+          const date = new Date(rowData.timestamp);
+          if (!isNaN(date.getTime())) {
+            const dateKey = date.toISOString().split('T')[0]; // YYYY-MM-DD
+            timeSeries[dateKey] = (timeSeries[dateKey] || 0) + 1;
+          }
+        } catch (e) {
+          // Skip invalid timestamps
+        }
+      }
+    }
+
+    // Sort time series by date
+    const sortedTimeSeries: Record<string, number> = {};
+    Object.keys(timeSeries)
+      .sort()
+      .forEach((key) => {
+        sortedTimeSeries[key] = timeSeries[key];
+      });
+
+    // Update cache
+    cachedStats = {
+      sourceBreakdown,
+      timeSeries: sortedTimeSeries,
+      lastUpdated: Date.now(),
+    };
+
+    return NextResponse.json({
+      sourceBreakdown,
+      timeSeries: sortedTimeSeries,
+      processedCount,
+      cached: false,
+      sampleSize: maxRows,
+      note: processedCount < maxRows ? 'Full sample processed' : `Processed ${processedCount} rows (limited to avoid timeout)`,
+    });
+  } catch (error) {
+    console.error('Error processing dataset:', error);
+    return NextResponse.json(
+      { error: 'Failed to process dataset', details: error instanceof Error ? error.message : 'Unknown error' },
+      { status: 500 }
+    );
+  }
+}

app/globals.css

@@ -0,0 +1,27 @@
+* {
+  box-sizing: border-box;
+  padding: 0;
+  margin: 0;
+}
+
+html,
+body {
+  max-width: 100vw;
+  overflow-x: hidden;
+  font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Roboto', 'Oxygen',
+    'Ubuntu', 'Cantarell', 'Fira Sans', 'Droid Sans', 'Helvetica Neue',
+    sans-serif;
+  -webkit-font-smoothing: antialiased;
+  -moz-osx-font-smoothing: grayscale;
+  background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
+  min-height: 100vh;
+}
+
+body {
+  color: #333;
+}
+
+a {
+  color: inherit;
+  text-decoration: none;
+}

app/layout.tsx

@@ -0,0 +1,19 @@
+import type { Metadata } from 'next'
+import './globals.css'
+
+export const metadata: Metadata = {
+  title: 'ShareLM Dataset Analysis',
+  description: 'Analysis dashboard for the ShareLM Hugging Face dataset',
+}
+
+export default function RootLayout({
+  children,
+}: {
+  children: React.ReactNode
+}) {
+  return (
+    <html lang="en">
+      <body>{children}</body>
+    </html>
+  )
+}

app/page.module.css

@@ -0,0 +1,68 @@
+.mainContainer {
+  padding: 2rem;
+  max-width: 1400px;
+  margin: 0 auto;
+}
+
+.header {
+  text-align: center;
+  margin-bottom: 3rem;
+  color: white;
+}
+
+.header h1 {
+  font-size: 3rem;
+  font-weight: 700;
+  margin-bottom: 0.5rem;
+  text-shadow: 2px 2px 4px rgba(0, 0, 0, 0.2);
+}
+
+.subtitle {
+  font-size: 1.2rem;
+  opacity: 0.9;
+}
+
+
+.errorBanner {
+  background: #fee;
+  border: 1px solid #fcc;
+  border-radius: 8px;
+  padding: 1rem;
+  margin-bottom: 2rem;
+  display: flex;
+  justify-content: space-between;
+  align-items: center;
+}
+
+.errorBanner button {
+  background: #667eea;
+  color: white;
+  border: none;
+  padding: 0.5rem 1rem;
+  border-radius: 4px;
+  cursor: pointer;
+  font-size: 0.9rem;
+}
+
+.errorBanner button:hover {
+  background: #5568d3;
+}
+
+.statsInfo {
+  background: white;
+  border-radius: 12px;
+  padding: 1.5rem;
+  box-shadow: 0 10px 30px rgba(0, 0, 0, 0.2);
+  display: flex;
+  justify-content: space-around;
+  flex-wrap: wrap;
+  gap: 1rem;
+  text-align: center;
+}
+
+.statsInfo p {
+  font-size: 1rem;
+  color: #333;
+  font-weight: 500;
+}
+

app/page.tsx

@@ -0,0 +1,96 @@
+'use client';
+
+import { useEffect, useState } from 'react';
+import { Charts } from '@/components/Charts';
+import styles from './page.module.css';
+
+interface SourceData {
+  name: string;
+  value: number;
+}
+
+interface TimeSeriesData {
+  date: string;
+  count: number;
+}
+
+export default function Home() {
+  const [sourceData, setSourceData] = useState<SourceData[]>([]);
+  const [timeSeriesData, setTimeSeriesData] = useState<TimeSeriesData[]>([]);
+  const [loading, setLoading] = useState(true);
+  const [error, setError] = useState<string | null>(null);
+
+  useEffect(() => {
+    async function fetchData() {
+      try {
+        setLoading(true);
+        setError(null);
+
+        // Fetch stats from API
+        const response = await fetch('/api/stats');
+        
+        if (!response.ok) {
+          throw new Error(`Failed to fetch data: ${response.statusText}`);
+        }
+
+        const data = await response.json();
+
+        if (data.error) {
+          throw new Error(data.error);
+        }
+
+        // Format source breakdown data
+        const sourceBreakdown = Object.entries(data.sourceBreakdown || {}).map(([name, value]) => ({
+          name,
+          value: value as number,
+        })).sort((a, b) => b.value - a.value);
+
+        // Format time series data
+        const timeSeries = Object.entries(data.timeSeries || {}).map(([date, count]) => ({
+          date,
+          count: count as number,
+        })).sort((a, b) => a.date.localeCompare(b.date));
+
+        setSourceData(sourceBreakdown);
+        setTimeSeriesData(timeSeries);
+      } catch (err) {
+        console.error('Error fetching data:', err);
+        setError(err instanceof Error ? err.message : 'An unknown error occurred');
+      } finally {
+        setLoading(false);
+      }
+    }
+
+    fetchData();
+  }, []);
+
+  return (
+    <main className={styles.mainContainer}>
+      <div className={styles.header}>
+        <h1>ShareLM Dataset Analysis</h1>
+        <p className={styles.subtitle}>Analyzing conversations from the ShareLM Hugging Face dataset</p>
+      </div>
+
+      {error && (
+        <div className={styles.errorBanner}>
+          <p>Error: {error}</p>
+          <button onClick={() => window.location.reload()}>Retry</button>
+        </div>
+      )}
+
+      <Charts 
+        sourceData={sourceData} 
+        timeSeriesData={timeSeriesData} 
+        loading={loading} 
+      />
+
+      {!loading && !error && (
+        <div className={styles.statsInfo}>
+          <p>Total sources: {sourceData.length}</p>
+          <p>Total time points: {timeSeriesData.length}</p>
+          <p>Total conversations analyzed: {sourceData.reduce((sum, item) => sum + item.value, 0).toLocaleString()}</p>
+        </div>
+      )}
+    </main>
+  );
+}

components/Charts.module.css

@@ -0,0 +1,40 @@
+.chartContainer {
+  background: white;
+  border-radius: 12px;
+  padding: 2rem;
+  box-shadow: 0 10px 30px rgba(0, 0, 0, 0.2);
+}
+
+.chartContainer h2 {
+  margin-bottom: 1.5rem;
+  color: #333;
+  font-size: 1.5rem;
+  text-align: center;
+}
+
+.loadingState,
+.errorState {
+  text-align: center;
+  padding: 3rem;
+  color: #666;
+  font-size: 1.1rem;
+}
+
+.chartsWrapper {
+  display: grid;
+  grid-template-columns: 1fr;
+  gap: 2rem;
+  margin-bottom: 2rem;
+}
+
+@media (min-width: 768px) {
+  .chartsWrapper {
+    grid-template-columns: 1fr 1fr;
+  }
+}
+
+@media (min-width: 1200px) {
+  .chartsWrapper {
+    grid-template-columns: 1fr 1fr;
+  }
+}

components/Charts.tsx

@@ -0,0 +1,116 @@
+'use client';
+
+import { PieChart, Pie, Cell, ResponsiveContainer, Tooltip, Legend } from 'recharts';
+import { LineChart, Line, XAxis, YAxis, CartesianGrid, ResponsiveContainer as LineResponsiveContainer } from 'recharts';
+import styles from './Charts.module.css';
+
+interface SourceData {
+  name: string;
+  value: number;
+}
+
+interface ChartsProps {
+  sourceData: SourceData[];
+  timeSeriesData: Array<{ date: string; count: number }>;
+  loading: boolean;
+}
+
+const COLORS = ['#0088FE', '#00C49F', '#FFBB28', '#FF8042', '#8884d8', '#82ca9d', '#ffc658', '#ff7300'];
+
+export function SourceDoughnutChart({ data, loading }: { data: SourceData[]; loading: boolean }) {
+  if (loading) {
+    return (
+      <div className={styles.chartContainer}>
+        <div className={styles.loadingState}>Loading source breakdown...</div>
+      </div>
+    );
+  }
+
+  if (!data || data.length === 0) {
+    return (
+      <div className={styles.chartContainer}>
+        <div className={styles.errorState}>No data available</div>
+      </div>
+    );
+  }
+
+  return (
+    <div className={styles.chartContainer}>
+      <h2>Source Breakdown</h2>
+      <ResponsiveContainer width="100%" height={400}>
+        <PieChart>
+          <Pie
+            data={data}
+            cx="50%"
+            cy="50%"
+            labelLine={false}
+            label={({ name, percent }) => `${name}: ${(percent * 100).toFixed(1)}%`}
+            outerRadius={120}
+            innerRadius={60}
+            fill="#8884d8"
+            dataKey="value"
+          >
+            {data.map((entry, index) => (
+              <Cell key={`cell-${index}`} fill={COLORS[index % COLORS.length]} />
+            ))}
+          </Pie>
+          <Tooltip />
+          <Legend />
+        </PieChart>
+      </ResponsiveContainer>
+    </div>
+  );
+}
+
+export function TimeSeriesChart({ data, loading }: { data: Array<{ date: string; count: number }>; loading: boolean }) {
+  if (loading) {
+    return (
+      <div className={styles.chartContainer}>
+        <div className={styles.loadingState}>Loading time series data...</div>
+      </div>
+    );
+  }
+
+  if (!data || data.length === 0) {
+    return (
+      <div className={styles.chartContainer}>
+        <div className={styles.errorState}>No data available</div>
+      </div>
+    );
+  }
+
+  // Format data for chart (sample if too many points)
+  const chartData = data.length > 100 
+    ? data.filter((_, i) => i % Math.ceil(data.length / 100) === 0)
+    : data;
+
+  return (
+    <div className={styles.chartContainer}>
+      <h2>Total Count Over Time</h2>
+      <LineResponsiveContainer width="100%" height={400}>
+        <LineChart data={chartData}>
+          <CartesianGrid strokeDasharray="3 3" />
+          <XAxis 
+            dataKey="date" 
+            angle={-45}
+            textAnchor="end"
+            height={100}
+            interval="preserveStartEnd"
+          />
+          <YAxis />
+          <Tooltip />
+          <Line type="monotone" dataKey="count" stroke="#8884d8" strokeWidth={2} dot={{ r: 3 }} />
+        </LineChart>
+      </LineResponsiveContainer>
+    </div>
+  );
+}
+
+export function Charts({ sourceData, timeSeriesData, loading }: ChartsProps) {
+  return (
+    <div className={styles.chartsWrapper}>
+      <SourceDoughnutChart data={sourceData} loading={loading} />
+      <TimeSeriesChart data={timeSeriesData} loading={loading} />
+    </div>
+  );
+}

next.config.js

@@ -0,0 +1,10 @@
+/** @type {import('next').NextConfig} */
+const nextConfig = {
+  experimental: {
+    serverActions: {
+      bodySizeLimit: '10mb',
+    },
+  },
+}
+
+module.exports = nextConfig

package.json

@@ -0,0 +1,25 @@
+{
+  "name": "sharelm-analysis",
+  "version": "0.1.0",
+  "private": true,
+  "scripts": {
+    "dev": "next dev",
+    "build": "next build",
+    "start": "next start",
+    "lint": "next lint"
+  },
+  "dependencies": {
+    "next": "^14.2.0",
+    "react": "^18.3.0",
+    "react-dom": "^18.3.0",
+    "recharts": "^2.12.0"
+  },
+  "devDependencies": {
+    "@types/node": "^20.11.0",
+    "@types/react": "^18.2.0",
+    "@types/react-dom": "^18.2.0",
+    "typescript": "^5.3.0",
+    "eslint": "^8.56.0",
+    "eslint-config-next": "^14.2.0"
+  }
+}

tsconfig.json

@@ -0,0 +1,27 @@
+{
+  "compilerOptions": {
+    "target": "ES2020",
+    "lib": ["dom", "dom.iterable", "esnext"],
+    "allowJs": true,
+    "skipLibCheck": true,
+    "strict": true,
+    "noEmit": true,
+    "esModuleInterop": true,
+    "module": "esnext",
+    "moduleResolution": "bundler",
+    "resolveJsonModule": true,
+    "isolatedModules": true,
+    "jsx": "preserve",
+    "incremental": true,
+    "plugins": [
+      {
+        "name": "next"
+      }
+    ],
+    "paths": {
+      "@/*": ["./*"]
+    }
+  },
+  "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"],
+  "exclude": ["node_modules"]
+}

vercel.json

@@ -0,0 +1,7 @@
+{
+  "functions": {
+    "app/api/**/*.ts": {
+      "maxDuration": 30
+    }
+  }
+}