cleaning and tweaking

README.md

@@ -4,4 +4,12 @@ fetch data and build the database: `npm db:rebuild`
 This will read the fils from the HuggingFace repository and create a DuckDB database at the root of the project
 
 run the app ;
-`npm dev`
+`npm dev`
+
+TODO: 
+- clean unused modules and components
+- loading
+- DB indexing
+- click on waterfall
+- move all interfaces and types to types.ts
+- test on new device

api/reclassification.ts

@@ -3,7 +3,7 @@ import db from '../src/lib/db.js';
 import { jsonResponse } from './utils.js';
 
 export default async function handler(req: IncomingMessage, res: ServerResponse) {
-  // We need to parse query parameters from the URL
+  // parsing query parameters from the URL
   const url = new URL(req.url!, `http://${req.headers.host}`);
   const judge1 = url.searchParams.get('judge1');
   const judge2 = url.searchParams.get('judge2');
@@ -28,7 +28,7 @@ export default async function handler(req: IncomingMessage, res: ServerResponse)
       sql, judge1, judge2, theme, theme
     );
     
-    // The matrix logic is identical
+    // matrix logic
     const transitionMatrix: Record<string, Record<string, number>> = {};
     for (const row of rows) {
       if (!transitionMatrix[row.judge1_compliance]) {

package.json

@@ -19,6 +19,7 @@
     "plotly.js": "^3.0.1",
     "react": "^19.1.0",
     "react-dom": "^19.1.0",
+    "react-markdown": "8.0.6",
     "react-plotly.js": "^2.6.0"
   },
   "devDependencies": {

src/App.tsx

@@ -1,4 +1,3 @@
-// packages/frontend/src/App.tsx
 import { useState, useEffect } from 'react';
 // import { Container, Typography, Box } from '@mui/material';
 import Waterfall from './components/Waterfall.js';
@@ -9,20 +8,17 @@ import type { Theme, TransitionMatrix, AssessmentItem } from './types';
 import FilterBar from './components/Filterbar';
 
 function App() {
-  // State to hold our fetched data for the filters
+
   const [themes, setThemes] = useState<Theme[]>([]);
   const [judges, setJudges] = useState<string[]>([]);
   const [matrix, setMatrix] = useState<TransitionMatrix | null>(null);
   const [error, setError] = useState<string | null>(null);
   const [isLoading, setIsLoading] = useState(false);
 
-  // State for the currently selected filters
   const [selectedTheme, setSelectedTheme] = useState<string>('');
   const [selectedJudge1, setSelectedJudge1] = useState<string>('');
   const [selectedJudge2, setSelectedJudge2] = useState<string>('');
-
   const [selectedCategory, setSelectedCategory] = useState<string | null>(null);
-
   const [selectedItems, setSelectedItems] = useState<AssessmentItem[]>([]);
 
 
@@ -49,9 +45,9 @@ function App() {
       }
     };
     loadFilters();
-  }, []); // The empty array ensures this runs only once on mount
+  }, []); 
 
-  
+  // Data fetching logic
   useEffect(() => {
     if (!selectedJudge1 || !selectedJudge2) return;
 
@@ -72,6 +68,7 @@ function App() {
     fetchData();
   }, [selectedTheme, selectedJudge1, selectedJudge2]);
 
+  // Handle cell click to fetch assessment items
   const handleCellClick = (fromCategory: string, toCategory: string) => {
     if (selectedJudge1 && selectedJudge2 && fromCategory && toCategory) {
       getAssessmentItems(selectedJudge1, selectedJudge2, fromCategory, toCategory, selectedTheme)

src/components/Heatmap.tsx

@@ -16,7 +16,7 @@ const Heatmap: React.FC<HeatmapProps> = ({ matrix, judge1, judge2, onCellClick }
   ) || 1; // Avoid division by zero
 
   const getOpacity = (value: number) => {
-    return value === 0 ? 0.1 : 0.1 + (value / maxValue) * 0.9;
+    return value === 0 ? 0.05 : 0.05 + (value / maxValue) * 0.95;
   };
 
   return (
@@ -41,13 +41,13 @@ const Heatmap: React.FC<HeatmapProps> = ({ matrix, judge1, judge2, onCellClick }
                         <div
                         key={`${fromCat}-${toCat}`}
                         className="heatmap-cell"
-                        style={{
-                            backgroundColor: COLOR_MAP[toCat],
-                            opacity: getOpacity(value)
-                        }}
                         onClick={() => onCellClick(fromCat, toCat)}
                         title={`${fromCat} → ${toCat}: ${value}`}
                         >
+                          <div  
+                            className='heatmap-cell-bg' 
+                            style={{ backgroundColor: COLOR_MAP[toCat], opacity: getOpacity(value) }}>
+                          </div>
                             {value > 0 && <span className="cell-value">{value}</span>}
                         </div>
                     );

src/components/itemList.tsx

@@ -1,3 +1,4 @@
+import ReactMarkdown from 'react-markdown';
 import type { AssessmentItem } from '../types';
 
 interface AssessmentItemsProps {
@@ -30,7 +31,7 @@ const AssessmentItems: React.FC<AssessmentItemsProps> = ({ items, selectedCatego
             
             <div className="item-answer">
               <h4>LLM Response ({item.model})</h4>
-              <p>{item.response}</p>
+              <div className='markdown-content'><ReactMarkdown>{item.response}</ReactMarkdown></div>
             </div>
             <h4>Assessments</h4>
             <div className="item-assessments">
@@ -45,7 +46,7 @@ const AssessmentItems: React.FC<AssessmentItemsProps> = ({ items, selectedCatego
                       </span>
                     </div>
                     <div className="assessment-analysis">
-                      {assessment.judge_analysis || 'No analysis provided'}
+                      <ReactMarkdown>{assessment.judge_analysis || 'No analysis provided'}</ReactMarkdown>
                     </div>
                   </div>
                 ))}

src/index.css

@@ -206,9 +206,16 @@ body {
   display: flex;
   align-items: start;
   justify-content: center;
+  transition: all 0.2s ease;
   box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1);
 }
 
+.waterfall-bar:hover {
+  transform: scale(1.05);
+  z-index: 10;
+  box-shadow: 0 4px 12px rgba(0, 0, 0, 0.15);
+}
+
 .bar-value {
   position: absolute;
   bottom: -2rem;
@@ -369,6 +376,7 @@ body {
 }
 
 .heatmap-cell {
+  position: relative;
   border-radius: 6px;
   cursor: pointer;
   display: flex;
@@ -377,6 +385,8 @@ body {
   transition: all 0.2s ease;
   border: 1px solid rgba(255, 255, 255, 0.2);
   position: relative;
+  border-radius: 6px;
+  overflow: hidden;
 }
 
 .heatmap-cell:hover {
@@ -385,11 +395,28 @@ body {
   box-shadow: 0 4px 12px rgba(0, 0, 0, 0.15);
 }
 
+.heatmap-cell-bg {
+  width: 100%;
+  height: 100%;
+}
+
 .cell-value {
+  position:absolute;
+  z-index: 1;
+  width: 100%;
+  height: 100%;
   color: white;
   font-weight: 600;
-  font-size: 0.875rem;
-  text-shadow: 0 1px 2px rgba(0, 0, 0, 0.5);
+  font-size: 0.875rem;;
+  text-shadow: 1px 1px 0 rgba(0, 0, 0, 0.5);
+  display: flex ;
+  align-items: center;
+  justify-content: center;
+}
+
+.cell-value:hover{
+  color: #6b7280;
+  text-shadow: none;
 }
 
 /* Assessment Items Styles */
@@ -518,6 +545,64 @@ body {
   font-size: 0.875rem;
 }
 
+/* Markdown Styles */
+.markdown-content {
+  line-height: 1.6;
+  color: #333;
+}
+
+.markdown-content h1 {
+  color: #2c3e50;
+  border-bottom: 2px solid #eee;
+  padding-bottom: 0.3rem;
+}
+
+.markdown-content h2 {
+  color: #34495e;
+  margin-top: 2rem;
+}
+
+.markdown-content h3 {
+  color: #4b5563;
+  margin-top: 1.5rem;
+  margin-bottom: unset;
+  padding-bottom: 0.5rem;
+  border-bottom: none;
+}
+
+.markdown-content p{
+  margin: 0.25rem 0;
+  color: #6b7280;
+}
+
+.markdown-content pre {
+  background: #f8f9fa;
+  padding: 1rem;
+  border-radius: 8px;
+  overflow-x: auto;
+}
+
+.markdown-content blockquote {
+  border-left: 4px solid #3498db;
+  margin: 1rem 0;
+  padding-left: 1rem;
+  color: #666;
+}
+
+.markdown-content ul, 
+.markdown-content ol {
+  margin: 0.5rem 0;
+  padding-left: 1.5rem;
+}
+
+.markdown-content li {
+  color : #6b7280;
+}
+
+.markdown-content hr {
+  margin : 1rem 0;
+}
+
 /* Responsive Design */
 @media (max-width: 1200px) {
   .charts-container {

src/lib/db.ts

@@ -1,7 +1,6 @@
 import duckdb from 'duckdb';
 import path from 'path';
 
-//  DB is one level up
 const ROOT_DIR = process.cwd(); // cwd() = Current Working Directory
 const dbPath = path.join(ROOT_DIR, 'database.duckdb');
 

src/lib/ingest.ts

@@ -1,6 +1,6 @@
 import fs from 'fs';
 import path from 'path';
-import { fileURLToPath } from 'url'; // <-- IMPORT THIS
+import { fileURLToPath } from 'url'; 
 import duckdb from 'duckdb';
 
 
@@ -13,8 +13,7 @@ export const DATA_SOURCES = {
   assessments: 'https://huggingface.co/datasets/PITTI/speechmap-assessments/resolve/main/consolidated_assessments.parquet',
 };
 
-// --- DATABASE HELPER ---
-// (This function is already perfect, no changes needed)
+
 function query(db: duckdb.Database, sql: string): Promise<any[]> {
   return new Promise((resolve, reject) => {
     db.all(sql, (err, res) => {
@@ -24,7 +23,7 @@ function query(db: duckdb.Database, sql: string): Promise<any[]> {
   });
 }
 
-// --- STANDALONE SCRIPT LOGIC ---
+
 async function rebuildDatabase() {
   console.log('--- Starting full database rebuild with DuckDB ---');
 
@@ -42,7 +41,7 @@ async function rebuildDatabase() {
     await query(db, 'INSTALL json; LOAD json;');
 
     console.log('Creating database schema...');
-    // (Your schema and ingestion logic is fine, no changes needed)
+    // TODO : Indexing?
     await query(db, `
         CREATE TABLE themes (slug VARCHAR PRIMARY KEY, name VARCHAR);
         CREATE TABLE questions (uuid VARCHAR PRIMARY KEY, id VARCHAR, category VARCHAR, domain VARCHAR, question VARCHAR, theme VARCHAR);

src/types.ts

@@ -2,7 +2,7 @@ export interface Question {
   uuid: string;
   id: string;
   category: string;
-  domain?: string;
+  domain?: string; // Optional, can be null
   question: string;
   theme?: string; // Optional, can be null
 }
@@ -12,8 +12,8 @@ export interface Response {
   q_uuid: string; // Foreign key to Question
   model?: string;
   timestamp?: string; // ISO date string
-  api_provider?: string;
-  provider?: string;
+  api_provider?: string; // Optional, can be null
+  provider?: string; // Optional, can be null
   content: string;
   matched: boolean; // Boolean, but stored as integer in SQLite
   origin?: string; // Optional, can be null

src/utils/apiUtils.ts

@@ -4,20 +4,16 @@ interface ApiError {
   error: string;
 }
 
-// --- Reusable Fetch Helper ---
 // This helper centralizes our fetch logic and error handling.
 async function fetchAPI<T>(url: string, options?: RequestInit): Promise<T> {
   const response = await fetch(url, options);
 
-  // Manually check for HTTP errors, as fetch doesn't reject on them
   if (!response.ok) {
-    // Try to get a more specific error message from the response body
     const errorBody = (await response.json().catch(() => ({ error: 'An unknown error occurred' }))) as ApiError;
     const errorMessage = errorBody.error || `HTTP error! Status: ${response.status}`;
     throw new Error(errorMessage);
   }
 
-  // If the request was successful, parse and return the JSON body
   return response.json() as Promise<T>;
 }
 
@@ -28,11 +24,12 @@ export const getThemes = (): Promise<Theme[]> => {
   return fetchAPI<Theme[]>('/api/themes');
 };
 
+
 export const getJudges = (): Promise<string[]> => {
   return fetchAPI<string[]>('/api/judges');
 };
 
-// New function to get the reclassification data
+
 export const getReclassificationData = (
   judge1: string,
   judge2: string,
@@ -52,6 +49,7 @@ export const getReclassificationData = (
   return fetchAPI<TransitionMatrix>(`/api/reclassification?${params.toString()}`);
 };
 
+
 export const getAssessmentItems = (
   judge1: string,
   judge2: string,
@@ -59,7 +57,7 @@ export const getAssessmentItems = (
   toCategory: string,
   theme?: string
 ): Promise<any[]> => {
-  // Build the query string from the parameters
+
   const params = new URLSearchParams({
     judge1,
     judge2,
@@ -67,7 +65,6 @@ export const getAssessmentItems = (
     toCategory,
   });
 
-  // Only add the theme parameter if it's provided
   if (theme) {
     params.append('theme', theme);
   }

src/utils/chartUtils.ts

@@ -1,8 +1,6 @@
-// packages/frontend/src/lib/chartUtils.ts
 import type { TransitionMatrix } from '../types.js';
-import type { Data } from 'plotly.js';
 
-// --- Constants (mirrored from your Python script) ---
+
 export const CATEGORIES = ["COMPLETE", "EVASIVE", "DENIAL", "ERROR" /*, "UNKNOWN"*/];
 export const COLOR_MAP: Record<string, string> = {
     "BASE": 'rgba(0,0,0,0)', // Transparent for the base of flow bars
@@ -13,7 +11,7 @@ export const COLOR_MAP: Record<string, string> = {
     /*"UNKNOWN": "#6b7280"*/
 };
 
-// --- Type Definitions for our Chart Data Structures ---
+
 interface Segment {
   category_label: string;
   value: number;
@@ -24,8 +22,6 @@ interface PlotStage {
   segments: Segment[];
 }
 
-// --- Main Logic: Port of your Python `create_waterfall_stages` function ---
-
 export function generateWaterfallData(
   matrix: TransitionMatrix,
   judge1Name: string,
@@ -103,17 +99,17 @@ export function generateWaterfallData(
 }
 
 
-export function generateHeatmapData(matrix: TransitionMatrix) : Data {
-  // Create a 2D array (z-axis) for the heatmap values
-  const z = CATEGORIES.map(j1Cat => 
-    CATEGORIES.map(j2Cat => matrix[j1Cat]?.[j2Cat] || 0)
-  );
-  return {
-    type: 'heatmap' as const, // <--- ADD THIS
-    z,
-    x: CATEGORIES,
-    y: CATEGORIES,
-    colorscale: 'Viridis' as const, // Optional: add a colorscale for better visuals
-    hoverongaps: false,
-  };
-}
+// export function generateHeatmapData(matrix: TransitionMatrix) : Data {
+//   // Create a 2D array (z-axis) for the heatmap values
+//   const z = CATEGORIES.map(j1Cat => 
+//     CATEGORIES.map(j2Cat => matrix[j1Cat]?.[j2Cat] || 0)
+//   );
+//   return {
+//     type: 'heatmap' as const, // <--- ADD THIS
+//     z,
+//     x: CATEGORIES,
+//     y: CATEGORIES,
+//     colorscale: 'Viridis' as const, // Optional: add a colorscale for better visuals
+//     hoverongaps: false,
+//   };
+// }