Add audio examples, wip

backend/examples.py

@@ -178,6 +178,26 @@ def build_infos(abs_path: Path, datatype: str, dataset_name: str, db_key: str):
                     )
                     files = files[2:] + files[:2]
 
+                    # For audio and video, we need to check the image_url and make sure it is an image, else convert it:
+                    if datatype == "audio":
+                        files = [
+                            (
+                                (f, n)
+                                if f.endswith(".png")
+                                else (f.replace(".wav", ".png"), n)
+                            )
+                            for f, n in files
+                        ]
+                    elif datatype == "video":
+                        files = [
+                            (
+                                (f, n)
+                                if f.endswith(".mkv")
+                                else (f.replace(".png", ".mkv"), n)
+                            )
+                            for f, n in files
+                        ]
+
                     files = [
                         {
                             "image_url": f,

frontend/src/components/AudioGallery.tsx

@@ -1,6 +1,10 @@
 import React from 'react'
 import AudioPlayer from './AudioPlayer'
 import type { ExamplesData } from './Examples'
+import { groupByNameAndVariant } from './galleryUtils'
+import ExampleMetadata from './ExampleMetadata'
+import ExampleDetailsSection from './ExampleDetailsSection'
+import ExampleVariantSelector from './ExampleVariantSelector'
 
 interface GalleryProps {
   selectedModel: string
@@ -14,15 +18,78 @@ interface GalleryProps {
 
 const AudioGallery: React.FC<GalleryProps> = ({ selectedModel, selectedAttack, examples }) => {
   const exampleItems = examples[selectedModel][selectedAttack]
+  const grouped = groupByNameAndVariant(exampleItems)
+  console.log('Audio examples:', exampleItems)
+  console.log('Grouped audio examples:', grouped)
+  const audioNames = Object.keys(grouped)
+  const [selectedAudio, setSelectedAudio] = React.useState(audioNames[0] || '')
+  const variants = grouped[selectedAudio] || {}
+  const variantKeys = Object.keys(variants)
+  const [selectedVariant, setSelectedVariant] = React.useState(variantKeys[0] || '')
+
+  React.useEffect(() => {
+    setSelectedVariant(variantKeys[0] || '')
+  }, [selectedAudio])
+
+  if (!audioNames.length) {
+    return (
+      <div className="w-full mt-12 flex items-center justify-center">
+        <div className="text-gray-500">
+          No audio examples available. Please select another model and attack.
+        </div>
+      </div>
+    )
+  }
+
+  console.log(variants[selectedVariant])
+
   return (
-    <div className="example-display">
-      {exampleItems.map((item, index) => (
-        <div key={index} className="example-item">
-          <p>{item.name}</p>
-          {item.audio_url && <AudioPlayer src={item.audio_url} />}
-          <img src={item.image_url} alt={item.name} className="example-image" />
+    <div className="w-full overflow-auto" style={{ minHeight: '100vh' }}>
+      <div className="example-display">
+        <div className="mb-4">
+          <fieldset className="fieldset">
+            <legend className="fieldset-legend">Audio Example</legend>
+            <select
+              className="select select-bordered"
+              value={selectedAudio || ''}
+              onChange={(e) => {
+                setSelectedAudio(e.target.value || '')
+              }}
+            >
+              {audioNames.map((name) => (
+                <option key={name} value={name}>
+                  {name}
+                </option>
+              ))}
+            </select>
+          </fieldset>
         </div>
-      ))}
+        {selectedAudio && selectedVariant && variants[selectedVariant] && (
+          <>
+            <ExampleVariantSelector
+              variantKeys={variantKeys}
+              selectedVariant={selectedVariant}
+              setSelectedVariant={setSelectedVariant}
+            />
+            <ExampleMetadata metadata={variants[selectedVariant].metadata || {}} />
+            <ExampleDetailsSection>
+              <div className="flex flex-col items-center gap-4">
+                {variants[selectedVariant].image_url && (
+                  <img
+                    src={variants[selectedVariant].image_url}
+                    alt={selectedAudio}
+                    className="example-image"
+                    style={{ display: 'block' }}
+                  />
+                )}
+                {variants[selectedVariant].audio_url && (
+                  <AudioPlayer src={variants[selectedVariant].audio_url} />
+                )}
+              </div>
+            </ExampleDetailsSection>
+          </>
+        )}
+      </div>
     </div>
   )
 }

frontend/src/components/AudioPlayer.tsx

@@ -1,46 +1,46 @@
-import { useEffect, useMemo, useRef, useState } from 'react'
+import { useEffect, useRef, useState } from 'react'
 import WaveSurfer from 'wavesurfer.js'
 // @ts-ignore: No types for timeline.esm.js
-// import Timeline from 'wavesurfer.js/dist/plugins/timeline.esm.js'
 import TimelinePlugin from 'wavesurfer.js/dist/plugins/timeline.esm.js'
-import API from '../API' // Correct import for the API class
+import API from '../API'
 
 const AudioPlayer = ({ src }: { src: string }) => {
   const containerRef = useRef<HTMLDivElement>(null)
   const wavesurferRef = useRef<WaveSurfer | null>(null)
   const [isPlaying, setIsPlaying] = useState(false)
-  // const plugins = useMemo(() => [TimelinePlugin.create()], [])
 
-  const bottomTimeline = TimelinePlugin.create({
-    height: 16,
-    timeInterval: 0.1,
-    primaryLabelInterval: 1,
-    style: {
-      fontSize: '10px',
-      // color: '#6A3274',
-    },
-  })
-
-  // Initialize WaveSurfer when component mounts
   useEffect(() => {
     if (!containerRef.current) return
 
+    // Destroy previous instance if exists
+    if (wavesurferRef.current) {
+      wavesurferRef.current.destroy()
+      wavesurferRef.current = null
+    }
+
     // Get proxied URL to bypass CORS
     const proxiedUrl = API.getProxiedUrl(src)
 
+    // Create plugin instance inside effect
+    const bottomTimeline = TimelinePlugin.create({
+      height: 16,
+      timeInterval: 0.1,
+      primaryLabelInterval: 1,
+      style: { fontSize: '10px' },
+    })
+
     // Create an instance of WaveSurfer
     wavesurferRef.current = WaveSurfer.create({
       container: containerRef.current,
       waveColor: 'rgb(200, 0, 200)',
       progressColor: 'rgb(100, 0, 100)',
-      url: proxiedUrl, // Use the proxied URL
+      url: proxiedUrl,
       minPxPerSec: 100,
-      barWidth: 10,
-      barRadius: 10,
-      barGap: 2,
+      // barWidth: 10,
+      // barRadius: 10,
+      // barGap: 2,
       mediaControls: true,
-
-      // plugins: [bottomTimeline],
+      plugins: [bottomTimeline],
     })
 
     // Play on click
@@ -48,29 +48,27 @@ const AudioPlayer = ({ src }: { src: string }) => {
       wavesurferRef.current?.play()
       setIsPlaying(true)
     })
-
-    // Rewind to the beginning on finished playing
     wavesurferRef.current.on('finish', () => {
       wavesurferRef.current?.setTime(0)
       setIsPlaying(false)
     })
-
-    // Update playing state
     wavesurferRef.current.on('play', () => setIsPlaying(true))
     wavesurferRef.current.on('pause', () => setIsPlaying(false))
 
     // Cleanup on unmount
     return () => {
       wavesurferRef.current?.destroy()
+      wavesurferRef.current = null
     }
   }, [src])
 
-  const handlePlayPause = () => {
-    wavesurferRef.current?.playPause()
-  }
+  // Optionally, add a play/pause button
+  // const handlePlayPause = () => {
+  //   wavesurferRef.current?.playPause()
+  // }
 
   return (
-    <div className="">
+    <div className="w-full">
       <div ref={containerRef} />
       {/* <button onClick={handlePlayPause}>{isPlaying ? 'Pause' : 'Play'}</button> */}
     </div>

frontend/src/components/ExampleDetailsSection.tsx

@@ -0,0 +1,14 @@
+import React from 'react'
+
+interface ExampleDetailsSectionProps {
+  children: React.ReactNode
+}
+
+const ExampleDetailsSection: React.FC<ExampleDetailsSectionProps> = ({ children }) => (
+  <fieldset className="fieldset w-full p-4 rounded border border-gray-700 bg-base-200 mt-6">
+    <legend className="fieldset-legend font-semibold">Example</legend>
+    {children}
+  </fieldset>
+)
+
+export default ExampleDetailsSection

frontend/src/components/ExampleMetadata.tsx

@@ -0,0 +1,20 @@
+import React from 'react'
+
+interface ExampleMetadataProps {
+  metadata: Record<string, string | number | boolean>
+}
+
+const ExampleMetadata: React.FC<ExampleMetadataProps> = ({ metadata }) => (
+  <fieldset className="fieldset w-full p-4 rounded border border-gray-700 bg-base-200 mt-6">
+    <legend className="fieldset-legend font-semibold">Example Info</legend>
+    <div className="flex flex-wrap gap-x-6 gap-y-2 text-xs">
+      {Object.entries(metadata).map(([k, v]) => (
+        <div key={k} className="flex items-center">
+          <span className="font-mono">{k}</span>: {String(v)}
+        </div>
+      ))}
+    </div>
+  </fieldset>
+)
+
+export default ExampleMetadata

frontend/src/components/ExampleVariantSelector.tsx

@@ -0,0 +1,46 @@
+import React from 'react';
+
+interface ExampleVariantSelectorProps {
+  variantKeys: string[];
+  selectedVariant: string;
+  setSelectedVariant: (v: string) => void;
+}
+
+const ExampleVariantSelector: React.FC<ExampleVariantSelectorProps> = ({ variantKeys, selectedVariant, setSelectedVariant }) => {
+  // Keyboard shortcut for variant switching (keys 1-N)
+  React.useEffect(() => {
+    const handler = (e: KeyboardEvent) => {
+      if (document.activeElement && (document.activeElement as HTMLElement).tagName === 'INPUT') return;
+      const idx = parseInt(e.key, 10);
+      if (!isNaN(idx) && idx > 0 && idx <= variantKeys.length) {
+        setSelectedVariant(variantKeys[idx - 1]);
+      }
+    };
+    window.addEventListener('keydown', handler);
+    return () => window.removeEventListener('keydown', handler);
+  }, [variantKeys, setSelectedVariant]);
+
+  return (
+    <fieldset className="fieldset w-full p-4 rounded border border-gray-700 bg-base-200 mb-4">
+      <legend className="fieldset-legend font-semibold">Variants</legend>
+      <div className="mb-2 flex gap-4 flex-wrap">
+        {variantKeys.map((variant, idx) => (
+          <label key={variant} className="flex items-center gap-1 cursor-pointer">
+            <input
+              type="radio"
+              name={`variant-selector`}
+              value={variant}
+              checked={selectedVariant === variant}
+              onChange={() => setSelectedVariant(variant)}
+            />
+            <span className="text-xs font-semibold">
+              {variant} <span className="opacity-60">[{idx + 1}]</span>
+            </span>
+          </label>
+        ))}
+      </div>
+    </fieldset>
+  );
+};
+
+export default ExampleVariantSelector;

frontend/src/components/ImageGallery.tsx

@@ -1,6 +1,9 @@
 import React from 'react'
 import type { ExamplesData } from './Examples'
 import { groupByNameAndVariant } from './galleryUtils'
+import ExampleMetadata from './ExampleMetadata'
+import ExampleDetailsSection from './ExampleDetailsSection'
+import ExampleVariantSelector from './ExampleVariantSelector'
 
 interface GalleryProps {
   selectedModel: string
@@ -12,61 +15,6 @@ interface GalleryProps {
   }
 }
 
-const Metadata: React.FC<{ metadata: Record<string, string | number | boolean> }> = ({
-  metadata,
-}) => (
-  <fieldset className="fieldset w-full p-4 rounded border border-gray-700 bg-base-200 mb-4">
-    <legend className="fieldset-legend font-semibold">Metadata</legend>
-    <ul className="text-xs flex flex-row gap-4">
-      {(() => {
-        const entries = Object.entries(metadata || {}).filter(([k]) => k !== 'image_url')
-        const detectedIdx = entries.findIndex(([k]) => k === 'detected')
-        if (detectedIdx > -1) {
-          const [detected] = entries.splice(detectedIdx, 1)
-          entries.unshift(detected)
-        }
-        return entries.map(([k, v]) => (
-          <li key={k}>
-            <span className="font-mono">{k}</span>: {String(v)}
-          </li>
-        ))
-      })()}
-    </ul>
-  </fieldset>
-)
-
-const VariantSelector: React.FC<{
-  variantKeys: string[]
-  selectedVariant: string
-  setSelectedVariant: (v: string) => void
-}> = ({ variantKeys, selectedVariant, setSelectedVariant }) => (
-  <fieldset className="fieldset w-full p-4 rounded border border-gray-700 bg-base-200 mb-4">
-    <legend className="fieldset-legend font-semibold">Variants</legend>
-    <div className="mb-2 flex gap-4 flex-wrap">
-      {variantKeys.map((variant, idx) => (
-        <label key={variant} className="flex items-center gap-1 cursor-pointer">
-          <input
-            type="radio"
-            name={`variant-selector`}
-            value={variant}
-            checked={selectedVariant === variant}
-            onChange={() => setSelectedVariant(variant)}
-          />
-          <span className="text-xs font-semibold">
-            {variant} <span className="opacity-60">[{idx + 1}]</span>
-          </span>
-        </label>
-      ))}
-    </div>
-  </fieldset>
-)
-
-const ExampleDetailsSection: React.FC<{ children: React.ReactNode }> = ({ children }) => (
-  <fieldset className="fieldset w-full p-4 rounded border border-gray-700 bg-base-200 mt-6">
-    {children}
-  </fieldset>
-)
-
 const ImageGallery: React.FC<GalleryProps> = ({ selectedModel, selectedAttack, examples }) => {
   const exampleItems = examples[selectedModel][selectedAttack]
 
@@ -90,20 +38,6 @@ const ImageGallery: React.FC<GalleryProps> = ({ selectedModel, selectedAttack, e
   const ZOOM_BOX_SIZE = 300
   const [zoomLevel, setZoomLevel] = React.useState(2) // 2x default zoom
 
-  // Keyboard shortcut for variant switching
-  React.useEffect(() => {
-    const handler = (e: KeyboardEvent) => {
-      if (document.activeElement && (document.activeElement as HTMLElement).tagName === 'INPUT')
-        return
-      const idx = parseInt(e.key, 10)
-      if (!isNaN(idx) && idx > 0 && idx <= variantKeys.length) {
-        setSelectedVariant(variantKeys[idx - 1])
-      }
-    }
-    window.addEventListener('keydown', handler)
-    return () => window.removeEventListener('keydown', handler)
-  }, [variantKeys])
-
   // Calculate the minimum width and height across all variants for the selected image
   const variantImages = variantKeys.map((v) => variants[v]?.image_url).filter(Boolean)
   const [naturalDims, setNaturalDims] = React.useState<{ width: number; height: number } | null>(
@@ -192,14 +126,15 @@ const ImageGallery: React.FC<GalleryProps> = ({ selectedModel, selectedAttack, e
         </div>
         {selectedImage && selectedVariant && variants[selectedVariant] && (
           <>
-            <VariantSelector
+            <ExampleVariantSelector
               variantKeys={variantKeys}
               selectedVariant={selectedVariant}
               setSelectedVariant={setSelectedVariant}
             />
-            <Metadata metadata={variants[selectedVariant].metadata || {}} />
-            {zoomSlider}
+            <ExampleMetadata metadata={variants[selectedVariant].metadata || {}} />
+
             <ExampleDetailsSection>
+              {zoomSlider}
               <div
                 style={{
                   width: '100%',

frontend/src/components/VideoGallery.tsx

@@ -1,5 +1,9 @@
 import React from 'react'
 import type { ExamplesData } from './Examples'
+import { groupByNameAndVariant } from './galleryUtils'
+import ExampleMetadata from './ExampleMetadata'
+import ExampleDetailsSection from './ExampleDetailsSection'
+import ExampleVariantSelector from './ExampleVariantSelector'
 
 interface GalleryProps {
   selectedModel: string
@@ -13,14 +17,71 @@ interface GalleryProps {
 
 const VideoGallery: React.FC<GalleryProps> = ({ selectedModel, selectedAttack, examples }) => {
   const exampleItems = examples[selectedModel][selectedAttack]
+  const grouped = groupByNameAndVariant(exampleItems)
+  const videoNames = Object.keys(grouped)
+  const [selectedVideo, setSelectedVideo] = React.useState(videoNames[0] || '')
+  const variants = grouped[selectedVideo] || {}
+  const variantKeys = Object.keys(variants)
+  const [selectedVariant, setSelectedVariant] = React.useState(variantKeys[0] || '')
+
+  React.useEffect(() => {
+    setSelectedVariant(variantKeys[0] || '')
+  }, [selectedVideo])
+
+  if (!videoNames.length) {
+    return (
+      <div className="w-full mt-12 flex items-center justify-center">
+        <div className="text-gray-500">
+          No video examples available. Please select another model and attack.
+        </div>
+      </div>
+    )
+  }
+
   return (
-    <div className="example-display">
-      {exampleItems.map((item, index) => (
-        <div key={index} className="example-item">
-          <p>{item.name}</p>
-          <video controls src={item.video_url} className="example-video" />
+    <div className="w-full overflow-auto" style={{ minHeight: '100vh' }}>
+      <div className="example-display">
+        <div className="mb-4">
+          <fieldset className="fieldset">
+            <legend className="fieldset-legend">Video Example</legend>
+            <select
+              className="select select-bordered"
+              value={selectedVideo || ''}
+              onChange={(e) => {
+                setSelectedVideo(e.target.value || '')
+              }}
+            >
+              {videoNames.map((name) => (
+                <option key={name} value={name}>
+                  {name}
+                </option>
+              ))}
+            </select>
+          </fieldset>
         </div>
-      ))}
+        {selectedVideo && selectedVariant && variants[selectedVariant] && (
+          <>
+            <ExampleVariantSelector
+              variantKeys={variantKeys}
+              selectedVariant={selectedVariant}
+              setSelectedVariant={setSelectedVariant}
+            />
+            <ExampleMetadata metadata={variants[selectedVariant].metadata || {}} />
+            <ExampleDetailsSection>
+              <div className="flex flex-col items-center gap-4">
+                {variants[selectedVariant].video_url && (
+                  <video
+                    controls
+                    src={variants[selectedVariant].video_url}
+                    className="example-video"
+                    style={{ maxWidth: 400, maxHeight: 300 }}
+                  />
+                )}
+              </div>
+            </ExampleDetailsSection>
+          </>
+        )}
+      </div>
     </div>
   )
 }

frontend/src/components/galleryUtils.ts

@@ -1,5 +1,7 @@
 // Common parsing and grouping utilities for galleries
 
+import { ExamplesData } from './Examples'
+
 export const VARIANT_PREFIX_MAP: { [variant: string]: string } = {
   attacked_wmd: 'attacked_wmd_',
   attacked: 'attacked_',
@@ -13,24 +15,24 @@ export function getVariant(name: string): string {
   )
 }
 
-export function getImageName(name: string, variant: string): string {
-  if (variant !== 'original' && variant in VARIANT_PREFIX_MAP) {
+export function getExampleName(name: string, variant: string): string {
+  if (variant in VARIANT_PREFIX_MAP) {
     return name.replace(VARIANT_PREFIX_MAP[variant], '')
   }
   return name
 }
 
-export function groupByNameAndVariant<T extends { name: string }>(
-  items: T[]
-): {
-  [name: string]: { [variant: string]: T }
+export function groupByNameAndVariant(examples: ExamplesData[]): {
+  [name: string]: { [variant: string]: ExamplesData }
 } {
-  const grouped: { [name: string]: { [variant: string]: T } } = {}
-  items.forEach((item) => {
-    const variant = getVariant(item.name)
-    const imageName = getImageName(item.name, variant)
-    if (!grouped[imageName]) grouped[imageName] = {}
-    grouped[imageName][variant] = item
+  const grouped: { [name: string]: { [variant: string]: ExamplesData } } = {}
+  examples.forEach((example) => {
+    const variant = getVariant(example.name)
+    const imageName = getExampleName(example.name, variant)
+    if (!grouped[imageName]) {
+      grouped[imageName] = {}
+    }
+    grouped[imageName][variant] = example
   })
   return grouped
 }