First prototype of the engine with parrot

app.py

@@ -4,25 +4,30 @@ Virtual MIDI Keyboard - Gradio Application
 This application provides a browser-based MIDI keyboard that can:
 - Play notes with various synthesizer sounds (Tone.js)
 - Record MIDI events with timestamps
+- Process recordings through various engines
 - Export recordings as .mid files
 - Support computer keyboard input
 - Monitor MIDI events in real-time
 
 File Structure:
 - app.py: Gradio server and MIDI conversion (this file)
+- engines.py: MIDI processing engines
 - keyboard.html: Main UI structure
 - static/styles.css: All application styles
 - static/keyboard.js: Client-side logic and interactivity
+- static/engine.js: Client-side engine abstraction
 """
 
 import base64
 import html
 import io
+import json
 
 import mido
 from mido import MidiFile, MidiTrack, Message, MetaMessage
 
 import gradio as gr
+from engines import EngineRegistry, ParrotEngine
 
 
 # =============================================================================
@@ -109,6 +114,94 @@ def save_midi_api(events):
     return {"midi_base64": midi_b64}
 
 
+# =============================================================================
+# ENGINE API ENDPOINTS
+# =============================================================================
+
+
+def list_engines():
+    """
+    API endpoint to list available MIDI engines
+
+    Returns:
+        List of engine info dictionaries
+    """
+    engines = EngineRegistry.list_engines()
+    return {"engines": [EngineRegistry.get_engine_info(e) for e in engines]}
+
+
+def process_with_engine(engine_id: str, events: list):
+    """
+    API endpoint to process MIDI events through an engine
+
+    Args:
+        engine_id: The engine to use (e.g. 'parrot')
+        events: List of MIDI event dictionaries
+
+    Returns:
+        Processed events or error message
+    """
+    if not engine_id or not events:
+        return {"error": "Missing engine_id or events"}
+
+    try:
+        engine = EngineRegistry.get_engine(engine_id)
+        processed = engine.process(events)
+        return {"success": True, "events": processed}
+    except ValueError as e:
+        return {"error": str(e)}
+    except Exception as e:
+        return {"error": f"Processing error: {str(e)}"}
+
+
+def process_engine_api(payload: dict):
+    """
+    Wrapper API endpoint that accepts JSON payload
+
+    Args:
+        payload: Dict with 'engine_id' and 'events' keys
+
+    Returns:
+        Processed events or error message
+    """
+    try:
+        print(f"[DEBUG] process_engine_api called with payload type: {type(payload)}")
+        print(
+            f"[DEBUG] payload keys: {payload.keys() if isinstance(payload, dict) else 'N/A'}"
+        )
+
+        # Handle both direct dict and wrapped dict formats
+        data = payload
+        if isinstance(payload, dict) and "data" in payload:
+            # If wrapped in a data field, unwrap it
+            data = payload["data"]
+            if isinstance(data, list) and len(data) > 0:
+                data = data[0]
+
+        print(
+            f"[DEBUG] extracted data type: {type(data)}, has engine_id: {'engine_id' in data if isinstance(data, dict) else False}"
+        )
+
+        engine_id = data.get("engine_id") if isinstance(data, dict) else None
+        events = data.get("events", []) if isinstance(data, dict) else []
+
+        print(
+            f"[DEBUG] engine_id: {engine_id}, events count: {len(events) if isinstance(events, list) else 'N/A'}"
+        )
+
+        result = process_with_engine(engine_id, events)
+        print(
+            f"[DEBUG] process_engine_api returning: {result.keys() if isinstance(result, dict) else type(result)}"
+        )
+        return result
+    except Exception as e:
+        print(f"[DEBUG] Exception in process_engine_api: {str(e)}")
+        import traceback
+
+        traceback.print_exc()
+        return {"error": f"API error: {str(e)}"}
+
+
 # =============================================================================
 # GRADIO APPLICATION
 # =============================================================================
@@ -138,19 +231,37 @@ iframe_html = (
 )
 
 # Create Gradio interface
-with gr.Blocks() as demo:
+with gr.Blocks(title="Virtual MIDI Keyboard") as demo:
     gr.HTML(iframe_html)
 
-    # Hidden API endpoint components (required for Gradio 6.x)
-    with gr.Row(visible=False):
-        api_input = gr.JSON()
-        api_output = gr.JSON()
-        api_btn = gr.Button("save")
-
-    # Connect API endpoint
-    api_btn.click(
-        fn=save_midi_api, inputs=api_input, outputs=api_output, api_name="save_midi"
-    )
+    # Hidden API endpoints using Gradio's function API
+    with gr.Group(visible=False) as api_group:
+        # Process engine endpoint
+        engine_input = gr.Textbox(label="engine_payload")
+        engine_output = gr.Textbox(label="engine_result")
+
+        def call_engine_api(payload_json):
+            """Wrapper to call engine API with JSON input"""
+            import json
+
+            try:
+                payload = (
+                    json.loads(payload_json)
+                    if isinstance(payload_json, str)
+                    else payload_json
+                )
+                result = process_engine_api(payload)
+                return json.dumps(result)
+            except Exception as e:
+                return json.dumps({"error": str(e)})
+
+        engine_btn = gr.Button("process_engine", visible=False)
+        engine_btn.click(
+            fn=call_engine_api,
+            inputs=engine_input,
+            outputs=engine_output,
+            api_name="process_engine",
+        )
 
 
 # =============================================================================

engines.py

@@ -0,0 +1,180 @@
+"""
+Virtual MIDI Keyboard - Engines
+
+This module contains MIDI processing engines that can transform,
+analyze, or manipulate MIDI events in various ways.
+"""
+
+from abc import ABC, abstractmethod
+from typing import List, Dict, Any, Callable
+import time
+
+
+# =============================================================================
+# BASE ENGINE CLASS
+# =============================================================================
+
+
+class MIDIEngine(ABC):
+    """Abstract base class for MIDI engines"""
+
+    def __init__(self, name: str):
+        self.name = name
+        self.recordings = {}
+        self.current_recording_id = None
+
+    @abstractmethod
+    def process(self, events: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        """
+        Process MIDI events and return transformed events.
+
+        Args:
+            events: List of MIDI event dictionaries
+
+        Returns:
+            List of processed MIDI event dictionaries
+        """
+        pass
+
+    def start_recording(self, recording_name: str = None) -> str:
+        """Start recording a new performance"""
+        if recording_name is None:
+            recording_name = f"recording_{int(time.time() * 1000)}"
+
+        self.current_recording_id = recording_name
+        self.recordings[recording_name] = {
+            "name": recording_name,
+            "events": [],
+            "created_at": time.time(),
+        }
+        return recording_name
+
+    def stop_recording(self) -> Dict[str, Any]:
+        """Stop recording and return the recording"""
+        if not self.current_recording_id:
+            return None
+
+        recording_id = self.current_recording_id
+        self.current_recording_id = None
+        return self.recordings[recording_id]
+
+    def add_event(self, event: Dict[str, Any]) -> bool:
+        """Add event to current recording"""
+        if not self.current_recording_id:
+            return False
+
+        recording = self.recordings[self.current_recording_id]
+        recording["events"].append(event)
+        return True
+
+    def get_recording(self, recording_id: str) -> Dict[str, Any]:
+        """Get a recording by ID"""
+        return self.recordings.get(recording_id)
+
+    def get_all_recordings(self) -> List[Dict[str, Any]]:
+        """Get all recordings"""
+        return list(self.recordings.values())
+
+    def delete_recording(self, recording_id: str) -> bool:
+        """Delete a recording"""
+        if recording_id in self.recordings:
+            if self.current_recording_id == recording_id:
+                self.stop_recording()
+            del self.recordings[recording_id]
+            return True
+        return False
+
+
+# =============================================================================
+# PARROT ENGINE - Returns exactly what was played
+# =============================================================================
+
+
+class ParrotEngine(MIDIEngine):
+    """
+    Parrot Engine - Captures and plays back MIDI exactly as recorded.
+
+    This is the simplest engine - it just repeats what the user played.
+    Perfect for learning and basic playback.
+    """
+
+    def __init__(self):
+        super().__init__("Parrot")
+        self.description = "Captures and plays back MIDI exactly as recorded"
+
+    def process(self, events: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        """
+        Process MIDI events - in Parrot mode, just return them unchanged.
+
+        Args:
+            events: List of MIDI event dictionaries
+
+        Returns:
+            The same events unmodified
+        """
+        if not events:
+            return []
+
+        # Simply return the events as-is
+        processed_events = []
+        for event in events:
+            processed_events.append(
+                {
+                    "type": event.get("type"),
+                    "note": event.get("note"),
+                    "velocity": event.get("velocity"),
+                    "time": event.get("time"),
+                    "channel": event.get("channel", 0),
+                }
+            )
+
+        return processed_events
+
+
+# =============================================================================
+# ENGINE REGISTRY
+# =============================================================================
+
+
+class EngineRegistry:
+    """Registry for managing available MIDI engines"""
+
+    _engines = {"parrot": ParrotEngine}
+
+    @classmethod
+    def register(cls, engine_id: str, engine_class: type):
+        """Register a new engine"""
+        cls._engines[engine_id] = engine_class
+
+    @classmethod
+    def get_engine(cls, engine_id: str) -> MIDIEngine:
+        """Get an engine instance by ID"""
+        if engine_id not in cls._engines:
+            raise ValueError(f"Unknown engine: {engine_id}")
+        return cls._engines[engine_id]()
+
+    @classmethod
+    def list_engines(cls) -> List[str]:
+        """List all available engines"""
+        return list(cls._engines.keys())
+
+    @classmethod
+    def get_engine_info(cls, engine_id: str) -> Dict[str, str]:
+        """Get info about an engine"""
+        if engine_id not in cls._engines:
+            raise ValueError(f"Unknown engine: {engine_id}")
+
+        engine = cls._engines[engine_id]()
+        return {
+            "id": engine_id,
+            "name": engine.name,
+            "description": getattr(engine, "description", "No description"),
+        }
+
+
+# =============================================================================
+# GLOBAL ENGINE INSTANCES
+# =============================================================================
+
+# Create default engine instance
+default_engine = ParrotEngine()

keyboard.html

@@ -31,8 +31,16 @@
           </select>
         </label>
         
+        <label>
+          Engine:
+          <select id="engineSelect">
+            <option value="parrot">Parrot</option>
+          </select>
+        </label>
+        
         <button id="recordBtn">Record</button>
         <button id="stopBtn" disabled>Stop</button>
+        <button id="playbackBtn" disabled>Playback</button>
         <button id="saveBtn" disabled>Save MIDI</button>
         
         <label>
@@ -57,6 +65,9 @@
   <!-- External Dependencies -->
   <script src="https://unpkg.com/tone@next/build/Tone.js"></script>
   
+  <!-- Engine -->
+  <script src="/file=static/engine.js"></script>
+  
   <!-- Application Logic -->
   <script src="/file=static/keyboard.js"></script>
 </body>

static/engine.js

@@ -0,0 +1,242 @@
+/**
+ * Virtual MIDI Keyboard - Engine
+ * 
+ * The engine processes MIDI signals and can:
+ * - Capture and store MIDI events
+ * - Play back captured sequences
+ * - Process and transform MIDI data
+ * 
+ * This is the core processing unit for all signal manipulation.
+ */
+
+// =============================================================================
+// ENGINE STATE
+// =============================================================================
+
+class MIDIEngine {
+  constructor() {
+    this.recordings = {}; // Store multiple recordings
+    this.currentRecordingId = null;
+    this.isPlayingBack = false;
+    this.playbackSpeed = 1.0; // 1.0 = normal speed
+    this.callbacks = {
+      onNoteOn: null,
+      onNoteOff: null,
+      onPlaybackStart: null,
+      onPlaybackEnd: null,
+      onProgress: null
+    };
+  }
+
+  /**
+   * Start recording a new performance
+   * Returns the recording ID
+   */
+  startRecording(recordingName = 'Recording_' + Date.now()) {
+    this.currentRecordingId = recordingName;
+    this.recordings[recordingName] = {
+      name: recordingName,
+      events: [],
+      startTime: performance.now(),
+      duration: 0,
+      metadata: {
+        createdAt: new Date().toISOString(),
+        instrument: 'default'
+      }
+    };
+    return recordingName;
+  }
+
+  /**
+   * Stop recording and return the recording
+   */
+  stopRecording() {
+    if (!this.currentRecordingId) return null;
+
+    const recording = this.recordings[this.currentRecordingId];
+    if (recording && recording.events.length > 0) {
+      const lastEvent = recording.events[recording.events.length - 1];
+      recording.duration = lastEvent.time;
+    }
+
+    const recordingId = this.currentRecordingId;
+    this.currentRecordingId = null;
+    return recording;
+  }
+
+  /**
+   * Add a MIDI event to the current recording
+   */
+  addEvent(event) {
+    if (!this.currentRecordingId) return false;
+
+    const recording = this.recordings[this.currentRecordingId];
+    if (!recording) return false;
+
+    recording.events.push({
+      ...event,
+      id: recording.events.length
+    });
+
+    return true;
+  }
+
+  /**
+   * Get a recording by ID
+   */
+  getRecording(recordingId) {
+    return this.recordings[recordingId];
+  }
+
+  /**
+   * Get all recordings
+   */
+  getAllRecordings() {
+    return Object.values(this.recordings);
+  }
+
+  /**
+   * Delete a recording
+   */
+  deleteRecording(recordingId) {
+    if (this.currentRecordingId === recordingId) {
+      this.stopRecording();
+    }
+    delete this.recordings[recordingId];
+  }
+
+  /**
+   * Play back a recording
+   * Calls the provided callbacks for each note
+   */
+  async playback(recordingId, callbacks = {}) {
+    const recording = this.getRecording(recordingId);
+    if (!recording || recording.events.length === 0) {
+      console.warn('No recording found or recording is empty');
+      return;
+    }
+
+    // Merge provided callbacks with instance callbacks
+    const finalCallbacks = { ...this.callbacks, ...callbacks };
+
+    this.isPlayingBack = true;
+    if (finalCallbacks.onPlaybackStart) {
+      finalCallbacks.onPlaybackStart(recording);
+    }
+
+    const events = recording.events;
+    let playbackStartTime = performance.now();
+
+    for (let i = 0; i < events.length; i++) {
+      if (!this.isPlayingBack) break; // Stop if playback was cancelled
+
+      const event = events[i];
+      const scheduledTime = (event.time / this.playbackSpeed) * 1000; // ms
+      const now = performance.now() - playbackStartTime;
+      const waitTime = scheduledTime - now;
+
+      if (waitTime > 0) {
+        await this.sleep(waitTime);
+      }
+
+      // Execute the event callback
+      if (event.type === 'note_on' && finalCallbacks.onNoteOn) {
+        finalCallbacks.onNoteOn(event);
+      } else if (event.type === 'note_off' && finalCallbacks.onNoteOff) {
+        finalCallbacks.onNoteOff(event);
+      }
+
+      // Progress callback
+      if (finalCallbacks.onProgress) {
+        finalCallbacks.onProgress({
+          currentIndex: i,
+          totalEvents: events.length,
+          currentTime: event.time,
+          totalDuration: recording.duration,
+          progress: (i / events.length) * 100
+        });
+      }
+    }
+
+    this.isPlayingBack = false;
+    if (finalCallbacks.onPlaybackEnd) {
+      finalCallbacks.onPlaybackEnd(recording);
+    }
+  }
+
+  /**
+   * Stop playback
+   */
+  stopPlayback() {
+    this.isPlayingBack = false;
+  }
+
+  /**
+   * Set playback speed (1.0 = normal, 2.0 = double speed, etc.)
+   */
+  setPlaybackSpeed(speed) {
+    this.playbackSpeed = Math.max(0.1, Math.min(2.0, speed));
+  }
+
+  /**
+   * Export recording as JSON
+   */
+  exportAsJSON(recordingId) {
+    const recording = this.getRecording(recordingId);
+    if (!recording) return null;
+    return JSON.stringify(recording, null, 2);
+  }
+
+  /**
+   * Import recording from JSON
+   */
+  importFromJSON(jsonString, recordingName = null) {
+    try {
+      const recording = JSON.parse(jsonString);
+      const id = recordingName || recording.name || 'imported_' + Date.now();
+      this.recordings[id] = recording;
+      return id;
+    } catch (error) {
+      console.error('Failed to import recording:', error);
+      return null;
+    }
+  }
+
+  /**
+   * Get recording statistics
+   */
+  getStats(recordingId) {
+    const recording = this.getRecording(recordingId);
+    if (!recording) return null;
+
+    const events = recording.events;
+    const noteOnEvents = events.filter(e => e.type === 'note_on');
+    const noteOffEvents = events.filter(e => e.type === 'note_off');
+
+    return {
+      recordingName: recording.name,
+      totalEvents: events.length,
+      noteOnCount: noteOnEvents.length,
+      noteOffCount: noteOffEvents.length,
+      duration: recording.duration,
+      avgNotesPerSecond: (noteOnEvents.length / recording.duration).toFixed(2),
+      minNote: Math.min(...noteOnEvents.map(e => e.note)),
+      maxNote: Math.max(...noteOnEvents.map(e => e.note)),
+      instrument: recording.metadata?.instrument || 'unknown'
+    };
+  }
+
+  /**
+   * Helper: Sleep for ms milliseconds
+   */
+  sleep(ms) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+  }
+}
+
+// =============================================================================
+// EXPORT
+// =============================================================================
+
+// Create a global engine instance
+const midiEngine = new MIDIEngine();

static/keyboard.js

@@ -63,9 +63,11 @@ const keyboardEl = document.getElementById('keyboard');
 const statusEl = document.getElementById('status');
 const recordBtn = document.getElementById('recordBtn');
 const stopBtn = document.getElementById('stopBtn');
+const playbackBtn = document.getElementById('playbackBtn');
 const saveBtn = document.getElementById('saveBtn');
 const keyboardToggle = document.getElementById('keyboardToggle');
 const instrumentSelect = document.getElementById('instrumentSelect');
+const engineSelect = document.getElementById('engineSelect');
 const terminal = document.getElementById('terminal');
 const clearTerminal = document.getElementById('clearTerminal');
 
@@ -78,6 +80,7 @@ let recording = false;
 let startTime = 0;
 let events = [];
 const pressedKeys = new Set();
+let selectedEngine = 'parrot'; // Default engine
 
 // =============================================================================
 // INSTRUMENT CONFIGURATIONS
@@ -206,7 +209,12 @@ function beginRecord() {
   statusEl.textContent = 'Recording...';
   recordBtn.disabled = true;
   stopBtn.disabled = false;
+  playbackBtn.disabled = true;
   saveBtn.disabled = true;
+  
+  // Start engine recording
+  const recordingId = midiEngine.startRecording('keyboard_' + Date.now());
+  
   logToTerminal('', '');
   logToTerminal('▶▶▶ RECORDING STARTED ◀◀◀', 'timestamp');
   logToTerminal('', '');
@@ -218,6 +226,11 @@ function stopRecord() {
   recordBtn.disabled = false;
   stopBtn.disabled = true;
   saveBtn.disabled = events.length === 0;
+  playbackBtn.disabled = events.length === 0;
+  
+  // Stop engine recording
+  midiEngine.stopRecording();
+  
   logToTerminal('', '');
   logToTerminal(`■■■ RECORDING STOPPED (${events.length} events captured) ■■■`, 'timestamp');
   logToTerminal('', '');
@@ -239,13 +252,16 @@ function noteOn(midiNote, velocity = 100) {
   );
   
   if (recording) {
-    events.push({
+    const event = {
       type: 'note_on',
       note: midiNote,
       velocity: Math.max(1, velocity | 0),
       time: nowSec() - startTime,
       channel: 0
-    });
+    };
+    events.push(event);
+    // Also add to engine
+    midiEngine.addEvent(event);
   }
 }
 
@@ -261,13 +277,16 @@ function noteOff(midiNote) {
   );
   
   if (recording) {
-    events.push({
+    const event = {
       type: 'note_off',
       note: midiNote,
       velocity: 0,
       time: nowSec() - startTime,
       channel: 0
-    });
+    };
+    events.push(event);
+    // Also add to engine
+    midiEngine.addEvent(event);
   }
 }
 
@@ -460,6 +479,93 @@ recordBtn.addEventListener('click', async () => {
 
 stopBtn.addEventListener('click', () => stopRecord());
 
+engineSelect.addEventListener('change', (e) => {
+  selectedEngine = e.target.value;
+  logToTerminal(`Engine switched to: ${selectedEngine}`, 'timestamp');
+});
+
+playbackBtn.addEventListener('click', async () => {
+  if (events.length === 0) return alert('No recording to play back');
+  
+  statusEl.textContent = 'Playing back...';
+  playbackBtn.disabled = true;
+  recordBtn.disabled = true;
+  
+  logToTerminal('', '');
+  logToTerminal('♫♫♫ PLAYBACK STARTED ♫♫♫', 'timestamp');
+  logToTerminal('', '');
+  
+  try {
+    // For now, skip engine processing and directly play recorded events
+    // TODO: Call engine API when Gradio routing is figured out
+    const processedEvents = events;
+    
+    console.log(`Playing back ${processedEvents.length} events`);
+    
+    // Play back the recorded events
+    statusEl.textContent = 'Playing back...';
+    let eventIndex = 0;
+    
+    const playEvent = () => {
+      if (eventIndex >= processedEvents.length) {
+        // Playback complete
+        statusEl.textContent = 'Playback complete';
+        playbackBtn.disabled = false;
+        recordBtn.disabled = false;
+        
+        logToTerminal('', '');
+        logToTerminal('♫♫♫ PLAYBACK FINISHED ♫♫♫', 'timestamp');
+        logToTerminal('', '');
+        return;
+      }
+      
+      const event = processedEvents[eventIndex];
+      const nextTime = eventIndex + 1 < processedEvents.length
+        ? processedEvents[eventIndex + 1].time
+        : event.time;
+      
+      if (event.type === 'note_on') {
+        const freq = Tone.Frequency(event.note, "midi").toFrequency();
+        synth.triggerAttack(freq, undefined, event.velocity / 127);
+        
+        const noteName = midiToNoteName(event.note);
+        logToTerminal(
+          `[${event.time.toFixed(3)}s] ► ${noteName} (${event.note})`,
+          'note-on'
+        );
+        
+        // Highlight the key being played
+        const keyEl = getKeyElement(event.note);
+        if (keyEl) keyEl.style.filter = 'brightness(0.7)';
+      } else if (event.type === 'note_off') {
+        const freq = Tone.Frequency(event.note, "midi").toFrequency();
+        synth.triggerRelease(freq);
+        
+        const noteName = midiToNoteName(event.note);
+        logToTerminal(
+          `[${event.time.toFixed(3)}s] ◄ ${noteName}`,
+          'note-off'
+        );
+        
+        // Remove key highlight
+        const keyEl = getKeyElement(event.note);
+        if (keyEl) keyEl.style.filter = '';
+      }
+      
+      eventIndex++;
+      const deltaTime = Math.max(0, nextTime - event.time);
+      setTimeout(playEvent, deltaTime * 1000);
+    };
+    
+    playEvent();
+  } catch (err) {
+    console.error('Playback error:', err);
+    statusEl.textContent = 'Playback error: ' + err.message;
+    playbackBtn.disabled = false;
+    recordBtn.disabled = false;
+  }
+});
+
 saveBtn.addEventListener('click', () => saveMIDI());
 
 // =============================================================================
@@ -476,6 +582,7 @@ function init() {
   recordBtn.disabled = false;
   stopBtn.disabled = true;
   saveBtn.disabled = true;
+  playbackBtn.disabled = true;
 }
 
 // Start the application