full pipeline

portiloop/capture.py

@@ -12,6 +12,7 @@ import multiprocessing as mp
 import warnings
 import shutil
 from threading import Thread, Lock
+import alsaaudio
 
 import matplotlib.pyplot as plt
 from EDFlib.edfwriter import EDFwriter
@@ -198,7 +199,7 @@ class FilterPipeline:
                  sampling_rate,
                  power_line_fq=60,
                  use_custom_fir=False,
-                 custom_fir_order=10,
+                 custom_fir_order=20,
                  custom_fir_cutoff=30,
                  alpha_avg=0.1,
                  alpha_std=0.001,
@@ -411,7 +412,7 @@ class Capture:
         self.polyak_std = 0.001
         self.epsilon = 0.000001
         self.custom_fir = False
-        self.custom_fir_order = 10
+        self.custom_fir_order = 20
         self.custom_fir_cutoff = 30
         self.filter = True
         self.record = False
@@ -436,6 +437,19 @@ class Capture:
         self.detector_cls = detector_cls
         self.stimulator_cls = stimulator_cls
         
+        self._test_stimulus_lock = Lock()
+        self._test_stimulus = False
+        
+        mixers = alsaaudio.mixers()
+        if 'PCM' in mixers:
+            self.mixer = alsaaudio.Mixer(control='PCM')
+        else:
+            assert len(mixers) > 0, 'No ALSA mixer found'
+            warnings.warn(f"Could not find mixer PCM, using {mixers[0]} instead.")
+            self.mixer = alsaaudio.Mixer(control=mixers[0])
+        self.volume = self.mixer.getvolume()[0]  # we will set the same volume on all channels
+        
+        
         # widgets ===============================
         
         # CHANNELS ------------------------------
@@ -657,6 +671,22 @@ class Capture:
             indent=False
         )
         
+        self.b_volume = widgets.IntSlider(
+            value=self.volume, 
+            min=0,
+            max=100,
+            step=1,
+            description="Volume",
+            disabled=False
+        )
+        
+        self.b_test_stimulus = widgets.Button(
+            description='Test stimulus',
+            disabled=True,
+            button_style='', # 'success', 'info', 'warning', 'danger' or ''
+            tooltip='Send a test stimulus'
+        )
+        
         # CALLBACKS ----------------------
         
         self.b_capture.observe(self.on_b_capture, 'value')
@@ -684,6 +714,8 @@ class Capture:
         self.b_polyak_mean.observe(self.on_b_polyak_mean, 'value')
         self.b_polyak_std.observe(self.on_b_polyak_std, 'value')
         self.b_epsilon.observe(self.on_b_epsilon, 'value')
+        self.b_volume.observe(self.on_b_volume, 'value')
+        self.b_test_stimulus.on_click(self.on_b_test_stimulus)
         
         self.display_buttons()
 
@@ -698,7 +730,8 @@ class Capture:
                               self.b_power_line,
                               self.b_clock,
                               widgets.HBox([self.b_filter, self.b_detect, self.b_stimulate, self.b_record, self.b_lsl, self.b_display]),
-                              self.b_threshold,
+                              widgets.HBox([self.b_threshold, self.b_test_stimulus]),
+                              self.b_volume,
                               self.b_accordion_filter,
                               self.b_capture]))
 
@@ -727,6 +760,7 @@ class Capture:
         self.b_custom_fir_cutoff.disabled = not self.custom_fir
         self.b_stimulate.disabled = not self.detect
         self.b_threshold.disabled = not self.detect
+        self.b_test_stimulus.disabled = True # only enabled when running
     
     def disable_buttons(self):
         self.b_frequency.disabled = True
@@ -754,6 +788,7 @@ class Capture:
         self.b_custom_fir_order.disabled = True
         self.b_custom_fir_cutoff.disabled = True
         self.b_threshold.disabled = True
+        self.b_test_stimulus.disabled = not self.stimulate # only enabled when running
     
     def on_b_radio_ch2(self, value):
         self.channel_states[1] = value['new']
@@ -789,6 +824,7 @@ class Capture:
                 return
             detector_cls = self.detector_cls if self.detect else None
             stimulator_cls = self.stimulator_cls if self.stimulate else None
+            
             self._t_capture = Thread(target=self.start_capture,
                                 args=(self.filter,
                                       detector_cls,
@@ -918,6 +954,16 @@ class Capture:
     def on_b_display(self, value):
         val = value['new']
         self.display = val
+        
+    def on_b_volume(self, value):
+        val = value['new']
+        if val >= 0 and val <= 100:
+            self.volume = val
+            self.mixer.setvolume(self.volume)
+    
+    def on_b_test_stimulus(self, b):
+        with self._test_stimulus_lock:
+            self._test_stimulus = True
     
     def open_recording_file(self):
         nb_signals = self.nb_signals
@@ -1011,8 +1057,9 @@ class Capture:
             lsl_info = StreamInfo(name='Portiloop',
                                   type='EEG',
                                   channel_count=8,
+                                  nominal_srate=self.frequency,
                                   channel_format='float32',
-                                  source_id='')  # TODO: replace this by unique device identifier
+                                  source_id='portiloop1')  # TODO: replace this by unique device identifier
             lsl_outlet = StreamOutlet(lsl_info)
 
         buffer = []
@@ -1046,9 +1093,13 @@ class Capture:
             
             if detector is not None:
                 detection_signal = detector.detect(filtered_point)
-                
                 if stimulator is not None:
                     stimulator.stimulate(detection_signal)
+                    with self._test_stimulus_lock:
+                        test_stimulus = self._test_stimulus
+                        self._test_stimulus = False
+                    if test_stimulus:
+                        stimulator.test_stimulus()
             
             if lsl:
                 lsl_outlet.push_sample(filtered_point[-1])

portiloop/detection.py

@@ -82,51 +82,68 @@ class SleepSpindleRealTimeDetector(Detector):
         return res
 
     def add_datapoint(self, input_float):
+        '''
+        Add one datapoint to the buffer
+        '''
         input_float = input_float[self.channel - 1]
         result = None
+        # Add to current buffer
         self.buffer.append(input_float)
         if len(self.buffer) > self.window_size:
+            # Remove the end of the buffer
             self.buffer = self.buffer[1:]
             self.current_stride_counter += 1
             if self.current_stride_counter == self.stride_counters[self.interpreter_counter]:
-                result = self.call_model(self.interpreter_counter, self.buffer)
+                # If we have reached the next window size, we send the current buffer to the inference function and update the hidden state
+                result, self.h[self.interpreter_counter] = self.forward_tflite(self.interpreter_counter, self.buffer, self.h[self.interpreter_counter])
                 self.interpreter_counter += 1
                 self.interpreter_counter %= self.num_models_parallel
                 self.current_stride_counter = 0
         return result
-
-    def call_model(self, idx, input_float=None):
-        if input_float is None:
-            # For debugging purposes
-            input_shape = self.input_details[0]['shape']
-            input = np.array(np.random.random_sample(input_shape), dtype=np.int8)
-        else:
-            # Convert float input to Int
-            input_scale, input_zero_point = self.input_details[0]["quantization"]
-            input = np.asarray(input_float) / input_scale + input_zero_point
-            input = input.astype(self.input_details[0]["dtype"])
-        input = input.reshape((1, 1, -1))
-        
-        # TODO: Milo please implement this:
-#         self.interpreters[idx].set_tensor(self.input_details[0]['index'], (self.h[idx], input))
         
-#         if self.verbose:
-#             start_time = time.time()
+    def forward_tflite(self, idx, input_x, input_h):
+        input_details = self.interpreters[idx].get_input_details()
+        output_details = self.interpreters[idx].get_output_details()
         
-#         self.interpreters[idx].invoke()
+        # convert input to int 
+        input_scale, input_zero_point = input_details[1]["quantization"]
+        input_x = np.asarray(input_x) / input_scale + input_zero_point
+        input_data_x = input_x.astype(input_details[1]["dtype"])
+        input_data_x = np.expand_dims(input_data_x, (0, 1))
+
+        # input_scale, input_zero_point = input_details[0]["quantization"]
+        # input = np.asarray(input) / input_scale + input_zero_point
+
+        # Test the model on random input data.
+        input_shape_h = input_details[0]['shape']
+        input_shape_x = input_details[1]['shape']
+
+        # input_data_h = np.array(np.random.random_sample(input_shape_h), dtype=np.int8)
+        # input_data_x = np.array(np.random.random_sample(input_shape_x), dtype=np.int8)
+        self.interpreters[idx].set_tensor(input_details[0]['index'], input_h)
+        self.interpreters[idx].set_tensor(input_details[1]['index'], input_data_x)
         
-#         if self.verbose:
-#             end_time = time.time()
-#         output, self.h[idx] = self.interpreters[idx].get_tensor(self.output_details[0]['index'])
-#         output_scale, output_zero_point = self.input_details[0]["quantization"]
-#         output = float(output - output_zero_point) * output_scale
+        if self.verbose:
+            start_time = time.time()
+
+        self.interpreters[idx].invoke()
         
-        # TODO: remove this line:
-        output = np.random.uniform()  # FIXME: remove
+        if self.verbose:
+            end_time = time.time()
+
+        # The function `get_tensor()` returns a copy of the tensor data.
+        # Use `tensor()` in order to get a pointer to the tensor.
+        output_data_h = self.interpreters[idx].get_tensor(output_details[0]['index'])
+        output_data_y = self.interpreters[idx].get_tensor(output_details[1]['index'])
 
+        output_scale, output_zero_point = output_details[1]["quantization"]
+        output_data_y = float(output_data_y - output_zero_point) * output_scale
+        
         if self.verbose:
-            print(f"Computed output {output} in {end_time - start_time} seconds")
+            print(f"Computed output {output_data_y} in {end_time - start_time} seconds")
 
-        return output
+        return output_data_y, output_data_h
+        
+        
     
     

portiloop/notebooks/tests.ipynb

@@ -2,24 +2,61 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 1,
    "id": "7b2fc5da",
    "metadata": {
     "scrolled": false
    },
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "5bd498c14c0b47ef8fc0c7b25d6197c0",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "VBox(children=(Accordion(children=(GridBox(children=(Label(value='CH1'), Label(value='CH2'), Label(value='CH3'…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "DEBUG:/home/mendel/software/portiloop-software/portiloop/sounds/stimulus.wav\n",
+      "PID capture: 4311\n",
+      "DEBUG: new config[5]:0xe1\n",
+      "DEBUG: new config[6]:0xe1\n",
+      "DEBUG: new config[7]:0xe1\n",
+      "DEBUG: new config[8]:0xe1\n",
+      "DEBUG: new config[9]:0xe1\n",
+      "DEBUG: new config[10]:0xe1\n",
+      "DEBUG: new config[11]:0xe1\n",
+      "DEBUG: new config[12]:0xe1\n",
+      "DEBUG: new config[13]:0x0\n",
+      "DEBUG: new config[14]:0x0\n",
+      "DEBUG: new config[3]:0xe8\n"
+     ]
+    }
+   ],
    "source": [
     "from portiloop.capture import Capture\n",
     "from portiloop.detection import SleepSpindleRealTimeDetector\n",
     "from portiloop.stimulation import SleepSpindleRealTimeStimulator\n",
     "\n",
-    "cap = Capture(detector_cls=SleepSpindleRealTimeDetector, stimulator_cls=SleepSpindleRealTimeStimulator)"
+    "my_detector_class = SleepSpindleRealTimeDetector  # you may want to implement yours\n",
+    "my_stimulator_class = SleepSpindleRealTimeStimulator  # you may also want to implement yours\n",
+    "\n",
+    "cap = Capture(detector_cls=my_detector_class, stimulator_cls=my_stimulator_class)"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "9bd24d7a",
+   "id": "fd7c79a7",
    "metadata": {},
    "outputs": [],
    "source": []

portiloop/stimulation.py

@@ -1,8 +1,10 @@
 from abc import ABC, abstractmethod
 import time
-from playsound import playsound
 from threading import Thread, Lock
 from pathlib import Path
+import alsaaudio
+import wave
+import pylsl
 
 
 # Abstract interface for developers:
@@ -18,6 +20,12 @@ class Stimulator(ABC):
             detection_signal: Object: the output of the Detector.add_datapoints method.
         """
         raise NotImplementedError
+    
+    def test_stimulus(self):
+        """
+        Optional: this is called when the 'Test stimulus' button is pressed.
+        """
+        pass
 
         
 # Example implementation for sleep spindles
@@ -30,6 +38,52 @@ class SleepSpindleRealTimeStimulator(Stimulator):
         self._lock = Lock()
         self.last_detected_ts = time.time()
         self.wait_t = 0.4  # 400 ms
+        
+        lsl_markers_info = pylsl.StreamInfo(name='Portiloop_stimuli',
+                                  type='Markers',
+                                  channel_count=1,
+                                  channel_format='string',
+                                  source_id='portiloop1')  # TODO: replace this by unique device identifier
+        self.lsl_outlet_markers = pylsl.StreamOutlet(lsl_markers_info)
+        
+        # Initialize Alsa stuff
+        # Open WAV file and set PCM device
+        with wave.open(str(self._sound), 'rb') as f: 
+            device = 'default'
+
+            format = None
+
+            # 8bit is unsigned in wav files
+            if f.getsampwidth() == 1:
+                format = alsaaudio.PCM_FORMAT_U8
+            # Otherwise we assume signed data, little endian
+            elif f.getsampwidth() == 2:
+                format = alsaaudio.PCM_FORMAT_S16_LE
+            elif f.getsampwidth() == 3:
+                format = alsaaudio.PCM_FORMAT_S24_3LE
+            elif f.getsampwidth() == 4:
+                format = alsaaudio.PCM_FORMAT_S32_LE
+            else:
+                raise ValueError('Unsupported format')
+
+            self.periodsize = f.getframerate() // 8
+
+            self.pcm = alsaaudio.PCM(channels=f.getnchannels(), rate=f.getframerate(), format=format, periodsize=self.periodsize, device=device)
+            
+            # Store data in list to avoid reopening the file
+            data = f.readframes(self.periodsize)
+            self.wav_list = [data]
+            while data:
+                self.wav_list.append(data)
+                data = f.readframes(self.periodsize)
+            
+       
+    def play_sound(self):
+        '''
+        Open the wav file and play a sound
+        '''
+        for data in self.wav_list:
+            self.pcm.write(data) 
     
     def stimulate(self, detection_signal):
         for sig in detection_signal:
@@ -43,6 +97,13 @@ class SleepSpindleRealTimeStimulator(Stimulator):
                 self.last_detected_ts = ts
                 
     def _t_sound(self):
-        playsound(self._sound)
+        self.lsl_outlet_markers.push_sample(['STIM'])
+        self.play_sound()
         with self._lock:
             self._thread = None
+    
+    def test_stimulus(self):
+        with self._lock:
+            if self._thread is None:
+                self._thread = Thread(target=self._t_sound, daemon=True)
+                self._thread.start()