Add : Adding the french voice feature

app.py

@@ -1,16 +1,17 @@
 from flask import Flask, render_template,request, redirect,url_for, jsonify , session
 from helper_functions import predict_class , inference , predict , align_predictions_with_sentences , load_models , load_fr_models
-from helper_functions import predict_fr_class, fr_inference , align_fr_predictions_with_sentences 
+from helper_functions import predict_fr_class, fr_inference , align_fr_predictions_with_sentences , transcribe_speech
 import fitz  # PyMuPDF
 import os, shutil
 import torch
 import tempfile
 from pydub import AudioSegment
 import logging
+import torchaudio
  
 app = Flask(__name__)
 app.config['UPLOAD_FOLDER'] = 'static/uploads' 
-
+device = "cpu"
 # Global variables for models
 global_model = None
 global_neptune = None
@@ -18,17 +19,20 @@ global_pipe = None
 global_fr_model = None
 global_fr_neptune = None
 global_fr_pipe = None
-
+global_fr_wav2vec2_processor = None
+global_fr_wav2vec2_model = None
 
   
 def init_app():
     global global_model, global_neptune, global_pipe
+    global global_fr_model, global_fr_neptune, global_fr_wav2vec2_processor, global_fr_wav2vec2_model
+    
     print("Loading English models...")
     global_model, global_neptune, global_pipe = load_models()
     
-    global global_fr_model, global_fr_neptune, global_fr_pipe
     print("Loading French models...")
-    global_fr_model , global_fr_neptune , global_fr_pipe = load_fr_models()
+    global_fr_model, global_fr_neptune, global_fr_wav2vec2_processor, global_fr_wav2vec2_model = load_fr_models()
+    
     print("Models loaded successfully!")
 
 init_app()
@@ -305,6 +309,7 @@ def treatment_fr():
         }
         print(predict_class)
         print(chart_data)
+        print(sentences)
          # clear the uploads folder
         for filename in os.listdir(app.config['UPLOAD_FOLDER']):
             file_path = os.path.join(app.config['UPLOAD_FOLDER'], filename)
@@ -341,31 +346,44 @@ def sentence_fr():
     # Render the initial form page
     return render_template('sentence_fr.html')
 
+from pydub import AudioSegment
+import io
+
 @app.route("/voice_fr", methods=['GET', 'POST'])
 def slu_fr():
-    global global_fr_neptune, global_fr_pipe, global_fr_model
+    global global_fr_neptune, global_fr_model, global_fr_wav2vec2_processor, global_fr_wav2vec2_model
 
     if request.method == 'POST':
-        logging.debug("Received POST request")
+        logging.info("Received POST request for /voice_fr")
         audio_file = request.files.get('audio')
 
         if audio_file:
-            logging.debug(f"Received audio file: {audio_file.filename}")
+            logging.info(f"Received audio file: {audio_file.filename}")
             
-            # Save audio data to a temporary file
-            with tempfile.NamedTemporaryFile(delete=False, suffix='.wav') as temp_audio:
-                audio_file.save(temp_audio)
-                temp_audio_path = temp_audio.name
-
-            logging.debug(f"Saved audio to temporary file: {temp_audio_path}")
+            # Lire le contenu du fichier audio
+            audio_data = audio_file.read()
+            
+            # Convertir l'audio en WAV si nécessaire
+            try:
+                audio = AudioSegment.from_file(io.BytesIO(audio_data))
+                audio = audio.set_frame_rate(16000).set_channels(1)
+                
+                # Sauvegarder l'audio converti dans un fichier temporaire
+                with tempfile.NamedTemporaryFile(delete=False, suffix='.wav') as temp_audio:
+                    audio.export(temp_audio.name, format="wav")
+                    temp_audio_path = temp_audio.name
+
+                logging.info(f"Converted and saved audio to temporary file: {temp_audio_path}")
+            except Exception as e:
+                logging.error(f"Error converting audio: {str(e)}")
+                return jsonify({'error': 'Unable to process audio file'}), 400
 
             try:
-                # Transcribe audio using Whisper
-                result = global_fr_pipe(temp_audio_path)
-                extracted_text = result["text"]
-                logging.debug(f"Transcribed text: {extracted_text}")
+                # Transcrire l'audio en utilisant la fonction de helper_functions
+                extracted_text = transcribe_speech(temp_audio_path, global_fr_wav2vec2_processor, global_fr_wav2vec2_model)
+                logging.info(f"Transcribed text: {extracted_text}")
 
-                # Process the transcribed text
+                # Traiter le texte transcrit
                 inference_batch, sentences = fr_inference(extracted_text)
                 predictions = predict(inference_batch, global_fr_neptune)
                 sentences_prediction = align_fr_predictions_with_sentences(sentences, predictions)
@@ -382,17 +400,17 @@ def slu_fr():
 
                 response_data = {
                     'extracted_text': extracted_text,
-                    'class_probabilities' : class_probabilities,
+                    'class_probabilities': class_probabilities,
                     'predicted_class': predicted_class,
                     'chart_data': chart_data,
                     'sentences_prediction': sentences_prediction
                 }
-                logging.debug(f"Prepared response data: {response_data}")
+                logging.info(f"Prepared response data: {response_data}")
 
                 return render_template('voice_fr.html', 
-                           class_probabilities= class_probabilities, 
-                           predicted_class= predicted_class, 
-                           chart_data= chart_data, 
+                           class_probabilities=class_probabilities, 
+                           predicted_class=predicted_class, 
+                           chart_data=chart_data, 
                            sentences_prediction=sentences_prediction)
 
             except Exception as e:
@@ -400,15 +418,15 @@ def slu_fr():
                 return jsonify({'error': str(e)}), 500
 
             finally:
-                # Remove temporary file
+                # Supprimer le fichier temporaire
                 os.unlink(temp_audio_path)
 
         else:
             logging.error("No audio file received")
             return jsonify({'error': 'No audio file received'}), 400
 
-    # For GET request
-    logging.debug("Received GET request")
+    # Pour la requête GET
+    logging.info("Received GET request for /voice_fr")
     return render_template('voice_fr.html', 
                            class_probabilities={}, 
                            predicted_class=[""], 

helper_functions.py

@@ -11,6 +11,10 @@ from FrModel import FR_BERT
 from Model import tokenizer , mult_token_id , cls_token_id , pad_token_id , max_pred , maxlen , sep_token_id 
 from FrModel import fr_tokenizer , fr_mult_token_id , fr_cls_token_id , fr_pad_token_id , fr_sep_token_id
 from transformers import pipeline
+from transformers import AutoModelForCTC, Wav2Vec2Processor
+import torchaudio
+import logging
+import soundfile as sf
 
 device = "cpu"
 # Load the model
@@ -48,17 +52,12 @@ def load_fr_models():
     model_save_path = "fr_neptune/fr_neptune/model.pt"
     fr_neptune.load_state_dict(torch.load(model_save_path, map_location=torch.device('cpu')))
     fr_neptune.to(device)
-    print("Loading speech recognition pipeline...")
-    pipe = pipeline(
-        "automatic-speech-recognition",
-        model="openai/whisper-tiny.en",
-        chunk_length_s=30,
-        device=device,
-    )
-    print(pipe)
-    return fr_model , fr_neptune , pipe
+    print("Loading Wav2Vec2 model for French...")
+    wav2vec2_processor = Wav2Vec2Processor.from_pretrained("bhuang/asr-wav2vec2-french")
+    wav2vec2_model = AutoModelForCTC.from_pretrained("bhuang/asr-wav2vec2-french").to(device)
+    return fr_model, fr_neptune, wav2vec2_processor, wav2vec2_model
 
-fr_class_labels = {0: ('Physics', 'primary', '#0f6fec'), 1: ('AI','cyan', '#0dcaf0'),
+fr_class_labels = {0: ('Physics', 'primary', '#478ce6'), 1: ('AI','cyan', '#0dcaf0'),
                    2: ('economies', 'warning' , '#f7c32e'), 3: ('environments','success' , '#0cbc87'), 
                    4: ('sports', 'orange', '#fd7e14')}
 class_labels = {
@@ -395,7 +394,7 @@ def prepare_fr_text(tokens_splitted: BatchEncoding):
             input_ids_a = [token for token in input_ids_a.view(-1).tolist() if token != pad_token_id]
             input_ids_b = []
             input_ids = [fr_cls_token_id] + [fr_mult_token_id] + input_ids_a + [fr_sep_token_id] + [fr_mult_token_id] + input_ids_b + [fr_sep_token_id]
-            text_input_a = fr_tokenizer.decode(input_ids_a)
+            text_input_a = fr_tokenizer.decode(input_ids_a , skip_special_tokens=True)
             sentences.append(text_input_a)
             segment_ids = [0] * (1 + 1 + len(input_ids_a) + 1) + [1] * (1 + len(input_ids_b) + 1)
             
@@ -426,8 +425,8 @@ def prepare_fr_text(tokens_splitted: BatchEncoding):
             input_ids_b = [token for token in input_ids_b.view(-1).tolist() if token != pad_token_id]
             input_ids = [fr_cls_token_id] + [fr_mult_token_id] + input_ids_a + [fr_sep_token_id] + [fr_mult_token_id] + input_ids_b + [fr_sep_token_id]
             segment_ids = [0] * (1 + 1 + len(input_ids_a) + 1) + [1] * (1 + len(input_ids_b) + 1)
-            text_input_a = fr_tokenizer.decode(input_ids_a)
-            text_input_b = fr_tokenizer.decode(input_ids_b)
+            text_input_a = fr_tokenizer.decode(input_ids_a , skip_special_tokens=True)
+            text_input_b = fr_tokenizer.decode(input_ids_b, skip_special_tokens=True)
             sentences.append(text_input_a)
             sentences.append(text_input_b)
 
@@ -542,4 +541,63 @@ def add_fr_special_tokens_at_beginning_and_end(input_id_chunks: list[Tensor], ma
         # adding CLS (token id 101) and SEP (token id 102) tokens
         input_id_chunks[i] = torch.cat([Tensor([5]), input_id_chunks[i], Tensor([6])])
         # adding attention masks  corresponding to special tokens
-        mask_chunks[i] = torch.cat([Tensor([1]), mask_chunks[i], Tensor([1])])
+        mask_chunks[i] = torch.cat([Tensor([1]), mask_chunks[i], Tensor([1])])
+                
+def transcribe_speech(audio_path, wav2vec2_processor, wav2vec2_model):
+    logging.info(f"Starting transcription of {audio_path}")
+    
+    try:
+        # Try loading with torchaudio first
+        waveform, sample_rate = torchaudio.load(audio_path)
+        waveform = waveform.squeeze().numpy()
+        logging.info(f"Audio loaded with torchaudio. Shape: {waveform.shape}, Sample rate: {sample_rate}")
+    except Exception as e:
+        logging.warning(f"torchaudio failed to load the audio. Trying with soundfile. Error: {str(e)}")
+        try:
+            # If torchaudio fails, try with soundfile
+            waveform, sample_rate = sf.read(audio_path)
+            waveform = torch.from_numpy(waveform).float()
+            logging.info(f"Audio loaded with soundfile. Shape: {waveform.shape}, Sample rate: {sample_rate}")
+        except Exception as e:
+            logging.error(f"Both torchaudio and soundfile failed to load the audio. Error: {str(e)}")
+            raise ValueError("Unable to load the audio file.")
+
+    # Ensure waveform is 1D
+    if waveform.ndim > 1:
+        waveform = np.mean(waveform, axis=0)  # Changed from axis=1 to axis=0
+        logging.info(f"Waveform reduced to 1D. New shape: {waveform.shape}")
+
+    # Resample if necessary
+    if sample_rate != wav2vec2_processor.feature_extractor.sampling_rate:
+        resampler = torchaudio.transforms.Resample(sample_rate, wav2vec2_processor.feature_extractor.sampling_rate)
+        waveform = resampler(torch.from_numpy(waveform).float())
+        logging.info(f"Audio resampled to {wav2vec2_processor.feature_extractor.sampling_rate}Hz")
+
+    # Normalize
+    try:
+        input_values = wav2vec2_processor(waveform, sampling_rate=wav2vec2_processor.feature_extractor.sampling_rate, return_tensors="pt").input_values
+        logging.info(f"Input values shape after processing: {input_values.shape}")
+    except Exception as e:
+        logging.error(f"Error during audio processing: {str(e)}")
+        raise
+
+    # Ensure input_values is 2D (batch_size, sequence_length)
+    input_values = input_values.squeeze()
+    if input_values.dim() == 0:  # If it's a scalar, unsqueeze twice
+        input_values = input_values.unsqueeze(0).unsqueeze(0)
+    elif input_values.dim() == 1:  # If it's 1D, unsqueeze once
+        input_values = input_values.unsqueeze(0)
+    logging.info(f"Final input values shape: {input_values.shape}")
+
+    try:
+        with torch.inference_mode():
+            logits = wav2vec2_model(input_values.to(device)).logits
+        logging.info(f"Model inference successful. Logits shape: {logits.shape}")
+    except Exception as e:
+        logging.error(f"Error during model inference: {str(e)}")
+        raise
+
+    predicted_ids = torch.argmax(logits, dim=-1)
+    predicted_sentence = wav2vec2_processor.batch_decode(predicted_ids)
+    logging.info(f"Transcription complete. Result: {predicted_sentence[0]}")
+    return predicted_sentence[0]

static/css/style2.css

@@ -35,7 +35,7 @@
   --bs-gray-700: #495057;
   --bs-gray-800: #343a40;
   --bs-gray-900: #212529;
-  --bs-primary: #0f6fec;
+  --bs-primary: #478ce6;
   --bs-secondary: #14191e;
   --bs-success: #0cbc87;
   --bs-info: #4f9ef8;
@@ -17332,7 +17332,7 @@ html[data-theme=dark] .light-mode-item {
   border-bottom: 0 !important;
 }
 
-.bg-primary {
+.bg-chat {
   --bs-bg-opacity: 1;
   background-color: #72AB5A !important;
 }

static/js/sentence.js

@@ -182,7 +182,7 @@ function createUserMessageElement(message) {
     userMessageContainer.classList.add('d-flex', 'flex-column', 'align-items-end');
     // Add message content
     var userMessageContent = document.createElement('div');
-    userMessageContent.classList.add('bg-primary', 'text-white', 'p-2', 'px-3', 'rounded-2', 'mw-80');
+    userMessageContent.classList.add('bg-chat', 'text-white', 'p-2', 'px-3', 'rounded-2', 'mw-80');
     var userMessageText = document.createTextNode(message);
     userMessageContent.appendChild(userMessageText);
     userMessageContainer.appendChild(userMessageContent);

static/js/sentence_fr.js

@@ -182,7 +182,7 @@ function createUserMessageElement(message) {
     userMessageContainer.classList.add('d-flex', 'flex-column', 'align-items-end');
     // Add message content
     var userMessageContent = document.createElement('div');
-    userMessageContent.classList.add('bg-primary', 'text-white', 'p-2', 'px-3', 'rounded-2', 'mw-80');
+    userMessageContent.classList.add('bg-chat', 'text-white', 'p-2', 'px-3', 'rounded-2', 'mw-80');
     var userMessageText = document.createTextNode(message);
     userMessageContent.appendChild(userMessageText);
     userMessageContainer.appendChild(userMessageContent);

static/js/voice_fr.js

@@ -0,0 +1,233 @@
+const reset = document.getElementById("reset");
+const currentClassProbabilitiesList = document.getElementById("class-probabilities");
+const currentPredictedClass = document.getElementById('predicted-class');
+const staticDiv = document.getElementById("static");
+const dynamicDiv = document.getElementById("dynamic");
+var chartData;
+
+let mediaRecorder;
+let audioChunks = [];
+
+document.addEventListener('DOMContentLoaded', function() {
+    loadResults();
+    attachEventListeners();
+});
+
+function attachEventListeners() {
+    document.getElementById('startRecord').addEventListener('click', startRecording);
+    document.getElementById('stopRecord').addEventListener('click', stopRecording);
+    document.getElementById('uploadAudio').addEventListener('click', handleAudioUpload);
+}
+
+function initializeChart(data, backgroundColor, borderColor) {
+    const canvas = document.getElementById('bestSellers');
+    
+    // Destroy existing chart if it exists
+    const existingChart = Chart.getChart(canvas);
+    if (existingChart) {
+        existingChart.destroy();
+    }
+
+    // Clear the canvas
+    const context = canvas.getContext('2d');
+    context.clearRect(0, 0, canvas.width, canvas.height);
+
+    data = data.map(function (element) {
+        return parseFloat(element).toFixed(2);
+    });
+
+    new Chart(canvas, {
+        type: 'doughnut',
+        data: {
+            datasets: [{
+                data: data,
+                backgroundColor: backgroundColor,
+                borderColor: borderColor,
+                borderWidth: 1
+                
+            }]
+        },
+        options: {
+            responsive: true,
+            cutout: '80%',
+            plugins: {
+                legend: {
+                    display: true,
+                },
+                tooltip: {
+                    enabled: false
+                }
+            },
+            layout: {
+                padding: 0
+            },
+            elements: {
+                arc: {
+                    borderWidth: 0
+                }
+            },
+            plugins: {
+                datalabels: {
+                    display: false,
+                    align: 'center',
+                    anchor: 'center'
+                }
+            }
+        }
+    });
+}
+
+function loadResults() {
+    fetch('/voice_fr')
+        .then(response => response.text())
+        .then(html => {
+            const responseDOM = new DOMParser().parseFromString(html, "text/html");
+            const classProbabilitiesList = responseDOM.getElementById("class-probabilities");
+            currentClassProbabilitiesList.innerHTML = classProbabilitiesList.innerHTML;
+            const PredictedClass = responseDOM.getElementById("predicted-class")
+            currentPredictedClass.innerHTML = PredictedClass.innerHTML;
+
+            var canvasElement = responseDOM.querySelector('.bestSellers');
+            console.log(canvasElement);
+            chartData = canvasElement.getAttribute('data-chart');
+            console.log(chartData);
+            if (chartData) {
+                var parsedChartData = JSON.parse(chartData);
+                var data = parsedChartData.datasets[0].data.slice(0, 5);
+                var backgroundColor = parsedChartData.datasets[0].backgroundColor.slice(0, 5);
+                var borderColor = parsedChartData.datasets[0].borderColor.slice(0, 5);
+                var labels = parsedChartData.labels.slice(0, 5);
+
+                initializeChart(data, backgroundColor, borderColor, labels);
+            }
+        })
+        .catch(error => console.error('Error:', error));
+}
+
+function startRecording() {
+    navigator.mediaDevices.getUserMedia({ audio: true })
+        .then(stream => {
+            mediaRecorder = new MediaRecorder(stream);
+            mediaRecorder.start();
+
+            audioChunks = [];
+            mediaRecorder.addEventListener("dataavailable", event => {
+                audioChunks.push(event.data);
+            });
+
+            document.getElementById('startRecord').disabled = true;
+            document.getElementById('stopRecord').disabled = false;
+        });
+}
+
+function stopRecording() {
+    mediaRecorder.stop();
+    document.getElementById('startRecord').disabled = false;
+    document.getElementById('stopRecord').disabled = true;
+
+    mediaRecorder.addEventListener("stop", () => {
+        const audioBlob = new Blob(audioChunks, { type: 'audio/wav' });
+        sendAudioToServer(audioBlob);
+    });
+}
+
+function handleAudioUpload() {
+    const fileInput = document.getElementById('audioFileInput');
+    if (fileInput.files.length > 0) {
+        const file = fileInput.files[0];
+        sendAudioToServer(file);
+    } else {
+        console.error('No file selected');
+    }
+}
+
+function sendAudioToServer(audioData) {
+    const formData = new FormData();
+    
+    // Créer un nouveau Blob avec le type MIME audio/wav
+    const audioBlob = new Blob([audioData], { type: 'audio/wav' });
+    
+    formData.append('audio', audioBlob, 'recorded_audio.wav');
+
+    document.getElementById('loadingIndicator').style.display = 'block';
+
+    // Effacer le graphique existant
+    const canvas = document.getElementById('bestSellers');
+    const existingChart = Chart.getChart(canvas);
+    if (existingChart) {
+        existingChart.destroy();
+    }
+    const context = canvas.getContext('2d');
+    context.clearRect(0, 0, canvas.width, canvas.height);
+
+    fetch('/voice_fr', {
+        method: 'POST',
+        body: formData
+    })
+    .then(response => response.text())
+    .then(html => {
+        const parser = new DOMParser();
+        const newDocument = parser.parseFromString(html, 'text/html');
+        
+        // Update other parts of the page as before
+        // Update only the necessary parts of the page
+        document.getElementById('class-probabilities').innerHTML = newDocument.getElementById('class-probabilities').innerHTML;
+        document.getElementById('predicted-class').innerHTML = newDocument.getElementById('predicted-class').innerHTML;
+        document.getElementById('transcribedText').innerHTML = newDocument.getElementById('transcribedText').innerHTML;
+        document.getElementById('classifiedText').innerHTML = newDocument.getElementById('classifiedText').innerHTML;
+        dynamicDiv.classList.remove('d-none');
+        staticDiv.classList.add('d-none');
+        // Update chart
+        const newCanvasElement = newDocument.querySelector('.bestSellers');
+        if (newCanvasElement) {
+            const newChartData = newCanvasElement.getAttribute('data-chart');
+            if (newChartData) {
+                const parsedChartData = JSON.parse(newChartData);
+                initializeChart(
+                    parsedChartData.datasets[0].data.slice(0, 5),
+                    parsedChartData.datasets[0].backgroundColor.slice(0, 5),
+                    parsedChartData.datasets[0].borderColor.slice(0, 5),
+                    parsedChartData.labels.slice(0, 5)
+                );
+            }
+        }
+
+        document.getElementById('loadingIndicator').style.display = 'none';
+    })
+    .catch(error => {
+        console.error('Error:', error);
+        document.getElementById('loadingIndicator').style.display = 'none';
+    });
+}
+fetch('/voice_fr', {
+    method: 'POST',
+    body: formData
+})
+.then(response => response.text())
+.then(html => {
+    const parser = new DOMParser();
+    const newDocument = parser.parseFromString(html, 'text/html');
+    
+    
+    
+    // Update chart
+    const newCanvasElement = newDocument.querySelector('.bestSellers');
+    if (newCanvasElement) {
+        const newChartData = newCanvasElement.getAttribute('data-chart');
+        if (newChartData) {
+            const parsedChartData = JSON.parse(newChartData);
+            initializeChart(
+                parsedChartData.datasets[0].data.slice(0, 5),
+                parsedChartData.datasets[0].backgroundColor.slice(0, 5),
+                parsedChartData.datasets[0].borderColor.slice(0, 5),
+                parsedChartData.labels.slice(0, 5)
+            );
+        }
+    }
+
+    document.getElementById('loadingIndicator').style.display = 'none';
+})
+.catch(error => {
+    console.error('Error:', error);
+    document.getElementById('loadingIndicator').style.display = 'none';
+});

templates/voice_fr.html

@@ -228,7 +228,7 @@
     </div>
 
     <script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
-    <script src="../static/js/voice.js" type="text/javascript"></script>
+    <script src="../static/js/voice_fr.js" type="text/javascript"></script>
     <script src="../static/js/vendor.bundle.base.js"></script>
 </body>